From b0a33bc59db24e42d5dd4594f8ed967e377aff8f Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:30:42 -0700 Subject: [PATCH 01/17] fix: isolate incompatible OpenTelemetry API linkage failures --- .github/workflows/conformance-tests.yml | 1 + .github/workflows/e2e-tests.yml | 1 + .github/workflows/otel-conformance-tests.yml | 6 ++ otel-plugin/README.md | 13 +++ .../durable/otel/OtelPluginSupport.java | 32 +++++++ .../otel/GlobalProviderLinkageTest.java | 54 +++++++++++ .../durable/PluginLinkageIntegrationTest.java | 85 +++++++++++++++++ .../lambda/durable/plugin/PluginRunner.java | 7 +- .../plugin/PluginLinkageErrorTest.java | 91 +++++++++++++++++++ 9 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/GlobalProviderLinkageTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginLinkageIntegrationTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginLinkageErrorTest.java diff --git a/.github/workflows/conformance-tests.yml b/.github/workflows/conformance-tests.yml index 06baa03f5..c066bf277 100644 --- a/.github/workflows/conformance-tests.yml +++ b/.github/workflows/conformance-tests.yml @@ -25,6 +25,7 @@ concurrency: # same stack -- mirrors e2e-tests.yml. group: conformance-tests cancel-in-progress: false + queue: max permissions: contents: read diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index e57070144..5dd246d57 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -26,6 +26,7 @@ on: concurrency: group: e2e-tests cancel-in-progress: false + queue: max # permission can be added at job level or workflow level permissions: diff --git a/.github/workflows/otel-conformance-tests.yml b/.github/workflows/otel-conformance-tests.yml index 68b3bc3f2..87077a82f 100644 --- a/.github/workflows/otel-conformance-tests.yml +++ b/.github/workflows/otel-conformance-tests.yml @@ -53,6 +53,12 @@ on: permissions: {} +# Serialize shared test resources and retain pending PR runs instead of replacing them. +concurrency: + group: otel-conformance-tests + cancel-in-progress: false + queue: max + jobs: opentelemetry: # 1. Run for non-PR events, such as scheduled runs and manual invocations diff --git a/otel-plugin/README.md b/otel-plugin/README.md index c51032aaa..80b37c00a 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -151,6 +151,19 @@ public class MyHandler extends DurableHandler { } ``` +### OpenTelemetry version compatibility + +Keep the OpenTelemetry API, context, SDK, and Java agent versions aligned. This plugin is built and tested against +OpenTelemetry 1.66.0. Global-provider binding needs `GlobalOpenTelemetry.isSet()` and `getOrNoop()`; when the visible +API lacks either method (for example, API 1.49.0), the plugin logs a compatibility diagnostic and disables its telemetry +for that invocation. It does not install a no-op global that would prevent a provider from being registered later. + +The existing 2.x plugin constructors, registration interfaces, and instance lifetime are retained. Nonfatal linkage +errors from plugin callbacks are logged and isolated so healthy plugins and the handler can continue. Fatal JVM errors +and `ThreadDeath` retain their existing propagation behavior. Provider registration and configuration validation remain +unchanged. Align incompatible dependencies to restore instrumentation; error isolation does not make every old +agent/API combination capable of exporting telemetry. + ### 4. Grant Permissions The function's execution role needs the `AWSXRayDaemonWriteAccess` managed policy (or equivalent permissions) to write traces to X-Ray. diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 63fbec908..dea21b447 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -151,6 +151,19 @@ record ProviderSetup(SdkTracerProvider sdkTracerProvider, Tracer tracer) {} * @return the resolved provider and tracer, or {@code null} when telemetry must be disabled for this invocation */ static ProviderSetup tryResolveGlobalProvider(String instrumentationName, String pluginName) { + try { + return resolveGlobalProvider(instrumentationName, pluginName); + } catch (LinkageError error) { + logger.warn( + "{} telemetry is disabled for this invocation because the visible OpenTelemetry dependencies " + + "are incompatible. Align the OpenTelemetry API, SDK, and Java agent versions.", + pluginName, + error); + return null; + } + } + + private static ProviderSetup resolveGlobalProvider(String instrumentationName, String pluginName) { if (!OtelPluginAutoConfigurationState.isInstalled()) { logger.warn( "{} telemetry is disabled for this invocation because " @@ -160,6 +173,9 @@ static ProviderSetup tryResolveGlobalProvider(String instrumentationName, String javaAgentExtensionsDiagnostic()); return null; } + if (!supportsGlobalProviderLookup(pluginName)) { + return null; + } if (!GlobalOpenTelemetry.isSet()) { logger.warn( "{} telemetry is disabled for this invocation because GlobalOpenTelemetry is not initialized yet. " @@ -186,6 +202,22 @@ static ProviderSetup tryResolveGlobalProvider(String instrumentationName, String getSdkTracerProviderForFlush(tracerProvider, pluginName), tracerProvider.get(instrumentationName)); } + private static boolean supportsGlobalProviderLookup(String pluginName) { + try { + GlobalOpenTelemetry.class.getMethod("isSet"); + GlobalOpenTelemetry.class.getMethod("getOrNoop"); + return true; + } catch (NoSuchMethodException missingApi) { + logger.warn( + "{} telemetry is disabled for this invocation because the visible OpenTelemetry API lacks {}. " + + "Global provider binding requires GlobalOpenTelemetry.isSet() and getOrNoop(); " + + "align the API, SDK, and Java agent versions.", + pluginName, + missingApi.getMessage()); + return false; + } + } + /** Returns the SdkTracerProvider for flushing, or null if the provider is wrapped by the agent classloader. */ static SdkTracerProvider getSdkTracerProviderForFlush(TracerProvider tracerProvider, String pluginName) { if (tracerProvider instanceof SdkTracerProvider sdkTracerProvider) { diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/GlobalProviderLinkageTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/GlobalProviderLinkageTest.java new file mode 100644 index 000000000..90e65d6b5 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/GlobalProviderLinkageTest.java @@ -0,0 +1,54 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class GlobalProviderLinkageTest { + @BeforeEach + void markCustomizerInstalled() { + OtelPluginAutoConfigurationState.markInstalled(); + } + + @AfterEach + void resetCustomizer() { + OtelPluginAutoConfigurationState.resetInstalledForTest(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void unavailableGlobalMethodDisablesTelemetryLocally(boolean getterFails) { + try (var global = mockStatic(GlobalOpenTelemetry.class)) { + if (getterFails) { + global.when(GlobalOpenTelemetry::isSet).thenReturn(true); + global.when(GlobalOpenTelemetry::getOrNoop).thenThrow(new NoSuchMethodError("old API getter")); + } else { + global.when(GlobalOpenTelemetry::isSet).thenThrow(new NoSuchMethodError("old API probe")); + } + assertNull(OtelPluginSupport.tryResolveGlobalProvider("scope", "test-plugin")); + global.verify(GlobalOpenTelemetry::get, never()); + } + } + + @Test + void fatalJvmFailureStillEscapesProviderLookup() { + var fatal = new InternalError("fatal JVM failure"); + try (var global = mockStatic(GlobalOpenTelemetry.class)) { + global.when(GlobalOpenTelemetry::isSet).thenThrow(fatal); + assertSame( + fatal, + assertThrows( + InternalError.class, + () -> OtelPluginSupport.tryResolveGlobalProvider("scope", "test-plugin"))); + } + } +} diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginLinkageIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginLinkageIntegrationTest.java new file mode 100644 index 000000000..c90a6ca63 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginLinkageIntegrationTest.java @@ -0,0 +1,85 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.Proxy; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class PluginLinkageIntegrationTest { + @Test + void incompatibleHooksLeaveHealthyPluginsAndReplayWorking() { + var starts = new AtomicInteger(); + var stepCalls = new AtomicInteger(); + var ends = Collections.synchronizedList(new ArrayList()); + var healthy = healthyPlugin(starts, ends); + var config = + DurableConfig.builder().withPlugins(brokenPlugin(), healthy).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> { + var saved = ctx.step("save", String.class, step -> { + stepCalls.incrementAndGet(); + return input; + }); + ctx.wait("pause", Duration.ofMinutes(1)); + return saved; + }, + config); + + assertEquals(ExecutionStatus.PENDING, runner.run("value").getStatus()); + runner.advanceTime(); + var result = runner.run("value"); + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("value", result.getResult(String.class)); + assertEquals(1, stepCalls.get(), "The completed step must not be repeated on resume"); + assertEquals(2, starts.get()); + assertEquals(List.of(InvocationStatus.PENDING, InvocationStatus.SUCCEEDED), ends); + assertSame( + healthy, + config.getPluginRunner().getPlugins().get(1), + "Existing plugin instances and their lifetime are retained"); + } + + private static DurableExecutionPlugin healthyPlugin(AtomicInteger starts, List ends) { + return new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo info) { + starts.incrementAndGet(); + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + ends.add(info.invocationStatus()); + } + }; + } + + private static DurableExecutionPlugin brokenPlugin() { + return (DurableExecutionPlugin) Proxy.newProxyInstance( + DurableExecutionPlugin.class.getClassLoader(), + new Class[] {DurableExecutionPlugin.class}, + (proxy, method, args) -> { + if (method.getDeclaringClass() == Object.class) { + return switch (method.getName()) { + case "toString" -> "incompatible plugin"; + case "hashCode" -> System.identityHashCode(proxy); + default -> proxy == args[0]; + }; + } + throw new NoSuchMethodError("incompatible optional instrumentation API"); + }); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java index e3a5707c4..bc4d8dd1d 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java @@ -11,7 +11,8 @@ /** * Composes multiple {@link DurableExecutionPlugin} instances into a single dispatcher. * - *

Event hooks are fire-and-forget: each plugin is called in order, errors are swallowed. + *

Event hooks call each plugin in order. Exceptions and nonfatal linkage failures are isolated; other errors, + * including fatal JVM failures, retain their existing propagation behavior. * *

{@code onInvocationEnd} is awaited (the SDK blocks until it returns) to allow plugins to flush data before Lambda * freezes. @@ -44,13 +45,15 @@ public List getPlugins() { // ─── Event hooks ───────────────────────────────────────────────────── - /** Calls a void hook on all plugins, swallowing any errors. */ + /** Calls a void hook on all plugins, isolating exceptions and incompatible binary dependencies. */ private void run(Consumer hook) { for (var plugin : plugins) { try { hook.accept(plugin); } catch (Exception e) { logger.warn("Plugin hook threw exception", e); + } catch (LinkageError e) { + logger.warn("Plugin hook could not link a dependency; check SDK/plugin dependency compatibility", e); } } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginLinkageErrorTest.java b/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginLinkageErrorTest.java new file mode 100644 index 000000000..276e60738 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginLinkageErrorTest.java @@ -0,0 +1,91 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.plugin; + +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.Proxy; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class PluginLinkageErrorTest { + @ParameterizedTest(name = "{0}: {1}") + @MethodSource("linkageFailures") + void linkageFailureDoesNotPreventTheNextPlugin( + String hook, String failureName, Consumer dispatch, Error failure) { + var healthyCalls = new AtomicInteger(); + var runner = new PluginRunner(List.of( + plugin(() -> { + throw failure; + }), + plugin(healthyCalls::incrementAndGet))); + + assertDoesNotThrow(() -> dispatch.accept(runner)); + assertEquals(1, healthyCalls.get(), "The next plugin must still receive the hook"); + } + + @ParameterizedTest(name = "{0}: {1}") + @MethodSource("otherErrors") + void otherErrorsRetainTheirExistingPropagation( + String hook, String failureName, Consumer dispatch, Error failure) { + var healthyCalls = new AtomicInteger(); + var runner = new PluginRunner(List.of( + plugin(() -> { + throw failure; + }), + plugin(healthyCalls::incrementAndGet))); + + assertSame(failure, assertThrows(Error.class, () -> dispatch.accept(runner))); + assertEquals(0, healthyCalls.get(), "Fatal and unrelated errors must not be blanket-caught"); + } + + static Stream linkageFailures() { + return hooks().flatMap(hook -> Stream.of( + new NoSuchMethodError("old API"), + new AbstractMethodError("old implementation"), + new NoClassDefFoundError("missing dependency"), + new ExceptionInInitializerError("dependency initialization")) + .map(error -> Arguments.of(hook.name(), error.getClass().getSimpleName(), hook.dispatch(), error))); + } + + static Stream otherErrors() { + return hooks().flatMap(hook -> Stream.of( + new InternalError("fatal JVM failure"), new ThreadDeath(), new AssertionError("unchanged")) + .map(error -> Arguments.of(hook.name(), error.getClass().getSimpleName(), hook.dispatch(), error))); + } + + private static Stream hooks() { + return Stream.of( + new Hook("invocation start", runner -> runner.onInvocationStart(null)), + new Hook("invocation end", runner -> runner.onInvocationEnd(null)), + new Hook("operation start", runner -> runner.onOperationStart(null)), + new Hook("operation end", runner -> runner.onOperationEnd(null)), + new Hook("operation change", runner -> runner.onOperationChange(null)), + new Hook("user function start", runner -> runner.onUserFunctionStart(null)), + new Hook("user function end", runner -> runner.onUserFunctionEnd(null))); + } + + private static DurableExecutionPlugin plugin(Runnable action) { + return (DurableExecutionPlugin) Proxy.newProxyInstance( + DurableExecutionPlugin.class.getClassLoader(), + new Class[] {DurableExecutionPlugin.class}, + (proxy, method, args) -> { + if (method.getDeclaringClass() == Object.class) { + return switch (method.getName()) { + case "toString" -> "test plugin"; + case "hashCode" -> System.identityHashCode(proxy); + default -> proxy == args[0]; + }; + } + action.run(); + return null; + }); + } + + private record Hook(String name, Consumer dispatch) {} +} From b66e5207ac958b82ba86f265f02a132057c8d01f Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:38:07 -0700 Subject: [PATCH 02/17] =?UTF-8?q?Revert=20"Revert=20"refactor(plugin)!:=20?= =?UTF-8?q?create=20one=20plugin=20instance=20per=20invocation=20=E2=80=A6?= =?UTF-8?q?"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 34c174f2819f715cf26e88aa8dbabd7d40cb9ad8. --- .github/workflows/ai-pr-review-address.yml | 5 +- .github/workflows/conformance-tests.yml | 3 +- .github/workflows/e2e-tests.yml | 22 + .github/workflows/otel-conformance-tests.yml | 9 +- AGENTS.md | 1 + README.md | 1 + conformance-tests-otel/README.md | 53 ++ .../otel/Otel21CompletedStepReplay.java | 20 + .../otel/Otel22UserFunctionContext.java | 96 ++ .../otel/Otel23CallbackFunctionContext.java | 109 +++ .../otel/Otel24InvocationRetryStatus.java | 28 + .../otel/OtelConformanceHandler.java | 10 +- conformance-tests-otel/template.yaml | 92 ++ conformance-tests/pom.xml | 2 +- .../java/plugin/PluginAttemptHooksRetry.java | 2 +- .../java/plugin/PluginErrorIsolation.java | 2 +- .../plugin/PluginExternalUpdateOnInvoke.java | 4 +- .../java/plugin/PluginFaultyAndHealthy.java | 2 +- .../plugin/PluginFirstInvocationFlag.java | 2 +- .../plugin/PluginInvocationLifecycle.java | 2 +- .../java/plugin/PluginMultiplePlugins.java | 4 +- .../plugin/PluginNestedParentLinkage.java | 4 +- .../java/plugin/PluginOperationChange.java | 2 +- .../java/plugin/PluginOperationLifecycle.java | 2 +- .../plugin/PluginParallelBranchHooks.java | 4 +- .../main/java/plugin/PluginReplayFlags.java | 4 +- .../java/plugin/PluginRetryExhaustion.java | 2 +- .../plugin/PluginSuspensionInvocationEnd.java | 4 +- .../java/plugin/PluginTerminalFailure.java | 2 +- .../java/plugin/PluginTerminalPayloads.java | 2 +- .../java/plugin/PluginWaitOperationHooks.java | 4 +- .../java/plugin/PluginWaitReplayFlag.java | 4 +- coverage-report/pom.xml | 2 +- docs/advanced/configuration.md | 18 +- docs/migration-2.x-to-3.x.md | 433 +++++++++ examples/pom.xml | 2 +- .../examples/general/PluginExample.java | 2 +- insight-plugin/README.md | 25 +- insight-plugin/pom.xml | 2 +- .../lambda/durable/insight/ContentConfig.java | 4 +- .../durable/insight/ExportScheduler.java | 900 ++++++++++++++++-- .../durable/insight/InsightExporter.java | 29 +- .../lambda/durable/insight/InsightPlugin.java | 458 +++++++++ .../durable/insight/InsightSettings.java | 59 ++ .../amazon/lambda/durable/insight/Json.java | 6 + .../durable/insight/WorkflowInsight.java | 447 ++------- .../durable/insight/exporters/LazyClient.java | 2 + .../durable/insight/internal/FatalErrors.java | 51 + .../ConcurrentExecutionsExportTest.java | 409 ++++++++ .../durable/insight/ErrorPrivacyGateTest.java | 14 +- .../lambda/durable/insight/Executions.java | 77 ++ .../ExportSchedulerFanOutReentryTest.java | 199 ++++ .../ExportSchedulerFlushCoalescingTest.java | 311 ++++++ ...ExportSchedulerFlushSerializationTest.java | 434 +++++++++ .../ExportSchedulerReentrantFlushTest.java | 131 +++ .../durable/insight/ExportSchedulerTest.java | 122 ++- .../insight/ExporterIsolationTest.java | 19 +- .../durable/insight/InputSnapshotTest.java | 25 +- .../insight/InsightFatalBoundaryTest.java | 237 +++++ .../insight/InsightFatalFailureTest.java | 172 ++++ .../durable/insight/JsonJavaTimeTest.java | 11 +- .../insight/MutableNumberIsolationTest.java | 13 +- .../insight/OperationErrorIdentityTest.java | 15 +- .../insight/OperationOrderingTest.java | 11 +- .../PluginThrowableContainmentTest.java | 109 ++- .../insight/RecordSupersessionTest.java | 282 ++++++ .../insight/StateCleanupLifecycleTest.java | 236 ++++- .../insight/TransformContractTest.java | 32 +- .../insight/UnrecoverableErrorUnwrapTest.java | 26 +- .../WorkflowInsightFlushCadenceTest.java | 230 +++++ .../insight/WorkflowInsightHookTest.java | 95 +- .../insight/WorkflowInsightPluginTest.java | 4 +- .../insight/exporters/LazyClientTest.java | 14 + otel-plugin/README.md | 141 ++- otel-plugin/pom.xml | 2 +- .../lambda/durable/otel/ContextExtractor.java | 19 +- .../lambda/durable/otel/DurableSampler.java | 2 +- .../durable/otel/DurableSamplingDecision.java | 39 +- .../durable/otel/ExecutionOtelPlugin.java | 361 ++++--- .../otel/ExecutionOtelPluginProvider.java | 20 +- .../durable/otel/ExecutionTraceContext.java | 5 +- .../durable/otel/InvocationOtelPlugin.java | 359 ++++--- .../otel/InvocationOtelPluginProvider.java | 20 +- .../lambda/durable/otel/OtelPluginConfig.java | 6 +- .../durable/otel/OtelPluginEnvironment.java | 104 ++ .../durable/otel/OtelPluginSupport.java | 38 +- .../lambda/durable/otel/SpanAttributes.java | 2 + .../durable/otel/XRayContextExtractor.java | 40 +- .../PluginLayerCompatibilityProbe.java | 76 ++ .../src/test/compatibility/run_matrix.py | 76 ++ .../durable/OtelViewRegistrationTest.java | 199 ++++ .../lambda/durable/otel/DelayedFlush.java | 57 ++ .../durable/otel/DurableSamplerTest.java | 40 +- ...urableSamplingDecisionClassLoaderTest.java | 5 +- .../ExecutionOtelPluginIntegrationTest.java | 10 +- .../durable/otel/ExecutionOtelPluginTest.java | 179 ++-- .../durable/otel/ExecutionRootTest.java | 362 +++++++ .../otel/HandlerContextIntegrationTest.java | 268 ++++++ .../otel/HandlerMdcIntegrationTest.java | 185 ++++ .../otel/HandlerScopeFinalizationTest.java | 456 +++++++++ .../otel/InvocationEndCleanupTest.java | 76 ++ .../durable/otel/InvocationHeaderTest.java | 171 ++++ .../InvocationInfoHeaderIntegrationTest.java | 153 +++ .../otel/InvocationMdcCleanupTest.java | 122 +++ .../InvocationOtelPluginIntegrationTest.java | 23 +- .../otel/InvocationOtelPluginTest.java | 274 +++--- .../lambda/durable/otel/Invocations.java | 27 + .../otel/LegacyEndThreadProbeTest.java | 188 ++++ .../otel/LegacySubclassMigrationTest.java | 81 ++ .../durable/otel/MdcSpanEnricherTest.java | 44 +- .../otel/OtelPluginFactoryConfigTest.java | 63 ++ .../durable/otel/PluginStartupOrderTest.java | 64 ++ .../otel/RootStartProcessorIsolationTest.java | 187 ++++ .../RuntimeHeaderFailureIntegrationTest.java | 140 +++ .../otel/UserFunctionStartCleanupTest.java | 145 +++ .../otel/XRayContextExtractorTest.java | 89 ++ .../XRayExtractorBinaryCompatibilityTest.java | 85 ++ pom.xml | 2 +- sdk-integration-tests/pom.xml | 2 +- ...DelayedCheckpointFatalIntegrationTest.java | 138 +++ ...ndlerScopeFinalizationIntegrationTest.java | 192 ++++ ...cationEndFatalDispatchIntegrationTest.java | 190 ++++ .../InvocationFailureCompatibilityTest.java | 97 ++ .../OperationHookFatalIntegrationTest.java | 314 ++++++ .../OperationWorkerFailureRegressionTest.java | 313 ++++++ .../PluginFatalDispatchIntegrationTest.java | 251 +++++ .../PluginFatalQueuedCheckpointTest.java | 142 +++ .../lambda/durable/PluginIntegrationTest.java | 241 ++++- sdk-testing/pom.xml | 2 +- .../testing/LocalDurableTestRunner.java | 11 +- .../TestingSdkCompatibilityProbe.java | 17 + .../ConfigurationCopyCompatibilityTest.java | 46 + .../testing/LocalDurableTestRunnerTest.java | 3 +- sdk/pom.xml | 2 +- .../amazon/lambda/durable/DurableConfig.java | 70 +- .../lambda/durable/DynamicPluginLoader.java | 193 +++- .../execution/ApiRequestDelayedBatcher.java | 86 +- .../durable/execution/CheckpointManager.java | 66 +- .../durable/execution/DurableExecutor.java | 290 +++++- .../durable/execution/ExecutionManager.java | 152 ++- .../durable/execution/RuntimeTraceHeader.java | 68 ++ .../operation/BaseDurableOperation.java | 40 +- .../operation/ChildContextOperation.java | 1 + .../durable/operation/StepOperation.java | 1 + .../operation/WaitForConditionOperation.java | 1 + .../plugin/DurableExecutionPlugin.java | 11 +- .../plugin/DurableExecutionPluginFactory.java | 41 + .../DurableExecutionPluginProvider.java | 36 +- .../lambda/durable/plugin/HandlerScoped.java | 30 + .../lambda/durable/plugin/InvocationInfo.java | 30 +- .../lambda/durable/plugin/PluginRunner.java | 348 ++++++- .../lambda/durable/util/ExceptionHelper.java | 17 + .../lambda/durable/DurableConfigTest.java | 60 +- .../DynamicPluginLoaderStaleProviderTest.java | 561 +++++++++++ .../durable/DynamicPluginLoaderTest.java | 267 +++--- .../ApiRequestDelayedBatcherTest.java | 120 +++ .../execution/DurableExecutionTest.java | 57 +- .../execution/ExecutionManagerTest.java | 64 +- .../FatalInvocationBoundaryTest.java | 294 ++++++ .../execution/HandlerScopeHandoffTest.java | 100 ++ .../PluginFatalCheckpointAbortTest.java | 158 +++ .../execution/RuntimeTraceHeaderTest.java | 256 +++++ .../BaseDurableOperationPluginTest.java | 21 +- .../durable/plugin/HandlerScopeTest.java | 253 +++++ .../InvocationInfoCompatibilityTest.java | 189 ++++ .../LegacyHandlerScopeCompatibilityTest.java | 87 ++ .../durable/plugin/PluginRunnerTest.java | 660 ++++++++++++- .../plugin/UserFunctionStartFailureTest.java | 129 +++ .../util/AsyncFailureUnwrappingTest.java | 36 + 169 files changed, 16167 insertions(+), 1763 deletions(-) create mode 100644 conformance-tests-otel/README.md create mode 100644 conformance-tests-otel/src/main/java/software/amazon/lambda/durable/conformance/otel/Otel21CompletedStepReplay.java create mode 100644 conformance-tests-otel/src/main/java/software/amazon/lambda/durable/conformance/otel/Otel22UserFunctionContext.java create mode 100644 conformance-tests-otel/src/main/java/software/amazon/lambda/durable/conformance/otel/Otel23CallbackFunctionContext.java create mode 100644 conformance-tests-otel/src/main/java/software/amazon/lambda/durable/conformance/otel/Otel24InvocationRetryStatus.java create mode 100644 docs/migration-2.x-to-3.x.md create mode 100644 insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightPlugin.java create mode 100644 insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightSettings.java create mode 100644 insight-plugin/src/main/java/software/amazon/lambda/durable/insight/internal/FatalErrors.java create mode 100644 insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ConcurrentExecutionsExportTest.java create mode 100644 insight-plugin/src/test/java/software/amazon/lambda/durable/insight/Executions.java create mode 100644 insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFanOutReentryTest.java create mode 100644 insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushCoalescingTest.java create mode 100644 insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushSerializationTest.java create mode 100644 insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerReentrantFlushTest.java create mode 100644 insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InsightFatalBoundaryTest.java create mode 100644 insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InsightFatalFailureTest.java create mode 100644 insight-plugin/src/test/java/software/amazon/lambda/durable/insight/RecordSupersessionTest.java create mode 100644 insight-plugin/src/test/java/software/amazon/lambda/durable/insight/WorkflowInsightFlushCadenceTest.java create mode 100644 otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginEnvironment.java create mode 100644 otel-plugin/src/test/compatibility/PluginLayerCompatibilityProbe.java create mode 100644 otel-plugin/src/test/compatibility/run_matrix.py create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/OtelViewRegistrationTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DelayedFlush.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerContextIntegrationTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerMdcIntegrationTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerScopeFinalizationTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndCleanupTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationHeaderTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationInfoHeaderIntegrationTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationMdcCleanupTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/Invocations.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacyEndThreadProbeTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacySubclassMigrationTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginFactoryConfigTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PluginStartupOrderTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RuntimeHeaderFailureIntegrationTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionStartCleanupTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayExtractorBinaryCompatibilityTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/DelayedCheckpointFatalIntegrationTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/HandlerScopeFinalizationIntegrationTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/InvocationEndFatalDispatchIntegrationTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/InvocationFailureCompatibilityTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/OperationHookFatalIntegrationTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/OperationWorkerFailureRegressionTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalDispatchIntegrationTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalQueuedCheckpointTest.java create mode 100644 sdk-testing/src/test/compatibility/TestingSdkCompatibilityProbe.java create mode 100644 sdk-testing/src/test/java/software/amazon/lambda/durable/testing/ConfigurationCopyCompatibilityTest.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/execution/RuntimeTraceHeader.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPluginFactory.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/plugin/HandlerScoped.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/DynamicPluginLoaderStaleProviderTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/execution/FatalInvocationBoundaryTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/execution/HandlerScopeHandoffTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/execution/PluginFatalCheckpointAbortTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/execution/RuntimeTraceHeaderTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/plugin/HandlerScopeTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/plugin/InvocationInfoCompatibilityTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/plugin/LegacyHandlerScopeCompatibilityTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/plugin/UserFunctionStartFailureTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/util/AsyncFailureUnwrappingTest.java diff --git a/.github/workflows/ai-pr-review-address.yml b/.github/workflows/ai-pr-review-address.yml index 2b6e25d79..5bd13defb 100644 --- a/.github/workflows/ai-pr-review-address.yml +++ b/.github/workflows/ai-pr-review-address.yml @@ -23,7 +23,10 @@ jobs: contents: read issues: read pull-requests: read - uses: aws/aws-durable-execution-ci/.github/workflows/ai-pr-review-address.yml@d6b017da14385908951d23e26c790b28a4e5f9f0 + uses: aws/aws-durable-execution-ci/.github/workflows/ai-work-item-resolver.yml@d6b017da14385908951d23e26c790b28a4e5f9f0 + with: + work-scope: review + upload-work-items: true address: if: >- diff --git a/.github/workflows/conformance-tests.yml b/.github/workflows/conformance-tests.yml index 06baa03f5..357fa8f97 100644 --- a/.github/workflows/conformance-tests.yml +++ b/.github/workflows/conformance-tests.yml @@ -21,10 +21,11 @@ on: concurrency: # Runs share fixed per-suite stack names, so serialize the whole workflow - # (queue, don't cancel) to avoid concurrent CloudFormation updates on the + # with multiple pending runs to avoid concurrent CloudFormation updates on the # same stack -- mirrors e2e-tests.yml. group: conformance-tests cancel-in-progress: false + queue: max permissions: contents: read diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index e57070144..36f1341e3 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -24,8 +24,10 @@ on: - 'pom.xml' concurrency: + # Shared stacks require serial deployment; preserve waiting runs from other PRs. group: e2e-tests cancel-in-progress: false + queue: max # permission can be added at job level or workflow level permissions: @@ -148,6 +150,26 @@ jobs: if [[ "$invariant_violated" == true ]]; then exit 1 fi + - name: Collect Lambda errors after failed E2E tests + if: failure() && env.E2E_LOG_START_TIME_MS != '' + env: + E2E_STACK_NAME: Java${{ matrix.java }}-JavaSDKCloudBasedIntegrationTestStack + run: | + set -euo pipefail + log_groups=$(aws cloudformation list-stack-resources \ + --stack-name "$E2E_STACK_NAME" \ + --query "StackResourceSummaries[?ResourceType=='AWS::Logs::LogGroup'].PhysicalResourceId" \ + --output text) + for log_group in $log_groups; do + echo "::group::Lambda errors: $log_group" + aws logs filter-log-events \ + --log-group-name "$log_group" \ + --start-time "$E2E_LOG_START_TIME_MS" \ + --filter-pattern '%ERROR|Error|Exception|timed.out|Invalid.suspension|not.active|already.registered%' \ + --limit 100 --no-paginate \ + --query 'events[].message' --output text + echo "::endgroup::" + done - name: Publish test case summary if: always() env: diff --git a/.github/workflows/otel-conformance-tests.yml b/.github/workflows/otel-conformance-tests.yml index 68b3bc3f2..f77472cd0 100644 --- a/.github/workflows/otel-conformance-tests.yml +++ b/.github/workflows/otel-conformance-tests.yml @@ -53,6 +53,11 @@ on: permissions: {} +concurrency: + group: otel-conformance-tests + cancel-in-progress: false + queue: max + jobs: opentelemetry: # 1. Run for non-PR events, such as scheduled runs and manual invocations @@ -62,13 +67,13 @@ jobs: actions: write contents: read id-token: write - uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@43872f8d5dd917fe3ee6d01a6c953c1ff67c5fe4 + uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@98b802cdb172614f98e217f7464784d47b9bb484 with: language: java resource_prefix: j sdk_repository: aws/aws-durable-execution-sdk-java sdk_ref: ${{ github.event.pull_request.head.sha || github.sha }} - conformance_test_ref: ${{ inputs.conformance_test_ref || '02d6dca971a38c13d94d6233d12f687e55b2a572' }} + conformance_test_ref: ${{ inputs.conformance_test_ref || '98b802cdb172614f98e217f7464784d47b9bb484' }} checkout_sdk: true # Build the handlers from this repo's checked-out module instead of the conformance repo's # bundled examples/java. Path is relative to the conformance workspace where the SDK is diff --git a/AGENTS.md b/AGENTS.md index 7894488db..6cc0114c9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -335,6 +335,7 @@ Run `mvn spotless:apply` after Java changes. Then run the narrowest relevant tes - [Error Handling](docs/advanced/error-handling.md) - [Logging](docs/advanced/logging.md) - [Migration from 1.x to 2.x](docs/migration-1.x-to-2.x.md) +- [Migration from 2.x to 3.x](docs/migration-2.x-to-3.x.md) ### Official AWS SDKs diff --git a/README.md b/README.md index 8b3184c63..c7e14c3d3 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ See [Deploy Lambda durable functions with Infrastructure as Code](https://docs.a - [Error Handling](docs/advanced/error-handling.md) - SDK exceptions for handling failures - [Logging](docs/advanced/logging.md) - How to use DurableLogger - [Migrating from 1.x to 2.x](docs/migration-1.x-to-2.x.md) - Upgrade guide for breaking changes since `v1.2.1` +- [Migrating from 2.x to 3.x](docs/migration-2.x-to-3.x.md) - Upgrade guide for the factory-only, per-invocation plugin contract - [Release Process](RELEASE.md) - Prepare and publish Maven releases - [Testing](docs/advanced/testing.md) - Utilities for local development and cloud-based integration testing diff --git a/conformance-tests-otel/README.md b/conformance-tests-otel/README.md new file mode 100644 index 000000000..7f4c31312 --- /dev/null +++ b/conformance-tests-otel/README.md @@ -0,0 +1,53 @@ +# Java OpenTelemetry conformance handlers + +The shared conformance repository owns the requirements and validators. This +module supplies public-API handlers and SAM resources for both tracing views. +Existing cases 1–20 and their resources are unchanged by the additions below. + +| Case | Scenario | Behavior | +| --- | --- | --- | +| 21 | `completed-step-replay` | Complete a step, suspend on a one-second durable wait, then complete another step. The first step body is skipped on replay. | +| 22 | `user-function-context` | Probe handler entry/restoration/resume, step, child, concurrent parallel branches and map iterations, and their nested steps. | +| 23 | `callback-function-context` | Probe step retry attempts, condition checks, callback submission, a wrapped retry helper's body and strategy, and a virtual child after the asynchronous work finishes. | +| 24 | `invocation-retry-status` | Throw the public retryable execution exception after a checkpointed step, then recover on replay. | + +Cases 22 and 23 require a valid active `SpanContext` and create/end ordinary +`conformance.

Each {@link WorkflowInsightRecord} is a complete snapshot of its execution, so a newer record for the - * same execution fully supersedes any record of that execution still waiting to be exported. While an export is in - * flight, additional updates are coalesced into a single pending slot per execution — intermediate records are dropped - * because the latest one already contains all of their information. Records of different executions never displace each - * other: one execution's snapshot carries none of another's information, so each execution keeps its own slot and the - * slots are served in the order the executions first became pending. This prevents overlapping {@code export()} calls - * when updates arrive faster than the exporters can keep up, and it keeps exporter I/O off the SDK threads that deliver - * plugin hooks. + *

One scheduler serves the whole execution environment — the exporters it fans out to belong to the environment, and + * an environment can host several durable executions at the same time (Lambda Managed Instances makes that routine). + * The work, by contrast, is held on the invocation it belongs to: each {@link InsightPlugin} instance is one + * invocation's plugin and owns that invocation's latest-record slot, drain signal, mid-export marker and drain-waiter + * count. Coalescing happens only within one invocation, because the slot belongs to it. * - *

Exports are otherwise fire-and-forget; {@link #drain()} is called before the invocation returns to guarantee the - * final record is delivered. + *

That shape is not a tidying-up. The same facts once lived in five structures keyed by execution ARN — the plugin's + * state map plus this class's {@code pending}, {@code settled}, {@code exporting} and {@code drainWaiters} — and two of + * them could disagree about one execution. They did: "nothing queued for this ARN" was read as "nothing outstanding", + * which is equally true of a record already taken and being exported, so an exiting pump completed another execution's + * drain signal mid-export and that invocation returned before its record was delivered. Every fact now exists exactly + * once, as a field of the one object the SDK gave that invocation, so the disagreement has no place to happen — and + * since the SDK creates that object and drops it, the scheduler has no execution registry to keep in step with + * anything. + * + *

Each {@link WorkflowInsightRecord} is a complete snapshot of its execution, so a newer record from the same + * invocation fully supersedes one still waiting to be exported. While an export is in flight, additional updates from + * that invocation are coalesced into its slot — intermediate records are dropped because the latest one already + * contains all of their information. A record from a different invocation never displaces another's record. + * + *

The slot takes whichever record is handed to it last and compares nothing, so "newer" has to be established before + * the hand-off. Customer code runs while a record is being built and can re-enter a hook of the same invocation, which + * builds and hands over a newer record first; the build it re-entered from then hands over an older snapshot last. + * {@code InsightPlugin}'s build revision identifies each build and {@link #scheduleIfNotSuperseded} drops a record + * whose build has been overtaken, so the slot only ever advances. + * + *

A single pump exports the queued records one at a time, in the order the invocations first queued work + * ({@link #queue}, which is ordering only — membership in it is the same fact as "this invocation has a record", + * written in one place), so exporters still never see two exports at once and each record keeps its per-exporter + * fan-out. {@link #flush()} requests are served by that same pump, between records, so an exporter never sees a + * {@code flush()} overlap an {@code export()} either. Requests are served as a batch — the cadence is at most one flush + * per requesting invocation end, not exactly one — and a flush is preceded by the queued records a drain is waiting + * for, so a burst of invocation ends is covered by one flush rather than one each. A request made while a flush runs + * waits for the next turn. Exports are otherwise fire-and-forget; {@link #drain(InsightPlugin)} is called before an + * invocation returns and waits for that invocation's own latest record to reach every exporter. + * + *

Because there is one pump, that wait can also cover records other invocations had already queued ahead of this + * one: a drain is not isolated from the queue's head-of-line cost. What per-invocation ownership guarantees is that + * another invocation's record can never displace this one — records coalesce only within their own invocation, + * and a drain cannot return until its own latest record has reached every exporter. */ final class ExportScheduler { private static final AtomicInteger THREAD_NUMBER = new AtomicInteger(); + /** + * Upper bound on the passes {@link #drainAll()} makes over the outstanding invocations. Only reached if new work + * keeps arriving for as long as the drain runs; a normal drain settles in two passes. + */ + private static final int MAX_DRAIN_ALL_PASSES = 1_000; + + /** How long one {@link #drainAll()} pass waits for a running pump before taking another pass. */ + private static final long PUMP_WAIT_MILLIS = 50; + /** Shared for the process lifetime; idle daemon workers are reclaimed, so nothing keeps the runtime alive. */ private static final ExecutorService WORKERS = Executors.newCachedThreadPool(runnable -> { var thread = new Thread(runnable, "workflow-insight-export-" + THREAD_NUMBER.incrementAndGet()); @@ -48,11 +93,68 @@ final class ExportScheduler { /** Completes when the current pump finishes; {@code null} while idle. Guarded by {@code this}. */ private CompletableFuture inFlight; + // A fatal worker failure is terminal for this shared scheduler. Keep it observable even + // after its pump/owner signals have been released, and wake fan-out joins immediately. + private volatile Error fatalFailure; + private final CompletableFuture fatalSignal = new CompletableFuture<>(); + private InsightPlugin activeExecution; // guarded by this monitor + + /** + * The thread serving the pump right now, or {@code null} while no pump is running. Deliberately not + * guarded by {@code this}: it is read by {@link #flush()} and the drains before they touch anything else, and a + * lock acquisition there would put the pump's own monitor on the path of every invocation end. + * + *

Written only by the thread that enters {@link #pump} — a worker, or the caller's own thread on the + * rejected-worker fallback, which is a case where the calling thread genuinely is the pump — and cleared + * by that same thread on the way out, only if it is still the recorded one. A compare-and-set on the way out rather + * than a blind clear: if some anomaly ever did leave two pumps running, the one that finishes first must not clear + * the other, and must not leave a stale thread behind that a later, legitimate {@code flush()} from that same + * thread would be mistaken for. + */ + private final AtomicReference pumpThread = new AtomicReference<>(); + + /** + * Marks the thread currently running one exporter's share of a fan-out for this scheduler, so that a + * {@code flush()} or {@code drain()} re-entered from an exporter callback can tell that the pump is waiting for it. + * + *

With a single exporter the fan-out runs on the pump thread and {@link #pumpThread} already recognizes it. With + * two or more, {@link #forEachExporterSettled} submits one task per exporter and the pump then joins them all, so + * the callback runs on a thread that is not the pump but that the pump cannot outlive: a wait for the pump issued + * from there is the same wait-for cycle, two threads wide instead of one. The pump parks in the join, so it never + * reaches the point in its loop that would complete the future the worker is parked on. + * + *

An instance field rather than a static: a fan-out worker of one scheduler is not pump-dependent on any other + * scheduler, and refusing its waits there would be a false positive. Set and cleared around each callback by the + * thread that runs it, restoring whatever was there before rather than blindly removing, so a callback that the + * pump ran inline (the rejected-worker fallback, where the fan-out thread is the pump) cannot clear a mark + * an enclosing frame still needs. + */ + private final ThreadLocal exporterFanOutThread = new ThreadLocal<>(); + + /** + * The invocations with a record no pump has picked up yet, in the order they first queued work. Ordering only — the + * record itself lives on the invocation's plugin instance. Guarded by {@code this}. + * + *

This is the only collection of per-invocation objects the scheduler has, and it holds an instance for exactly + * as long as that instance has a record waiting: nothing here has to be cleaned up at an invocation boundary, and + * an instance the SDK has dropped is unreachable from the scheduler the moment its last record is taken. + * + *

Invariant, and the only thing that could still be said twice: an invocation is in here exactly while its + * {@link InsightPlugin#record} is non-null. Every record moves through {@link #queueRecord}, {@link #takeRecord} or + * {@link #dropRecord}, which write both halves together, and a set makes a double entry impossible by construction. + */ + private final Set queue = new LinkedHashSet<>(); + /** - * The latest record of each execution not yet picked up by the pump, keyed by execution ARN and served in the order - * the executions first became pending. Guarded by {@code this}. + * One entry per outstanding {@link #flush()} request, in request order, completed when a {@code flush()} that + * started after that request was enqueued has reached every exporter. Guarded by {@code this}. + * + *

A queue of requests rather than a single flag: the pump takes the requests that are queued when its turn + * begins and satisfies all of them with one flush, so concurrent invocation ends share a flush; a request enqueued + * while that flush runs stays in the queue for the next turn, because a flush already in progress cannot be shown + * to have seen the new requester's records. */ - private final LinkedHashMap pending = new LinkedHashMap<>(); + private final Deque> flushRequests = new ArrayDeque<>(); ExportScheduler( List exporters, @@ -72,53 +174,214 @@ final class ExportScheduler { this.executor = executor; } + // --- Scheduling. --- + /** - * Queues the latest record of its execution for export. If an export is already running, the record is held in that - * execution's pending slot (replacing any earlier pending record of the same execution) and exported once the - * in-flight export completes. + * Queues the latest record of one invocation for export, with no ordering check. If an export is already running, + * the record is held in that invocation's own slot (replacing only an earlier record of the same + * invocation) and exported once the pump reaches it. + * + *

The slot takes whichever record is handed over last and does not compare record ages, so this is the right + * entry point only for a record that cannot be superseded. The plugin's RUNNING records go through + * {@link #scheduleIfNotSuperseded} and its final record through {@link #closeAndSchedule}; both add the ordering + * checks this one omits. */ - void schedule(WorkflowInsightRecord record) { + void schedule(InsightPlugin execution, WorkflowInsightRecord record) { CompletableFuture handle; synchronized (this) { - // put() on an existing key keeps its position, so a chatty execution cannot jump ahead of a quieter one. - pending.put(executionKey(record), record); - if (inFlight != null) { - return; + queueRecord(execution, record); + handle = claimPumpIfIdle(); + } + startPump(handle); + } + + /** + * Schedules a non-terminal record unless it has been superseded, which is two separate facts. + * + *

The invocation may already have ended. No RUNNING snapshot may follow the final record, so + * {@link InsightPlugin#closed} rejects it. + * + *

A newer build of this same invocation may already have started. Customer code runs inside a build — the + * content transforms, an operation result transform, a serializer for a customer type — and can re-enter a hook, so + * the build that hands its record over last is not necessarily the build that started last. Without the revision + * check the slot would take that older snapshot and the newer one would be lost, or, if a pump had already taken + * the newer one, an exporter would see the older snapshot after the newer one. + * + *

The superseded record is dropped rather than queued. Nothing is lost: a record is a complete snapshot of one + * execution, so the record that superseded it carries everything it carries. That is the same property that makes + * the slot's coalescing sound. + * + *

Both checks and the hand-off are one critical section, on the monitor that owns both fields, so a record + * cannot pass the checks and then be queued after the record that supersedes it. + * + * @param buildRevision the revision the caller took before it started building this record + * @return whether the record was queued + */ + boolean scheduleIfNotSuperseded(InsightPlugin execution, WorkflowInsightRecord record, long buildRevision) { + CompletableFuture handle; + synchronized (this) { + if (execution.closed || !execution.isNewestBuild(buildRevision)) { + return false; } - handle = new CompletableFuture<>(); - inFlight = handle; + queueRecord(execution, record); + handle = claimPumpIfIdle(); + } + startPump(handle); + return true; + } + + /** + * Marks the invocation ended and, when given a record, schedules it as the last one for that invocation. + * + *

The final record is queued without the build-revision check {@link #scheduleIfNotSuperseded} makes. Customer + * code running inside the final record's build can start a newer RUNNING build, which would leave the final + * record's revision stale, and a checked hand-off would then drop it and leave a RUNNING snapshot as the + * execution's last exported state. Exempting it cannot let a stale record win, because {@code closed} is set in + * this same critical section and every RUNNING record handed over afterwards is rejected. + */ + void closeAndSchedule(InsightPlugin execution, WorkflowInsightRecord finalRecord) { + if (finalRecord == null && execution.closed) { + // The idempotent second call from the hook's `finally`. A volatile read, so the common case of an + // invocation end that already scheduled its final record does not take the lock again. + return; + } + CompletableFuture handle = null; + synchronized (this) { + execution.closed = true; + if (finalRecord != null) { + queueRecord(execution, finalRecord); + handle = claimPumpIfIdle(); + } + } + startPump(handle); + } + + /** + * Claims the pump for the caller when none is running, returning the handle to run with, or {@code null} when a + * pump already owns the scheduler and will pick the work up. Caller holds the lock. + */ + private CompletableFuture claimPumpIfIdle() { + if (inFlight != null) { + return null; + } + CompletableFuture handle = new CompletableFuture<>(); + inFlight = handle; + return handle; + } + + /** Starts a claimed pump on a worker; a no-op when the caller claimed nothing. */ + private void startPump(CompletableFuture handle) { + if (handle == null) { + return; } try { executor.execute(() -> pump(handle)); } catch (Throwable t) { - // No worker could be started. Keep the pending record and return to idle so a later schedule() retries, - // and drain() runs whatever is still pending on the calling thread before the invocation returns. Complete - // the handle too: a drain() that already observed it must wake up and take that inline path. + rejectFatal(t); + // No worker could be started. Keep the queued record and return to idle so a later schedule() retries, and + // a drain runs whatever is still queued on the calling thread before the invocation returns. Complete the + // handle too: a drain that already observed it must wake up and take that inline path. synchronized (this) { if (inFlight == handle) { inFlight = null; } } - handle.complete(null); + completeSignal(handle); reportFailure(t); } } + // --- The record slot: the two halves of "this invocation has a record queued", always written together. --- + + /** Puts this invocation's latest record in its slot and makes sure it has a drain signal. Caller holds the lock. */ + private void queueRecord(InsightPlugin execution, WorkflowInsightRecord record) { + throwIfFailed(); + execution.record = record; + queue.add(execution); + if (execution.settled == null) { + execution.settled = new CompletableFuture<>(); + } + } + + /** + * Takes this invocation's queued record for export and marks it mid-export. Caller holds the lock and has checked + * that a record is there. + * + *

The marking is not a separate step in a separate structure: leaving the slot and becoming "inside the + * exporters" are one write of one object, so no reader can see the invocation between the two and conclude it has + * nothing outstanding. + */ + private WorkflowInsightRecord takeRecord(InsightPlugin execution) { + WorkflowInsightRecord record = execution.record; + execution.record = null; + queue.remove(execution); + execution.exporting = true; + activeExecution = execution; + return record; + } + + /** Drops this invocation's queued record without exporting it. Caller holds the lock. */ + private void dropRecord(InsightPlugin execution) { + execution.record = null; + queue.remove(execution); + } + + // --- Draining. --- + /** - * Waits for any in-flight and pending exports to complete. Safe to call when idle. Used before the invocation - * returns to guarantee the final record is delivered. + * Waits until the latest record of one invocation has been handed to every exporter. Safe to call when that + * invocation has nothing outstanding. Used before the invocation returns to guarantee the final record is + * delivered. + * + *

The wait is for that invocation's own latest record. Another invocation's record can never displace it, so + * this always returns having delivered this invocation's latest snapshot; but since one pump exports serially, the + * wait can also cover records other invocations had already queued ahead of it. + * + *

While this waits, the invocation counts a drain waiter — on the instance itself, so the count cannot come to + * describe a different one. It tells the pump that this record gates an invocation return, so the pump exports it + * before spending a flush fan-out. See {@link #exportRecordsADrainIsWaitingFor}. + * + *

Called from the pump thread itself, or from an exporter fan-out worker that pump is waiting for, the wait is + * refused and reported instead of made: see {@link #refuseWaitThatWouldBlockThePump}. */ - void drain() { + void drain(InsightPlugin execution) { + throwIfFailed(); + // Re-entered from a thread the pump's progress depends on: waiting here would park on a signal only that pump + // can settle. Refuse and return; the record stays queued and that same pump exports it once it resumes its + // loop. + if (refuseWaitThatWouldBlockThePump("drain(execution)")) { + return; + } + synchronized (this) { + throwIfFailed(); + if (execution.settled == null) { + return; + } + execution.drainWaiters++; + } + try { + drainUntilSettled(execution); + } finally { + synchronized (this) { + execution.drainWaiters--; + } + } + } + + private void drainUntilSettled(InsightPlugin execution) { while (true) { + CompletableFuture signal; CompletableFuture handle; boolean runInline = false; synchronized (this) { + throwIfFailed(); + signal = execution.settled; + if (signal == null) { + return; + } handle = inFlight; if (handle == null) { - if (pending.isEmpty()) { - return; - } - // Records are pending with no pump running (a worker could not be started): export them here. + // A record is outstanding with no pump running (a worker could not be started): export it here. handle = new CompletableFuture<>(); inFlight = handle; runInline = true; @@ -126,52 +389,447 @@ void drain() { } if (runInline) { pump(handle); - } else { - handle.join(); + if (nothingCanSettle(execution, signal)) { + // The pump this thread just ran found no record for this invocation and left none inside the + // exporters, yet the signal survives: no later step can complete it, so waiting again would only + // start empty pumps forever. Release it here and report — the pump's own exit does not sweep for + // this any more, because it has no registry of invocations to sweep and does not need one: the + // thread that would be stranded is this one, and it holds the instance. + abandon(execution); + reportFailure(new IllegalStateException( + "a drain signal survived a pump that had nothing to export for it; the drain was released" + + " rather than waiting for work nobody will do")); + return; + } + continue; + } + // Wake either when this invocation's record has been exported or when the current pump ends — the pump may + // have ended without taking this record (a rejected worker), in which case the loop re-evaluates and + // exports it inline. + try { + CompletableFuture.anyOf(signal, handle).join(); + } catch (Throwable t) { + // Never spin on an unexpected wait failure, and never let it escape into the execution. Abandon this + // invocation's outstanding record instead of leaving it queued: WORKERS is a static, process-wide pool, + // so a record left queued here would be exported later by some unrelated execution's pump — out of + // order, and after this invocation has already returned. Completing the signal also releases any other + // drain waiting on the same invocation rather than stranding it behind work nobody will do. + abandon(execution); + reportFailure(t); + return; } } } + /** + * True when this invocation still holds the same drain signal but has no queued record and none inside the + * exporters, so nothing that could complete the signal is left. No ordinary path produces that; this is the + * liveness backstop for the unwinds that are hard to enumerate exhaustively. + */ + private synchronized boolean nothingCanSettle(InsightPlugin execution, CompletableFuture signal) { + return execution.settled == signal && execution.record == null && !execution.exporting; + } + + /** + * Waits for every outstanding record. Test seam for an environment-wide drain; the per-invocation path uses + * {@link #drain(InsightPlugin)}. + * + *

Returns once nothing is queued and no pump owns the scheduler, which is exactly "every record scheduled so far + * has reached the exporters": a pump only returns to idle with its queue empty and the record it took settled. + * + *

Bounded by the number of passes, not by the set of invocations seen: an invocation that queues new work after + * it was already drained must still be waited for (dropping it would silently weaken every assertion made after + * this returns), while a producer that never stops cannot keep this spinning forever. + */ + void drainAll() { + throwIfFailed(); + // Every pass below is a drain, and each one would be refused; without this the loop spends all of its passes + // reporting the same refusal. + if (refuseWaitThatWouldBlockThePump("drainAll()")) { + return; + } + for (int pass = 0; pass < MAX_DRAIN_ALL_PASSES; pass++) { + List outstanding; + CompletableFuture handle; + synchronized (this) { + throwIfFailed(); + outstanding = new ArrayList<>(queue); + handle = inFlight; + if (outstanding.isEmpty() && handle == null) { + return; + } + } + for (InsightPlugin execution : outstanding) { + drain(execution); + } + if (outstanding.isEmpty()) { + // Nothing is queued for anyone, but a pump still owns the scheduler: it may be inside an exporter with + // a + // record whose only reference is its own local variable, and with no registry of invocations there is + // no + // way to name that record and drain it. Waiting for the pump itself covers it — in bounded steps, so a + // producer that keeps the pump permanently busy cannot make this unbounded, and so the wait is a real + // wait rather than a re-poll. + try { + handle.get(PUMP_WAIT_MILLIS, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + // Still running; take another pass. + } catch (Throwable t) { + reportFailure(t); + return; + } + } + } + } + + /** + * Test seam: how many invocations the scheduler still holds a reference to. + * + *

{@link #queue} is the only collection of per-invocation objects the scheduler has, so this is the whole of the + * per-invocation state the environment retains. Zero means the environment — which outlives every invocation — + * holds nothing belonging to any invocation it has served. + */ + synchronized int retainedInvocationCount() { + return queue.size(); + } + + /** Test seam: whether the scheduler still holds a reference to one particular invocation. */ + synchronized boolean retains(InsightPlugin execution) { + return queue.contains(execution); + } + + /** Gives up one invocation's outstanding work: drops its queued record and releases every drain waiting on it. */ + private void abandon(InsightPlugin execution) { + CompletableFuture signal; + synchronized (this) { + dropRecord(execution); + execution.exporting = false; + signal = execution.settled; + execution.settled = null; + } + if (signal != null) { + completeSignal(signal); + } + } + + // --- The pump. --- + private void pump(CompletableFuture handle) { + // Recorded for as long as this thread serves the pump — a worker, or a caller pumping inline after a rejected + // worker — so that a flush() or drain() re-entered from anything the fan-out calls synchronously can tell that + // it is asking itself. One atomic write per pump, and no lock: see the field. + Thread self = Thread.currentThread(); + pumpThread.set(self); + // The invocation this pump has taken a record from and not settled yet. Only this pump may release it, so an + // abnormal unwind cannot strand a drain, and no other pump can mistake it for orphaned work. + InsightPlugin taken = null; + // Likewise for the flush requests this pump has taken out of the queue and not completed yet. + List> takenFlushes = null; try { - // Serve pending executions in order until none is left. Taking a record and returning to idle both happen - // under the lock, so an update scheduled at any point is either exported by this pump or starts the next - // one — never lost. + // One record, then every flush request queued at that moment, alternating. Taking the record and returning + // to idle both happen under the lock, so a record scheduled at any point is either exported by this pump or + // starts the next one — never lost, and never displaced by another invocation's record. A flush therefore + // waits at most one fan-out (it cannot be starved by a queue that never runs dry) and still never overlaps + // an export, because this loop runs them one after the other. + // + // A loop, deliberately, not a pump that re-enters itself to pick up the next item: written that way, one + // frame per queued item accumulates until the stack overflows, and the rest of the queue is dropped. while (true) { - WorkflowInsightRecord record; + InsightPlugin next = null; + WorkflowInsightRecord record = null; synchronized (this) { - Iterator head = pending.values().iterator(); - if (!head.hasNext()) { - inFlight = null; + throwIfFailed(); + if (queue.isEmpty() && flushRequests.isEmpty()) { + if (inFlight == handle) { + inFlight = null; + } return; } - record = head.next(); - head.remove(); + if (!queue.isEmpty()) { + next = queue.iterator().next(); + // Leaving the slot and being marked mid-export are one write of one object, so the invocation + // is + // never momentarily indistinguishable from one with nothing outstanding. + record = takeRecord(next); + } + } + if (next != null) { + taken = next; + try { + exportToAll(record); + } finally { + signalSettled(next); + taken = null; + } + } + // Taken only now that the fan-out above has settled, and taken as a batch: every request queued at this + // instant is satisfied by the single flush below, so invocation ends that ask together cost one flush + // rather than one each. Sound because each requester drained its own record before asking, so a flush + // that starts after the request was enqueued already has that record in the exporter's buffer. + // + // Emptying the queue here — rather than after the flush — is what keeps a request that arrives while + // that flush runs out of this batch: it lands in the now-empty queue and is served by the next turn, + // never credited to a flush that was already in progress when it was made. + synchronized (this) { + if (!flushRequests.isEmpty()) { + takenFlushes = new ArrayList<>(flushRequests); + flushRequests.clear(); + } + } + if (takenFlushes != null) { + // Before spending the fan-out: export the queued records other invocations are still waiting on. + // Those ends cannot have asked for their flush yet — they are inside a drain — so without this the + // pump staggers them one record per turn, with a whole flush in between, and each pays for its own + // flush however aggressively the queue is coalesced. + exportRecordsADrainIsWaitingFor(); + // Re-take: the ends released above ask for their flush now, and one flush covers all of them since + // it starts after every one of those records reached the exporters. + synchronized (this) { + if (!flushRequests.isEmpty()) { + takenFlushes.addAll(flushRequests); + flushRequests.clear(); + } + } + try { + flushEveryExporter(); + } finally { + // In `finally`: a Throwable from a customer's flush() — an Error, not just an exception — must + // never leave the invocations waiting on these requests parked forever. + completeAll(takenFlushes); + takenFlushes = null; + } } - exportToAll(record); } + } catch (Throwable failure) { + rejectFatal(failure); + throw failure; } finally { + try { + // Before anything else, and before the handle below: whoever waits on these requests must be released + // even + // if this pump is unwinding for a reason none of the guards above anticipated. + if (takenFlushes != null) { + completeAll(takenFlushes); + } + CompletableFuture orphaned = null; + synchronized (this) { + if (inFlight == handle) { + inFlight = null; + } + if (taken != null) { + // Unwinding with a record still marked as being exported: this pump will never settle it. + // Release + // the marker and, unless a newer record for the same invocation is queued for a later pump to + // export, complete the drain waiting on it — the instance is right here, so no sweep over other + // invocations is needed to find it. + taken.exporting = false; + if (taken.record == null) { + orphaned = taken.settled; + taken.settled = null; + } + } + } + if (orphaned != null) { + completeSignal(orphaned); + } + completeSignal(handle); + // Last, because everything above is still this pump's work and a flush() re-entered from any of it + // would + // still have nobody to serve it. Conditional: a pump that recorded itself since must not be cleared + // here. + } finally { + pumpThread.compareAndSet(self, null); + } + } + } + + /** + * Exports the queued records that a drain is waiting for, one at a time, and returns once they have all reached the + * exporters. Called by the pump immediately before a flush. + * + *

Those records are the last records of invocations that cannot return until they are exported, and their ends + * cannot ask for their flush until then. Exporting them first is therefore what lets one flush serve a whole burst + * of invocation ends: without it the pump interleaves one record and one flush fan-out, and each end pays for a + * flush of its own even though every request is coalesced. + * + *

Bounded by the snapshot taken under the lock, so a producer that keeps scheduling for an invocation someone is + * draining cannot hold a flush back indefinitely — and records nobody waits for are not exported here at all, so a + * stream of {@code ON_CHANGE} snapshots still cannot starve a flush: it waits at most one ordinary fan-out plus + * this pass over the invocations whose return is already blocked on their own record. + */ + private void exportRecordsADrainIsWaitingFor() { + List awaited = null; + synchronized (this) { + for (InsightPlugin execution : queue) { + if (execution.drainWaiters > 0) { + if (awaited == null) { + awaited = new ArrayList<>(); + } + awaited.add(execution); + } + } + } + if (awaited == null) { + return; + } + for (InsightPlugin execution : awaited) { + WorkflowInsightRecord record; synchronized (this) { - if (inFlight == handle) { - inFlight = null; + record = execution.record == null ? null : takeRecord(execution); + } + if (record == null) { + continue; + } + try { + exportToAll(record); + } finally { + try { + signalSettled(execution); + } catch (Throwable t) { + // Nothing here is expected to throw, but a record left marked as being exported would strand the + // drain that is waiting for it, so release it and that drain rather than leave the invocation + // parked. + abandon(execution); + reportFailure(t); } } - handle.complete(null); } } - private static String executionKey(WorkflowInsightRecord record) { - return record.executionArn() != null ? record.executionArn() : ""; + /** Completes every taken flush request; one that cannot be completed must not stop the rest from being. */ + private void completeAll(List> requests) { + for (CompletableFuture request : requests) { + try { + completeSignal(request); + } catch (Throwable t) { + reportFailure(t); + } + } + } + + /** + * Completes one invocation's drain signal now that its record has been exported, unless a newer record from the + * same invocation arrived meanwhile — that one settles the signal instead, so a drain always waits for the latest. + */ + private void signalSettled(InsightPlugin execution) { + CompletableFuture signal; + synchronized (this) { + if (activeExecution == execution) activeExecution = null; + if (execution.record != null) { + // A newer record is queued for the same invocation. Leave it marked as being exported: it is still + // outstanding, and the export of that newer record settles the signal. + return; + } + execution.exporting = false; + signal = execution.settled; + execution.settled = null; + } + if (signal != null) { + completeSignal(signal); + } + } + + // --- Flushing. --- + + /** + * Flushes every exporter, serialized against exports: the request is queued and served by the pump between records, + * so an exporter never sees {@code flush()} overlap {@code export()} — not even an export belonging to a different + * execution running in the same environment. Returns once a flush that started after this request was enqueued has + * reached every exporter. + * + *

Requests are coalesced: the pump takes every request queued at the start of its turn, exports any queued + * record a drain is still waiting for, re-takes the requests those ends make as they are released, and satisfies + * them all with one flush. Invocation ends that overlap therefore share a flush instead of paying for one fan-out + * each. That is sound because a caller drains its own record before asking, so a flush that starts after + * the request was enqueued has that record in the exporter's buffer. A request enqueued while a flush is already + * running is never satisfied by it — it waits for the next turn. + * + *

A queue that never runs dry cannot starve a request either: the pump alternates one record and one batch of + * requests, so a flush waits at most one export fan-out. + * + *

Called from the pump thread itself — or from an exporter fan-out worker that pump is waiting for, which is + * what a callback re-entering the scheduler does when two or more exporters are configured — the request is refused + * and reported instead of made: see {@link #refuseWaitThatWouldBlockThePump}. + */ + void flush() { + throwIfFailed(); + // Re-entered from a thread the pump's progress depends on: the pump is the only thread that could serve the + // request, and it cannot while this caller has not returned. Refuse rather than enqueue a request nobody + // serves. + if (refuseWaitThatWouldBlockThePump("flush()")) { + return; + } + CompletableFuture request = new CompletableFuture<>(); + synchronized (this) { + throwIfFailed(); + flushRequests.add(request); + } + while (true) { + CompletableFuture handle; + boolean startPump = false; + synchronized (this) { + throwIfFailed(); + if (request.isDone()) { + return; + } + handle = inFlight; + if (handle == null) { + if (!flushRequests.contains(request)) { + // Liveness backstop: a pump took this request and unwound without serving it, which its + // `finally` is there to prevent. The request is no longer in the queue, so no future pump can + // find it — release the caller here instead of spinning up pumps that have nothing to do. + break; + } + // No pump is running (a worker could not be started earlier, or the pump went idle between the add + // above and this check): start one. + handle = new CompletableFuture<>(); + inFlight = handle; + startPump = true; + } + } + if (startPump) { + CompletableFuture started = handle; + try { + executor.execute(() -> pump(started)); + } catch (Throwable t) { + // No worker could be started. Serve the request on the calling thread, exactly as a drain exports a + // queued record inline: this pump owns `inFlight`, so no export can run beside it. + reportFailure(t); + pump(started); + continue; + } + } + // Wake either when this request has been served or when the current pump ends — a pump can end without + // serving it (a rejected worker), in which case the loop starts another one. + try { + CompletableFuture.anyOf(request, handle).join(); + } catch (Throwable t) { + // Never spin on an unexpected wait failure, and never let it escape into the execution. Drop the + // request rather than leaving it queued for some later, unrelated invocation's pump to serve. + synchronized (this) { + flushRequests.remove(request); + } + reportFailure(t); + break; + } + } + completeSignal(request); } /** * Flushes every exporter, each on its own worker, and waits for all of them to settle. A slow or failing flush on - * one exporter never delays or fails the others. + * one exporter never delays or fails the others. Environment-wide, like the exporters themselves. + * + *

Private and called only from the pump: routing every flush through the pump is what keeps a {@code flush()} + * from overlapping an {@code export()}, so this must not be reachable from outside. The per-exporter fan-out below + * is parallelism within one flush, not concurrency with an export. */ - void flushAll() { + private void flushEveryExporter() { forEachExporterSettled(InsightExporter::flush); } + // --- Exporting. --- + /** * Exports one record to every exporter, each on its own worker, and waits for all of them to settle. One failing or * slow exporter never blocks or fails the others, and an export error never propagates into the execution. @@ -183,26 +841,52 @@ private void exportToAll(WorkflowInsightRecord record) { /** Runs the action for every exporter concurrently and returns once all have settled, reporting each failure. */ private void forEachExporterSettled(Consumer action) { if (exporters.size() == 1) { + // On the pump thread itself, which the pump-thread check already refuses waits from. runSafely(() -> action.accept(exporters.get(0))); return; } - List> settled = new ArrayList<>(exporters.size()); + List> settledExporters = new ArrayList<>(exporters.size()); for (InsightExporter exporter : exporters) { - Runnable task = () -> runSafely(() -> action.accept(exporter)); + throwIfFailed(); + // Marked as a fan-out task: the pump joins every one of these below, so a wait for the pump issued from + // inside one must be refused exactly as one issued from the pump itself. + Runnable task = () -> runSafely(() -> runAsExporterFanOut(() -> action.accept(exporter))); try { - settled.add(CompletableFuture.runAsync(task, executor)); + settledExporters.add(CompletableFuture.runAsync(task, executor)); } catch (Throwable t) { reportFailure(t); task.run(); } } - for (CompletableFuture task : settled) { - runSafely(task::join); + for (CompletableFuture task : settledExporters) { + // A fatal on another worker must reach the invocation even if this peer is blocked. + runSafely(() -> CompletableFuture.anyOf(task, fatalSignal).join()); + } + } + + /** + * Runs one exporter's share of a fan-out with this thread marked pump-dependent, restoring the previous mark on the + * way out. The mark is what makes {@link #refuseWaitThatWouldBlockThePump} recognize a fan-out worker. + */ + private void runAsExporterFanOut(Runnable action) { + Boolean previous = exporterFanOutThread.get(); + exporterFanOutThread.set(Boolean.TRUE); + try { + action.run(); + } finally { + if (previous == null) { + // Removed rather than set back to null: these run on a shared, process-wide pool, so a thread must not + // keep an entry for this scheduler after its task ends. + exporterFanOutThread.remove(); + } else { + exporterFanOutThread.set(previous); + } } } private void runSafely(Runnable action) { try { + throwIfFailed(); action.run(); } catch (Throwable t) { reportFailure(t); @@ -210,10 +894,102 @@ private void runSafely(Runnable action) { } private void reportFailure(Throwable t) { + rejectFatal(t); try { failureHandler.accept(t); } catch (Throwable ignored) { - // A scheduler diagnostic must never disrupt durable execution. + rejectFatal(ignored); + // Ordinary diagnostic failures remain isolated. + } + } + + void throwIfFailed() { + Error failure = fatalFailure; + if (failure != null) throw failure; + } + + private void rejectFatal(Throwable failure) { + Error fatal = FatalErrors.find(failure); + if (fatal != null) { + fail(fatal); + throw fatal; + } + } + + private void completeSignal(CompletableFuture signal) { + Error failure = fatalFailure; + if (failure == null) signal.complete(null); + else signal.completeExceptionally(failure); + } + + /** Releases all owners and waiters without waiting for uncooperative exporter code. */ + synchronized void fail(Error failure) { + if (fatalFailure != null) return; + fatalFailure = failure; + CompletableFuture handle = inFlight; + inFlight = null; + fatalSignal.completeExceptionally(failure); + if (handle != null) handle.completeExceptionally(failure); + if (activeExecution != null) { + failOwner(activeExecution, failure); + activeExecution = null; + } + while (!queue.isEmpty()) failOwner(queue.iterator().next(), failure); + CompletableFuture request; + while ((request = flushRequests.poll()) != null) request.completeExceptionally(failure); + } + + private void failOwner(InsightPlugin execution, Error failure) { + dropRecord(execution); + execution.exporting = false; + CompletableFuture signal = execution.settled; + execution.settled = null; + if (signal != null) signal.completeExceptionally(failure); + } + + /** + * Reports and refuses a wait for the pump that was issued from a thread the pump's own progress depends on. Returns + * whether the caller is such a thread; when it is, the failure has already been reported and the caller must return + * without waiting. + * + *

Invariant: the thread that waits for the pump is never a thread the pump waits for. {@link #flush()} waits for + * a request only a pump can complete, and a drain waits for a signal only a pump can complete or for the running + * pump's own handle. All three are satisfied by the pump between records. + * + *

Two threads qualify. The pump thread itself: a wait issued from there is a wait-for cycle one thread wide — + * the pump parks on the future it would itself have completed, so it never reaches the point in its loop that + * completes it, and no other thread may take over because {@code inFlight} is this pump's. And an exporter fan-out + * worker: with two or more exporters the pump submits one task per exporter and joins them all, so a wait issued + * from a callback running on one of those workers is the same cycle two threads wide — the worker parks on a future + * only the pump can complete, and the pump is parked in the join waiting for that worker. Neither is a monitor + * deadlock, so the JVM's deadlock detection cannot see either one, and the invocation simply never returns. + * + *

Reachable through anything a fan-out calls synchronously: with a single exporter the fan-out runs on the pump + * thread, so a customer exporter's {@code export()} or {@code flush()} that asks the scheduler for a flush, or a + * non-conforming {@code exportOne}, is enough; with several it runs on a worker instead, and the same call is + * refused for the same reason. A conforming production {@code exportOne} does not re-enter the scheduler, so this + * is hardening. + * + *

So the call fails fast instead: the plugin's failure handler is told — it logs — and the caller returns as it + * would from any other flush or drain, with nothing propagating into the execution. The queued work itself is not + * dropped by refusing a drain: the record stays in the invocation's slot, and the pump that is waiting for this + * caller exports it as soon as this caller returns and the fan-out it belongs to settles. Callers that are neither + * — every SDK hook thread — never enter this branch and behave exactly as before; the check is a volatile read plus + * a thread-local read, so no lock is added to that path. + */ + private boolean refuseWaitThatWouldBlockThePump(String call) { + if (pumpThread.get() == Thread.currentThread()) { + reportFailure(new IllegalStateException(call + + " was called from the export pump thread, the only thread able to serve it; the call was refused" + + " rather than deadlocking the invocation")); + return true; + } + if (Boolean.TRUE.equals(exporterFanOutThread.get())) { + reportFailure(new IllegalStateException(call + + " was called from an exporter fan-out worker the export pump is waiting for, so the pump cannot" + + " serve it; the call was refused rather than deadlocking the invocation")); + return true; } + return false; } } diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightExporter.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightExporter.java index 69ff39739..59f04767e 100644 --- a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightExporter.java +++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightExporter.java @@ -11,7 +11,34 @@ public interface InsightExporter { /** Emits one record to the destination. */ void export(WorkflowInsightRecord record); - /** Flushes any buffered records; no-op by default. */ + /** + * Flushes any records this exporter has buffered. The default is a no-op; override it only if + * {@link #export(WorkflowInsightRecord)} buffers rather than emitting immediately. + * + *

Called at most once per sampled-in invocation end, after that invocation's own record — if it emitted one — + * has been handed to every exporter. An end that emits no record still flushes (a non-terminal suspend under + * {@code ON_COMPLETE}, a success under {@code ON_FAILURE}), so records buffered by that execution's earlier + * emissions are never left behind. Invocation ends that overlap may share a single flush: one flush is enough for + * all of them, because it starts only after each of their records has been handed to every exporter. An execution + * that is sampled out neither exports nor flushes. + * + *

Never called concurrently with {@link #export(WorkflowInsightRecord)} by the plugins one + * {@link WorkflowInsight#workflowInsight} factory creates. That factory owns the scheduler serializing them, so the + * guarantee is per factory rather than per environment: an exporter instance handed to two factories is served by + * two schedulers, which can call its {@code export} and {@code flush} at the same time. Build the factory once per + * handler — which is what a {@code DurableConfig} created once per handler does — and give each factory its own + * exporter instances if a single exporter cannot tolerate concurrent calls. + * + *

May cover records belonging to other executions running in the same environment, so it is not a per-execution + * barrier. + * + *

Must return promptly. No invocation whose end is waiting on this flush can return until it returns, and since + * overlapping ends may share one flush, a slow flush is billed to every one of those invocations — not only to the + * one that asked for it. + * + *

Non-fatal failures are reported and isolated without retries. Fatal VM/thread-termination errors propagate + * through the scheduler to the invocation caller; pending scheduler work is released. + */ default void flush() {} /** diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightPlugin.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightPlugin.java new file mode 100644 index 000000000..922f675de --- /dev/null +++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightPlugin.java @@ -0,0 +1,458 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicLong; +import software.amazon.lambda.durable.insight.internal.FatalErrors; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.OperationChangeInfo; +import software.amazon.lambda.durable.plugin.OperationChangeItemInfo; + +/** + * The Workflow Insight plugin instance for one Lambda invocation: both the state that invocation's records are built + * from and the slot the {@link ExportScheduler} exports them through. + * + *

The SDK creates one of these per invocation, from the {@link InvocationInfo} it is about to hand the first hook, + * and drops it when the invocation returns. So everything about an execution is a plain field here — the parsed ARN, + * the stable start time, the one-time sampling decision, the detached input snapshot, the latest queued record, the + * drain signal, the mid-export marker and the drain-waiter count. There is nothing to key by execution ARN and nothing + * to register or release: an instance is the registration, and its lifetime is the invocation's. + * + *

Two objects outlive the invocation and are shared by every instance the factory creates: the resolved + * {@link InsightSettings} and the {@link ExportScheduler}. The scheduler is shared on purpose — serializing exports is + * a property of the exporters, which belong to the environment, not to one invocation. + * + *

Ownership. Three groups of fields: + * + *

    + *
  • Identity — {@link #executionArn}, {@link #arn}, {@link #startTime}, {@link #sampledIn} — is taken from + * the {@link InvocationInfo} the factory receives and is {@code final}. It cannot be observed half-built, and + * there is no second invocation that could change it. + *
  • The input snapshot — {@link #cachedInput} — is written by the thread that fires + * {@code onInvocationStart} and read by the operation-change and invocation-end threads of the same invocation, + * which the SDK does not promise are the same thread; {@code volatile} for that publication. + *
  • The build revision — {@link #buildRevision} — counts the record builds this invocation has started, so + * that a build which was overtaken can be recognized at hand-off time and its record dropped. Atomic rather than + * {@code volatile}, because the case it exists for is two builds running at once. See the field. + *
  • Scheduling state — {@link #record}, {@link #settled}, {@link #exporting}, {@link #drainWaiters} and + * {@link #closed} — is shared with the export pump and guarded by the monitor of {@link #scheduler}. One monitor + * for the whole environment, not one per invocation, so the {@code closed} check and the hand-off of a record are + * a single critical section and there is no lock ordering between instances to get wrong. + *
+ * + *

{@link #closed} is additionally {@code volatile}: the hook threads read it without the lock as a fast pre-check. + * That read only ever skips work — the authoritative check is made under the monitor by + * {@link ExportScheduler#scheduleIfOpen}. It is written once, from false to true, and never back: an execution that + * suspends and resumes gets a new instance rather than a reset one. + */ +final class InsightPlugin implements DurableExecutionPlugin { + + /** Resolved configuration, shared by every instance of the environment. */ + private final InsightSettings settings; + + /** + * Shared with every other instance: exports are serialized across the whole environment. Package-private because it + * is the monitor this instance's scheduling fields are guarded by, which the tests in this package hold when they + * read them. + */ + final ExportScheduler scheduler; + + // --- Identity, from the InvocationInfo the factory was called with. --- + + /** The execution this instance observes. */ + final String executionArn; + + /** The parsed execution ARN, parsed once for every record this instance builds. */ + final ArnParser arn; + + /** Stable execution start time, from {@code InvocationInfo.executionStartTime()}. */ + final Instant startTime; + + /** The one-time sampling decision; deterministic in the ARN, so a resumed invocation decides the same way. */ + final boolean sampledIn; + + // --- The input snapshot. --- + + /** + * Detached snapshot of the execution input, the single source of truth for {@code input} on every emission of this + * invocation. Written by {@code onInvocationStart}, read by every later build. + */ + private volatile Object cachedInput; + + // --- Build ordering. --- + + /** + * Counts the record builds this invocation has started. The value a build takes identifies that build. + * + *

Customer code runs inside a build, on the hook thread: the input and output content transforms, an operation's + * result transform, and any Jackson serializer registered for a customer type. That code can call back into a hook + * of this same instance, and it runs before anything is scheduled, so a build can be overtaken by a newer build + * that starts and finishes inside it. Two hook threads for one invocation would produce the same overlap. + * + *

The scheduler's slot holds one record per invocation and takes whichever record is handed to it last, with no + * comparison of age. An overtaken build would therefore write its older snapshot over the newer one. Every build + * takes the next value here before it starts, and the scheduler queues the record only while that value is still + * the newest, so an overtaken build's record is dropped instead. + * + *

An {@link AtomicLong} rather than a {@code volatile long}: {@code ++} on a {@code volatile long} is a + * read-modify-write, so two concurrent builds can take the same value and each conclude its own record is the + * newest. That is the very case the check exists for, so a racy counter would guard nothing. + */ + private final AtomicLong buildRevision = new AtomicLong(); + + // --- Scheduling state: guarded by the scheduler's monitor. --- + + /** + * The latest record for this invocation that no pump has picked up yet, or {@code null} when none is queued. + * + *

A newer record replaces an older one here — each record is a complete snapshot, so the older one carries + * nothing the newer one lacks. That is the whole of coalescing: one slot, on the instance, which no other + * invocation can reach. + * + *

Which record is newer is decided by {@link #buildRevision}, not by the order the records reach this slot. The + * slot itself takes the last hand-off unconditionally, and the last hand-off is not the newest build when a build + * was overtaken by one that customer code started from inside it. + */ + WorkflowInsightRecord record; + + /** + * Completes once this invocation's latest record has been handed to every exporter; {@code null} when nothing is + * outstanding. + */ + CompletableFuture settled; + + /** + * Whether a pump has taken this invocation's record and is handing it to the exporters right now. + * + *

Set and cleared in the same critical sections that move {@link #record}, so "no queued record" is never + * mistaken for "nothing outstanding" while the record is inside the exporters. + */ + boolean exporting; + + /** + * How many {@code drain} calls are waiting for this invocation right now. + * + *

A record with a waiter gates an invocation return, so the pump exports it before it spends a flush fan-out. + */ + int drainWaiters; + + /** + * Set once invocation end begins; never cleared. Guarded by the scheduler's monitor — the same monitor that queues + * the record, so the check and the hand-off are one critical section — and {@code volatile} for the hook-side + * pre-check. + * + *

A checkpoint that completes while the end record is being drained still delivers an operation-change hook to + * this same instance, and that RUNNING snapshot must not supersede the final record. + * + *

This orders RUNNING records against the final record; {@link #buildRevision} orders RUNNING records against + * each other. Neither covers the other's case. A boolean cannot say which of two RUNNING builds is newer, and the + * revision cannot reject a RUNNING record that follows the final one, because the final record is queued without a + * revision check. See {@link ExportScheduler#closeAndSchedule}. + */ + volatile boolean closed; + + /** + * Creates the instance that serves one invocation. Identity comes from {@code info} rather than from the first + * hook, so every field a record is keyed by exists before any hook can fire. + * + * @throws RuntimeException if the invocation has no usable execution ARN; the SDK contains that exactly as it + * contains a hook failure, by skipping this plugin for the invocation + */ + InsightPlugin(InsightSettings settings, ExportScheduler scheduler, InvocationInfo info) { + this.settings = settings; + this.scheduler = scheduler; + this.executionArn = info.durableExecutionArn(); + this.arn = ArnParser.parse(executionArn); + this.startTime = info.executionStartTime(); + this.sampledIn = WorkflowInsight.shouldSample(executionArn, settings.samplingRate); + } + + /** Test seam: waits until every scheduled record has been handed to the exporters. */ + void drainExports() { + scheduler.drainAll(); + } + + @Override + public void onInvocationStart(InvocationInfo info) { + try { + scheduler.throwIfFailed(); + if (!sampledIn) { + return; + } + // Detach the execution input from the live handler value immediately, before the user handler or any + // content transform can mutate it. This raw, detached snapshot is the single source of truth for input + // on every emission (start / change / end); each build hands transforms a separate defensive copy so a + // mutating transform cannot corrupt it. Guard the snapshot: a Throwable here (e.g. a payload whose + // serialization overflows the stack) must omit the captured input, never fail the user handler. + try { + cachedInput = Json.deepCopyContent(info.executionInput()); + } catch (Throwable t) { + reportFailure("failed to snapshot execution input; omitting input", t); + cachedInput = null; + } + if (settings.emitMode == WorkflowInsightConfig.EmitMode.ON_CHANGE) { + // The revision is taken before the build, never after. Customer code runs inside buildRecord and can + // re-enter a hook of this instance, which builds a newer record; a revision read afterwards would + // already be that newer build's, and this older record would pass the check and overwrite it. + long revision = beginBuild(); + scheduler.scheduleIfNotSuperseded( + this, buildRecord("RUNNING", info.operations(), null, cachedInput, null, null), revision); + } + } catch (Throwable t) { + reportFailure("onInvocationStart failed", t); + } + } + + @Override + public void onOperationChange(OperationChangeInfo info) { + try { + scheduler.throwIfFailed(); + if (settings.emitMode != WorkflowInsightConfig.EmitMode.ON_CHANGE || !sampledIn) { + return; + } + // Lock-free pre-check: this invocation's end may already have begun, in which case no RUNNING snapshot may + // follow the final record. The authoritative check is made again under the scheduler's lock below. + if (closed) { + return; + } + long revision = beginBuild(); + scheduler.scheduleIfNotSuperseded( + this, buildRecord("RUNNING", info.operations(), null, cachedInput, null, null), revision); + } catch (Throwable t) { + reportFailure("onOperationChange failed", t); + } + } + + // onInvocationEnd is the hook the SDK awaits, so it is where the export queue is drained before the invocation + // returns; this guarantees the final record (scheduled above the drain) is delivered. The drain and flush run + // in finally so they also cover the paths where record construction fails. + @Override + public void onInvocationEnd(InvocationEndInfo info) { + try { + scheduler.throwIfFailed(); + String status = WorkflowInsight.mapStatus(info.invocationStatus()); + boolean isTerminal = "SUCCEEDED".equals(status) || "FAILED".equals(status); + boolean isFailure = "FAILED".equals(status); + boolean shouldEmit; + switch (settings.emitMode) { + case ON_CHANGE: + shouldEmit = true; + break; + case ON_FAILURE: + shouldEmit = isFailure; + break; + case ON_COMPLETE: + default: + shouldEmit = isTerminal; + break; + } + + WorkflowInsightRecord finalRecord = null; + if (sampledIn && shouldEmit) { + // No build revision is taken here. Customer code running inside this build can start a newer RUNNING + // build, which would make a revision taken here stale, and a checked hand-off would then drop the final + // record and leave a RUNNING snapshot as this execution's last exported state. The final record is + // instead ordered by `closed`, which closeAndSchedule sets in the same critical section that queues it. + finalRecord = buildRecord( + status, + info.operations(), + Instant.now(), + cachedInput, + info.executionResult(), + info.executionError()); + } + // Close before the drain below: an operation-change hook arriving from a checkpoint that completes + // during the drain is rejected, so no RUNNING snapshot can follow (or replace) the final record. + scheduler.closeAndSchedule(this, finalRecord); + } catch (Throwable t) { + // Ordinary hook/export failures remain isolated. Fatal VM/thread termination + // marks the scheduler failed and escapes without further exporter work. + reportFailure("onInvocationEnd failed", t); + } finally { + // If record construction failed above, this instance is still open: close it so a late change hook cannot + // schedule into the drain. Idempotent when already closed. + scheduler.closeAndSchedule(this, null); + // Sampled-out invocations never schedule a record, so there is nothing to drain or flush. The drain needs + // no lookup: this instance is the thing whose record it waits for. + if (sampledIn) { + drainAndFlush(); + } + // Nothing is released here. There is no per-execution entry to remove — this instance is the state, the SDK + // drops it when the invocation returns, and a suspended execution that resumes in the same container is + // served by a new instance built from the resume's own InvocationInfo (same stable start time, same + // deterministic sampling decision, its own input snapshot). A plugin failure therefore cannot turn into a + // state leak, because there is no place a leak could accumulate. + } + } + + /** + * Waits for this invocation's scheduled record to reach the exporters, then flushes each exporter once. The wait is + * per invocation: another execution running in the same environment can never displace this record, so this always + * returns having delivered this invocation's latest snapshot. It is not insulated from the queue, though — one pump + * exports serially, so records another execution had already queued ahead of this one are exported first and this + * drain waits for them too. + * + *

The flush goes through the scheduler's queue and is served by that same pump, between records, so no exporter + * ever sees this invocation's {@code flush()} overlap another's {@code export()}. Invocation ends that overlap + * share one flush: the cadence the exporter contract promises is at most one flush per sampled-in invocation end, + * not exactly one. + */ + private void drainAndFlush() { + try { + scheduler.drain(this); + } catch (Throwable t) { + reportFailure("failed to drain export scheduler", t); + } + try { + scheduler.flush(); + } catch (Throwable t) { + reportFailure("exporter flush failed", t); + } + } + + // --- Record building. --- + + /** Marks a fatal hook/logging failure before the invocation's cleanup can schedule more work. */ + private void reportFailure(String message, Throwable failure) { + try { + WorkflowInsight.logSafely(message, failure); + } catch (Throwable reportingFailure) { + Error fatal = FatalErrors.find(reportingFailure); + if (fatal != null) scheduler.fail(fatal); + throw reportingFailure; + } + } + + /** Starts a record build and returns the revision that identifies it. */ + private long beginBuild() { + return buildRevision.incrementAndGet(); + } + + /** + * Whether the identified build is still the newest one this invocation has started. + * + *

Read by the scheduler inside the critical section that queues the record, so a record that passes cannot be + * queued after a record that supersedes it. A build that starts after the check passes still supersedes this one: + * its record is handed over later and replaces this one in the slot, which is the order the slot should have. + */ + boolean isNewestBuild(long revision) { + return buildRevision.get() == revision; + } + + private WorkflowInsightRecord buildRecord( + String status, + Map operations, + Instant endTime, + Object input, + Object output, + Throwable error) { + ContentConfig content = settings.content; + WorkflowInsightRecord record = new WorkflowInsightRecord(); + record.emittedAt = Instant.now().toString(); + record.executionArn = executionArn; + record.executionName = WorkflowInsight.emptyToNull(arn.executionName()); + record.functionName = arn.functionName(); + record.functionQualifier = arn.qualifier(); + record.region = arn.region(); + record.accountId = arn.accountId(); + record.status = status; + record.startTime = startTime != null ? startTime.toString() : null; + if (endTime != null) { + record.endTime = endTime.toString(); + if (startTime != null) { + record.durationMs = endTime.toEpochMilli() - startTime.toEpochMilli(); + } + } + record.input = WorkflowInsight.applyDataContent( + "input", + input, + content == null || content.includeInput(), + content == null ? null : content.inputTransform()); + record.output = WorkflowInsight.applyDataContent( + "output", + output, + content == null || content.includeOutput(), + content == null ? null : content.outputTransform()); + // Honor ContentConfig.includeErrors for the execution-level error exactly as for operation-level errors + // below: with includeErrors(false) no execution error is emitted, so a sensitive failure message never + // reaches a record. Without this gate the execution error leaked even when errors were disabled. + if (settings.includeErrors && error != null) { + record.error = WorkflowInsight.toErrorInfo(error); + } + record.operations = buildOperationRecords(operations); + return record; + } + + private List buildOperationRecords(Map operations) { + List out = new ArrayList<>(); + if (operations == null) { + return out; + } + // The hook contract supplies a map with no iteration-order guarantee (the core snapshot originates from a + // concurrent map). Sort by startTimestamp ascending (null timestamps last), then by a stable operation id + // tie-breaker, so the emitted operations array is deterministic and OperationsIndex's "latest occurrence" + // scalar fields reflect true chronological order rather than arbitrary map iteration order. + List items = new ArrayList<>(operations.values()); + items.sort(Comparator.comparing( + OperationChangeItemInfo::startTimestamp, Comparator.nullsLast(Comparator.naturalOrder())) + .thenComparing(OperationChangeItemInfo::id, Comparator.nullsLast(Comparator.naturalOrder()))); + for (OperationChangeItemInfo item : items) { + // The SDK core tracks the invocation/execution itself as a pseudo-entry of type EXECUTION; it is not a + // customer operation and the record already carries the execution status/timing at top level. + if ("EXECUTION".equals(item.type())) { + continue; + } + // Unnamed operations can't be targeted or keyed — excluded by default (matches JS `if (!op.name)`). + if (item.name() == null) { + continue; + } + // top-level detail drops anything nested under a context (parallel branches, map items, nested steps). + if (settings.topLevelOnly && item.parentId() != null) { + continue; + } + OperationOverride override = settings.overridesByName.get(item.name()); + if (override != null && override.isExclude()) { + continue; + } + OperationRecord rec = new OperationRecord() + .id(item.id()) + .name(item.name()) + .type(item.type()) + .subType(item.subType()) + .parentId(item.parentId()) + .status(item.status() != null ? item.status().toString() : "UNKNOWN") + .startTime( + item.startTimestamp() != null + ? item.startTimestamp().toString() + : null) + .endTime(item.endTimestamp() != null ? item.endTimestamp().toString() : null) + .attempt(item.attempt()); + if (item.startTimestamp() != null && item.endTimestamp() != null) { + rec.durationMs(item.endTimestamp().toEpochMilli() + - item.startTimestamp().toEpochMilli()); + } + if (settings.includeErrors && item.error() != null) { + rec.error(WorkflowInsight.toErrorInfo(item.error())); + } + // Results are omitted unless an override explicitly opts in via a transform (matches JS). + if (override != null && override.result() != null) { + rec.result(WorkflowInsight.applyResultOverride(override.result(), item.result())); + } + out.add(rec); + } + return out; + } + + @Override + public String toString() { + return "InsightPlugin[" + executionArn + "]"; + } +} diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightSettings.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightSettings.java new file mode 100644 index 000000000..8729b8bec --- /dev/null +++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightSettings.java @@ -0,0 +1,59 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import software.amazon.lambda.durable.insight.exporters.LambdaLogExporter; + +/** + * The plugin's configuration, resolved once and then immutable: everything a record's shape depends on that does not + * depend on which invocation is being observed. + * + *

This belongs to the execution environment, not to an invocation. {@link WorkflowInsight#workflowInsight} resolves + * it once and the factory it returns hands the same instance to every {@link InsightPlugin} it creates, so resolving + * defaults, validating the sampling rate and indexing the operation overrides happen once per environment rather than + * once per invocation. + */ +final class InsightSettings { + + /** Sampling rate, clamped to [0, 1]; the per-invocation decision is derived from it and the execution ARN. */ + final double samplingRate; + + final WorkflowInsightConfig.EmitMode emitMode; + + /** True when nested operations (parallel branches, map items, nested steps) are dropped from the record. */ + final boolean topLevelOnly; + + final boolean includeErrors; + + /** May be null, which means "every default": include input, output and errors, with no transforms. */ + final ContentConfig content; + + /** Operation overrides indexed by operation name, in declaration order. */ + final Map overridesByName; + + /** The configured exporters, or the default single {@link LambdaLogExporter} when none were configured. */ + final List exporters; + + InsightSettings(WorkflowInsightConfig config) { + this.samplingRate = WorkflowInsight.resolveSamplingRate(config.samplingRate()); + this.emitMode = config.emitMode() != null ? config.emitMode() : WorkflowInsightConfig.EmitMode.ON_COMPLETE; + this.topLevelOnly = config.operationDetail() != WorkflowInsightConfig.OperationDetail.FULL_TREE; + this.content = config.content(); + this.includeErrors = content == null || content.includeErrors(); + Map overrides = new LinkedHashMap<>(); + if (content != null) { + for (OperationOverride override : content.overrides()) { + overrides.put(override.operationName(), override); + } + } + // Unmodifiable wrapper rather than Map.copyOf: declaration order is preserved and an override with a null + // operation name is tolerated exactly as the mutable map tolerated it. + this.overridesByName = Collections.unmodifiableMap(overrides); + this.exporters = + config.exporters().isEmpty() ? List.of(new LambdaLogExporter()) : List.copyOf(config.exporters()); + } +} diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/Json.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/Json.java index dc72a7092..fd843df59 100644 --- a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/Json.java +++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/Json.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Map; import software.amazon.lambda.durable.annotations.Experimental; +import software.amazon.lambda.durable.insight.internal.FatalErrors; /** Minimal JSON helper for emitting insight records and measuring their serialized size. */ @Experimental @@ -40,6 +41,7 @@ public static String stringify(Object value) { try { return MAPPER.writeValueAsString(value); } catch (JsonProcessingException e) { + FatalErrors.rethrow(e); throw new IllegalStateException("failed to serialize insight record", e); } } @@ -51,6 +53,7 @@ public static String prettyStringify(Object value) { try { return MAPPER.writer(PRETTY_PRINTER).writeValueAsString(value); } catch (JsonProcessingException e) { + FatalErrors.rethrow(e); throw new IllegalStateException("failed to serialize insight record", e); } } @@ -68,6 +71,7 @@ public static Integer byteSize(Object value) { try { return MAPPER.writeValueAsString(value).getBytes(StandardCharsets.UTF_8).length; } catch (JsonProcessingException e) { + FatalErrors.rethrow(e); return null; } } @@ -107,6 +111,8 @@ static Object deepCopyContent(Object value) { try { return MAPPER.convertValue(value, Object.class); } catch (IllegalArgumentException e) { + // convertValue transports Jackson serialization failures in IllegalArgumentException. + if (e.getCause() instanceof JsonProcessingException) FatalErrors.rethrow(e.getCause()); throw new IllegalStateException("failed to copy insight record content", e); } } diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/WorkflowInsight.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/WorkflowInsight.java index 531bbc475..ef50acaac 100644 --- a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/WorkflowInsight.java +++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/WorkflowInsight.java @@ -2,13 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.insight; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import com.fasterxml.jackson.core.JsonProcessingException; import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,12 +10,11 @@ import software.amazon.lambda.durable.annotations.Experimental; import software.amazon.lambda.durable.exception.DurableOperationException; import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; -import software.amazon.lambda.durable.insight.exporters.LambdaLogExporter; -import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.insight.internal.FatalErrors; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.InvocationStatus; -import software.amazon.lambda.durable.plugin.OperationChangeInfo; import software.amazon.lambda.durable.plugin.OperationChangeItemInfo; /** @@ -35,12 +28,18 @@ * {@link OperationChangeItemInfo#result()}; these are the fields PR #618 surfaced on the hook records, so * {@code input}, {@code output}, and operation {@code result} are now populated exactly as in the JS plugin. * - *

Per-execution state (keyed by execution ARN) holds only the stable start time, the parsed ARN, the one-time - * sampling decision, and a detached snapshot of the execution input. State is removed on every {@code onInvocationEnd} - * — including non-terminal PENDING/RETRYING suspends — so a suspended execution never leaks a retained entry for the - * lifetime of a warm container. Nothing is lost across a resume: the next invocation recreates the same stable start - * time from {@link InvocationInfo#executionStartTime()}, the same sampling decision deterministically from the ARN, and - * the input snapshot from {@link InvocationInfo#executionInput()}. + *

{@link #workflowInsight} returns a {@link DurableExecutionPluginFactory}, so the SDK creates one + * {@link InsightPlugin} per Lambda invocation and drops it when the invocation returns. Everything about an execution — + * the stable start time, the parsed ARN, the one-time sampling decision, the detached input snapshot, the queued + * record, the drain signal — is therefore a plain field of that instance. Nothing is keyed by execution ARN, and there + * is no per-execution entry to remove at invocation end, so a suspended execution cannot leak one for the lifetime of a + * warm container. Nothing is lost across a resume either: the resume's own {@link InvocationInfo} carries the same + * stable start time, the sampling decision is deterministic in the ARN, and the input snapshot is taken again from + * {@link InvocationInfo#executionInput()}. + * + *

What belongs to the execution environment rather than to an invocation stays in the factory: the resolved + * {@link InsightSettings}, the exporters, and the {@link ExportScheduler} that serializes exports across every + * execution the environment hosts. */ @Experimental public final class WorkflowInsight { @@ -49,378 +48,51 @@ public final class WorkflowInsight { private WorkflowInsight() {} - /** Creates a Workflow Insight plugin from the given config. Mirrors the JS {@code workflowInsight(config)}. */ - public static DurableExecutionPlugin workflowInsight(WorkflowInsightConfig config) { - return new InsightPlugin(config); - } - - /** Per-execution state, keyed by execution ARN, to prevent warm-container bleed and handle resume. */ - private static final class ExecutionState { - final Instant startTime; - final ArnParser arn; - final boolean sampledIn; - volatile Object cachedInput; - - /** - * Set once invocation end begins; guarded by {@code this}. A checkpoint that completes while the end record is - * being drained still delivers an operation-change hook, and that RUNNING snapshot must not supersede the final - * record. - */ - boolean closed; - - ExecutionState(Instant startTime, ArnParser arn, boolean sampledIn) { - this.startTime = startTime; - this.arn = arn; - this.sampledIn = sampledIn; - } - - /** Schedules the record unless the invocation has already ended; the check and the hand-off are atomic. */ - boolean scheduleIfOpen(ExportScheduler scheduler, WorkflowInsightRecord record) { - synchronized (this) { - if (closed) { - return false; - } - scheduler.schedule(record); - return true; - } - } - - /** Marks the invocation ended and, when given a record, schedules it as the last one for this execution. */ - void closeAndSchedule(ExportScheduler scheduler, WorkflowInsightRecord finalRecord) { - synchronized (this) { - closed = true; - if (finalRecord != null) { - scheduler.schedule(finalRecord); - } - } - } + /** + * Creates a Workflow Insight plugin factory from the given config. Mirrors the JS {@code workflowInsight(config)}. + * + *

The configuration is resolved once, here; the exporters and the scheduler that serializes exports across them + * are created once, here. The returned factory then builds one plugin instance per invocation, which is what lets + * that instance hold its execution's state in plain fields. + * + * @param config the plugin configuration + * @return a factory to hand to {@code DurableConfig.Builder.withPlugins} + */ + public static DurableExecutionPluginFactory workflowInsight(WorkflowInsightConfig config) { + InsightSettings settings = new InsightSettings(config); + ExportScheduler scheduler = new ExportScheduler( + settings.exporters, WorkflowInsight::exportRecord, t -> logSafely("export scheduling failed", t)); + return info -> new InsightPlugin(settings, scheduler, info); } - static final class InsightPlugin implements DurableExecutionPlugin { - private final double samplingRate; - private final WorkflowInsightConfig.EmitMode emitMode; - private final boolean topLevelOnly; - private final boolean includeErrors; - private final ContentConfig content; - private final Map overridesByName = new LinkedHashMap<>(); - private final List exporters; - private final ExportScheduler scheduler; - - private final Map byArn = new ConcurrentHashMap<>(); - - /** Test seam: number of live per-execution state entries retained across invocations. */ - int retainedStateCount() { - return byArn.size(); - } - - /** Test seam: waits until every scheduled record has been handed to the exporters. */ - void drainExports() { - scheduler.drain(); - } - - InsightPlugin(WorkflowInsightConfig config) { - this.samplingRate = resolveSamplingRate(config.samplingRate()); - this.emitMode = config.emitMode() != null ? config.emitMode() : WorkflowInsightConfig.EmitMode.ON_COMPLETE; - this.topLevelOnly = config.operationDetail() != WorkflowInsightConfig.OperationDetail.FULL_TREE; - this.content = config.content(); - this.includeErrors = content == null || content.includeErrors(); - if (content != null) { - for (OperationOverride o : content.overrides()) { - overridesByName.put(o.operationName(), o); - } - } - this.exporters = - config.exporters().isEmpty() ? List.of(new LambdaLogExporter()) : List.copyOf(config.exporters()); - this.scheduler = - new ExportScheduler(exporters, this::exportRecord, t -> logSafely("export scheduling failed", t)); - } - - private ExecutionState getState(String arn, Instant startTime) { - return byArn.computeIfAbsent( - arn, a -> new ExecutionState(startTime, ArnParser.parse(a), shouldSample(a, samplingRate))); - } - - @Override - public void onInvocationStart(InvocationInfo info) { - try { - ExecutionState state = getState(info.durableExecutionArn(), info.executionStartTime()); - if (!state.sampledIn) { - return; - } - // Detach the execution input from the live handler value immediately, before the user handler or any - // content transform can mutate it. This raw, detached snapshot is the single source of truth for input - // on every emission (start / change / end); each build hands transforms a separate defensive copy so a - // mutating transform cannot corrupt it. Guard the snapshot: a Throwable here (e.g. a payload whose - // serialization overflows the stack) must omit the captured input, never fail the user handler. - try { - state.cachedInput = Json.deepCopyContent(info.executionInput()); - } catch (Throwable t) { - logSafely("failed to snapshot execution input; omitting input", t); - state.cachedInput = null; - } - if (emitMode == WorkflowInsightConfig.EmitMode.ON_CHANGE) { - scheduler.schedule(buildRecord( - state, - info.durableExecutionArn(), - "RUNNING", - info.operations(), - null, - state.cachedInput, - null, - null)); - } - } catch (Throwable t) { - logSafely("onInvocationStart failed", t); - } - } - - @Override - public void onOperationChange(OperationChangeInfo info) { - try { - if (emitMode != WorkflowInsightConfig.EmitMode.ON_CHANGE) { - return; - } - ExecutionState state = byArn.get(info.durableExecutionArn()); - if (state == null || !state.sampledIn) { - return; - } - state.scheduleIfOpen( - scheduler, - buildRecord( - state, - info.durableExecutionArn(), - "RUNNING", - info.operations(), - null, - state.cachedInput, - null, - null)); - } catch (Throwable t) { - logSafely("onOperationChange failed", t); - } - } - - // onInvocationEnd is the hook the SDK awaits, so it is where the export queue is drained before the invocation - // returns; this guarantees the final record (scheduled above the drain) is delivered. The drain and flush run - // in finally so they also cover the paths where record construction fails. - @Override - public void onInvocationEnd(InvocationEndInfo info) { - ExecutionState state = null; - try { - state = getState(info.durableExecutionArn(), info.executionStartTime()); - String status = mapStatus(info.invocationStatus()); - boolean isTerminal = "SUCCEEDED".equals(status) || "FAILED".equals(status); - boolean isFailure = "FAILED".equals(status); - boolean shouldEmit; - switch (emitMode) { - case ON_CHANGE: - shouldEmit = true; - break; - case ON_FAILURE: - shouldEmit = isFailure; - break; - case ON_COMPLETE: - default: - shouldEmit = isTerminal; - break; - } - - WorkflowInsightRecord finalRecord = null; - if (state.sampledIn && shouldEmit) { - finalRecord = buildRecord( - state, - info.durableExecutionArn(), - status, - info.operations(), - Instant.now(), - state.cachedInput, - info.executionResult(), - info.executionError()); - } - // Close before the drain below: an operation-change hook arriving from a checkpoint that completes - // during the drain is rejected, so no RUNNING snapshot can follow (or replace) the final record. - state.closeAndSchedule(scheduler, finalRecord); - } catch (Throwable t) { - // A plugin failure at end-of-invocation (record construction, transforms, truncation, export/flush, - // or optional exporter class linkage) must never disrupt durable execution. - logSafely("onInvocationEnd failed", t); - } finally { - // If record construction failed above, the state is still open: close it so a late change hook cannot - // schedule into the drain. Idempotent when already closed. - if (state != null) { - state.closeAndSchedule(scheduler, null); - } - // Sampled-out executions never schedule a record, so there is nothing to drain or flush. If the state - // lookup itself failed, drain anyway: it is a no-op when idle and otherwise delivers what is pending. - if (state == null || state.sampledIn) { - drainAndFlush(); - } - // Remove per-execution state on EVERY invocation end, including non-terminal PENDING/RETRYING suspends, - // once any emission work above is done. Nothing durable is lost: the next invocation's onInvocation - // start recreates the stable startTime from InvocationInfo.executionStartTime() (stable across - // resumes), - // the one-time sampling decision deterministically from the ARN, and the input snapshot from - // InvocationInfo.executionInput(). Retaining state instead leaked one entry per suspended execution for - // the lifetime of the warm container. This runs even if emission above threw, so a plugin failure can - // never turn into a state leak. - byArn.remove(info.durableExecutionArn()); - } - } - - /** Waits for every scheduled record to reach the exporters, then flushes each exporter once, concurrently. */ - private void drainAndFlush() { - try { - scheduler.drain(); - } catch (Throwable t) { - logSafely("failed to drain export scheduler", t); - } - try { - scheduler.flushAll(); - } catch (Throwable t) { - logSafely("exporter flush failed", t); - } - } - - /** Shapes and exports one record to one exporter; runs on a scheduler worker, never on an SDK hook thread. */ - private void exportRecord(WorkflowInsightRecord record, InsightExporter exporter) { - try { - // Give each exporter its own deep copy: truncation returns the original record when it already fits, - // so without this a custom exporter that mutates operations or nested content would corrupt every - // other exporter's view of the same record. - WorkflowInsightRecord isolated = record.deepCopy(); - WorkflowInsightRecord shaped = - Truncation.truncateRecord(isolated, exporter.maxRecordSizeBytes(), exporter::render); - exporter.export(shaped); - } catch (Throwable t) { - // Catch Throwable, not just RuntimeException: deep copy, truncation, an exporter's render/export, or - // the linkage of an optional exporter class (a NoClassDefFoundError when the S3 / CloudWatch SDK is - // absent) can each fail with an Error. Isolating every Throwable here guarantees one failing exporter - // cannot affect the others, nor disrupt the execution. - logSafely("exporter failed", t); - } - } - - private WorkflowInsightRecord buildRecord( - ExecutionState state, - String arn, - String status, - Map operations, - Instant endTime, - Object input, - Object output, - Throwable error) { - WorkflowInsightRecord record = new WorkflowInsightRecord(); - ArnParser a = state.arn; - record.emittedAt = Instant.now().toString(); - record.executionArn = arn; - record.executionName = emptyToNull(a.executionName()); - record.functionName = a.functionName(); - record.functionQualifier = a.qualifier(); - record.region = a.region(); - record.accountId = a.accountId(); - record.status = status; - record.startTime = state.startTime != null ? state.startTime.toString() : null; - if (endTime != null) { - record.endTime = endTime.toString(); - if (state.startTime != null) { - record.durationMs = endTime.toEpochMilli() - state.startTime.toEpochMilli(); - } - } - record.input = applyDataContent( - "input", - input, - content == null || content.includeInput(), - content == null ? null : content.inputTransform()); - record.output = applyDataContent( - "output", - output, - content == null || content.includeOutput(), - content == null ? null : content.outputTransform()); - // Honor ContentConfig.includeErrors for the execution-level error exactly as for operation-level errors - // below: with includeErrors(false) no execution error is emitted, so a sensitive failure message never - // reaches a record. Without this gate the execution error leaked even when errors were disabled. - if (includeErrors && error != null) { - record.error = toErrorInfo(error); - } - record.operations = buildOperationRecords(operations); - return record; - } + // --- helpers --- - private List buildOperationRecords(Map operations) { - List out = new ArrayList<>(); - if (operations == null) { - return out; - } - // The hook contract supplies a map with no iteration-order guarantee (the core snapshot originates from a - // concurrent map). Sort by startTimestamp ascending (null timestamps last), then by a stable operation id - // tie-breaker, so the emitted operations array is deterministic and OperationsIndex's "latest occurrence" - // scalar fields reflect true chronological order rather than arbitrary map iteration order. - List items = new ArrayList<>(operations.values()); - items.sort(Comparator.comparing( - OperationChangeItemInfo::startTimestamp, Comparator.nullsLast(Comparator.naturalOrder())) - .thenComparing(OperationChangeItemInfo::id, Comparator.nullsLast(Comparator.naturalOrder()))); - for (OperationChangeItemInfo item : items) { - // The SDK core tracks the invocation/execution itself as a pseudo-entry of type EXECUTION; it is not a - // customer operation and the record already carries the execution status/timing at top level. - if ("EXECUTION".equals(item.type())) { - continue; - } - // Unnamed operations can't be targeted or keyed — excluded by default (matches JS `if (!op.name)`). - if (item.name() == null) { - continue; - } - // top-level detail drops anything nested under a context (parallel branches, map items, nested steps). - if (topLevelOnly && item.parentId() != null) { - continue; - } - OperationOverride override = overridesByName.get(item.name()); - if (override != null && override.isExclude()) { - continue; - } - OperationRecord rec = new OperationRecord() - .id(item.id()) - .name(item.name()) - .type(item.type()) - .subType(item.subType()) - .parentId(item.parentId()) - .status(item.status() != null ? item.status().toString() : "UNKNOWN") - .startTime( - item.startTimestamp() != null - ? item.startTimestamp().toString() - : null) - .endTime( - item.endTimestamp() != null - ? item.endTimestamp().toString() - : null) - .attempt(item.attempt()); - if (item.startTimestamp() != null && item.endTimestamp() != null) { - rec.durationMs(item.endTimestamp().toEpochMilli() - - item.startTimestamp().toEpochMilli()); - } - if (includeErrors && item.error() != null) { - rec.error(toErrorInfo(item.error())); - } - // Results are omitted unless an override explicitly opts in via a transform (matches JS). - if (override != null && override.result() != null) { - rec.result(applyResultOverride(override.result(), item.result())); - } - out.add(rec); - } - return out; + /** Shapes and exports one record to one exporter; runs on a scheduler worker, never on an SDK hook thread. */ + static void exportRecord(WorkflowInsightRecord record, InsightExporter exporter) { + try { + // Give each exporter its own deep copy: truncation returns the original record when it already fits, + // so without this a custom exporter that mutates operations or nested content would corrupt every + // other exporter's view of the same record. + WorkflowInsightRecord isolated = record.deepCopy(); + WorkflowInsightRecord shaped = + Truncation.truncateRecord(isolated, exporter.maxRecordSizeBytes(), exporter::render); + exporter.export(shaped); + } catch (Throwable t) { + // Ordinary Errors (such as missing optional dependencies) are isolated too. + // logSafely rethrows VM/thread termination before logging or continuing. + logSafely("exporter failed", t); } } - // --- helpers --- - /** * Applies a user-supplied result transform to an operation's checkpointed (serialized JSON) result. Parses the JSON * before handing it to the transform, so the transform always receives a detached, JSON-compatible value * (a {@code Map} for a former POJO, a {@code List} for an array, or a scalar such as a {@code String} for a Java * time value) — never the SDK's original Java object. The raw string is passed through only when the checkpointed - * result is not valid JSON. User transforms are untrusted: a throwing transform (any {@link Throwable}) omits the - * field rather than leaking the raw value or failing the execution, and the failure is logged for diagnosis. - * Because the value is freshly parsed from the immutable checkpoint string on every build, a transform that mutates - * its argument cannot corrupt any cached state or a later emission. + * result is not valid JSON. Non-fatal transform failures omit the field rather than leaking the raw value and are + * logged for diagnosis. VM/thread-termination errors propagate. Because the value is freshly parsed from the + * immutable checkpoint string on every build, a transform that mutates its argument cannot corrupt any cached state + * or a later emission. */ static Object applyResultOverride(Function transform, String rawResult) { if (rawResult == null) { @@ -429,7 +101,8 @@ static Object applyResultOverride(Function transform, String raw Object parsed; try { parsed = Json.MAPPER.readValue(rawResult, Object.class); - } catch (RuntimeException | com.fasterxml.jackson.core.JsonProcessingException e) { + } catch (RuntimeException | JsonProcessingException e) { + FatalErrors.rethrow(e); parsed = rawResult; } try { @@ -457,8 +130,8 @@ static Object applyDataContent(String label, Object value, boolean include, Func // reused across multiple emissions (ON_CHANGE), so a transform that mutates its argument in place must // not corrupt that snapshot or any later emission's view of it. deepCopyContent also normalizes POJOs // to Maps and Java-time types to their JSON representation, so the transform operates on the same - // JSON-compatible shape the record will emit. Omit and log on any Throwable so a failing redactor never - // leaks the raw value and never disrupts the execution. + // JSON-compatible shape the record will emit. Ordinary failures omit the raw value; + // fatal VM/thread termination is rethrown by logSafely. return transform.apply(Json.deepCopyContent(value)); } catch (Throwable e) { logSafely(label + " transform failed; value omitted", e); @@ -468,16 +141,18 @@ static Object applyDataContent(String label, Object value, boolean include, Func return value; } - /** Logs a plugin failure without ever letting the logging itself disrupt durable execution. */ - private static void logSafely(String message, Throwable t) { + /** Rethrows fatal failures; contains and logs ordinary plugin/logging failures. */ + static void logSafely(String message, Throwable t) { + FatalErrors.rethrow(t); try { logger.warn("[workflow-insight] {}", message, t); } catch (Throwable ignored) { - // Never allow a logging failure to propagate into the SDK control flow. + FatalErrors.rethrow(ignored); + // Ordinary logging failures stay isolated from SDK control flow. } } - private static ErrorInfo toErrorInfo(Throwable t) { + static ErrorInfo toErrorInfo(Throwable t) { // Operation and execution snapshot errors are exposed wrapped: operation failures as DurableOperationException // and unrecoverable execution failures as UnrecoverableDurableExecutionException. The wrapper's own // class/message would lose the original checkpointed failure identity. When the checkpointed ErrorObject is @@ -509,11 +184,11 @@ private static ErrorObject extractErrorObject(Throwable t) { return null; } - private static String emptyToNull(String s) { + static String emptyToNull(String s) { return s == null || s.isEmpty() ? null : s; } - private static String mapStatus(InvocationStatus status) { + static String mapStatus(InvocationStatus status) { if (status == InvocationStatus.SUCCEEDED) { return "SUCCEEDED"; } diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/exporters/LazyClient.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/exporters/LazyClient.java index bfd427288..7f6ad7d0c 100644 --- a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/exporters/LazyClient.java +++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/exporters/LazyClient.java @@ -7,6 +7,7 @@ import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.utils.builder.SdkBuilder; +import software.amazon.lambda.durable.insight.internal.FatalErrors; /** * Holds an exporter's transport client: an injected instance, or one created on first use from an optional AWS SDK @@ -66,6 +67,7 @@ private static T buildSdkClient(String clientClassName, String region) { } catch (ClassNotFoundException e) { throw new MissingArtifactException(e); } catch (ReflectiveOperationException e) { + FatalErrors.rethrow(e); Throwable cause = e instanceof InvocationTargetException ? e.getCause() : e; throw new IllegalStateException("Failed to create " + clientClassName, cause); } diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/internal/FatalErrors.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/internal/FatalErrors.java new file mode 100644 index 000000000..b9efab61f --- /dev/null +++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/internal/FatalErrors.java @@ -0,0 +1,51 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight.internal; + +import com.fasterxml.jackson.core.JsonProcessingException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.UndeclaredThrowableException; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Set; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; + +/** Internal fatal-failure policy for Insight hooks, serializers and transport adapters. */ +public final class FatalErrors { + private FatalErrors() {} + + /** Finds only direct fatal errors and known transport wrappers, not arbitrary business causes. */ + @SuppressWarnings("removal") // ThreadDeath remains supported on the SDK's Java 17 baseline. + public static Error find(Throwable failure) { + Set seen = null; + while (failure != null) { + if (failure instanceof VirtualMachineError fatal) return fatal; + if (failure instanceof ThreadDeath fatal) return fatal; + if (!isTransport(failure)) return null; + if (seen == null) seen = Collections.newSetFromMap(new IdentityHashMap<>()); + if (!seen.add(failure)) return null; + try { + failure = failure.getCause(); + } catch (VirtualMachineError | ThreadDeath fatal) { + return fatal; + } catch (Throwable ignored) { + return null; // A broken non-fatal cause accessor must not break fail-open handling. + } + } + return null; + } + + private static boolean isTransport(Throwable failure) { + return failure instanceof CompletionException + || failure instanceof ExecutionException + || failure instanceof InvocationTargetException + || failure instanceof UndeclaredThrowableException + || failure instanceof JsonProcessingException; + } + + public static void rethrow(Throwable failure) { + Error fatal = find(failure); + if (fatal != null) throw fatal; + } +} diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ConcurrentExecutionsExportTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ConcurrentExecutionsExportTest.java new file mode 100644 index 000000000..538e39240 --- /dev/null +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ConcurrentExecutionsExportTest.java @@ -0,0 +1,409 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.plugin.OperationChangeInfo; +import software.amazon.lambda.durable.plugin.OperationChangeItemInfo; + +/** + * One {@link ExportScheduler} — created once by {@code workflowInsight()} and shared by every plugin instance the + * factory makes — serves a whole execution environment, and an environment can host several durable executions at once + * (routine under Lambda Managed Instances). These tests pin the per-execution guarantees that concurrency demands: one + * execution's record never displaces another's, and each execution's drain returns only after its own record reached + * the exporters. + */ +class ConcurrentExecutionsExportTest { + + private static final Instant START = Instant.parse("2026-08-05T00:00:00Z"); + + private static String arn(int index) { + return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1"; + } + + private static class CapturingExporter implements InsightExporter { + final List records = new CopyOnWriteArrayList<>(); + + @Override + public void export(WorkflowInsightRecord record) { + records.add(record); + } + + /** Identity, not equality: these tests track the exact record instance an execution scheduled. */ + boolean exported(WorkflowInsightRecord record) { + for (WorkflowInsightRecord seen : records) { + if (seen == record) { + return true; + } + } + return false; + } + } + + private static WorkflowInsightRecord record(String executionArn, String status) { + var r = new WorkflowInsightRecord(); + r.executionArn = executionArn; + r.status = status; + return r; + } + + private static ExportScheduler scheduler(List failures, InsightExporter... exporters) { + return new ExportScheduler(List.of(exporters), (rec, exp) -> exp.export(rec), failures::add, workers()); + } + + private static Executor workers() { + return command -> new Thread(command, "test-export-worker").start(); + } + + /** + * Parks the pump task and then reports rejection, leaving the scheduler idle with the record still queued. Nothing + * about the scheduler is faked: the parked task is its own {@code () -> pump(handle)} lambda, run later verbatim. + */ + private static final class ParkingExecutor implements Executor { + final Deque parked = new ArrayDeque<>(); + + @Override + public void execute(Runnable command) { + parked.add(command); + throw new RejectedExecutionException("test: parked, reported as rejected"); + } + } + + /** Terminal records, by execution ARN, in the order the exporter received them. */ + private static List terminalArns(CapturingExporter exporter) { + List out = new ArrayList<>(); + for (WorkflowInsightRecord r : exporter.records) { + if ("SUCCEEDED".equals(r.status())) { + out.add(r.executionArn()); + } + } + return out; + } + + @Test + void everyConcurrentExecutionDeliversItsTerminalRecordExactlyOnce() throws Exception { + int executions = 10; + int changesEach = 3; + var failures = new CopyOnWriteArrayList(); + var exporter = new CapturingExporter(); + var scheduler = scheduler(failures, exporter); + + var barrier = new CyclicBarrier(executions); + // Executions whose drain returned before their own terminal record had reached the exporter. Each thread checks + // its own postcondition the instant its drain returns; inspecting the exporter only after joining every thread + // would also pass if a drain returned early and the export landed a moment later. + var returnedBeforeExport = Collections.synchronizedList(new ArrayList()); + var threads = new ArrayList(); + for (int i = 0; i < executions; i++) { + String executionArn = arn(i); + // One plugin instance per execution, as the SDK creates one per invocation; the thread below holds it + // exactly as an invocation's hooks do. + InsightPlugin execution = Executions.plugin(scheduler, executionArn); + var thread = new Thread( + () -> { + awaitBarrier(barrier); + for (int c = 0; c < changesEach; c++) { + scheduler.schedule(execution, record(executionArn, "RUNNING")); + } + WorkflowInsightRecord terminal = record(executionArn, "SUCCEEDED"); + scheduler.schedule(execution, terminal); + scheduler.drain(execution); + // This thread is the only one scheduling for this ARN, so no later record can supersede the + // terminal one: once drain returns, it must already have reached the exporter. + if (!exporter.exported(terminal)) { + returnedBeforeExport.add(executionArn); + } + }, + "execution-" + i); + threads.add(thread); + thread.start(); + } + for (Thread thread : threads) { + thread.join(30_000); + assertFalse(thread.isAlive(), "every execution's drain returned"); + } + + assertEquals( + List.of(), + returnedBeforeExport, + "drain returned before this execution's own terminal record reached the exporter"); + List delivered = terminalArns(exporter); + Set expected = new HashSet<>(); + for (int i = 0; i < executions; i++) { + expected.add(arn(i)); + } + assertEquals(expected, new HashSet<>(delivered), "no execution lost its terminal record"); + assertEquals(executions, delivered.size(), "and none was exported twice"); + assertTrue(failures.isEmpty(), "no scheduler failure was reported: " + failures); + } + + /** + * Regression: a pump that is exiting must not complete the drain signal of an execution whose record it does not + * own. Such a record has already left the queue — it is inside the exporters — so "no record queued for this ARN" + * is not enough to call the signal orphaned. If it were, the exiting pump would release {@code drain(arn)} mid + * export and the invocation could return before its final record was delivered. + * + *

The state is built through the {@link Executor} seam rather than by racing threads: the executor parks the + * pump task and reports rejection, which is the same shape the scheduler produces on its own in the window between + * the pump loop's return to idle and its {@code finally} — a live pump whose handle is no longer the installed one. + */ + @Test + void anExitingPumpDoesNotReleaseADrainWhoseRecordIsStillInsideTheExporter() throws Exception { + var executor = new ParkingExecutor(); + var exporting = new CountDownLatch(1); + var release = new CountDownLatch(1); + Set exported = ConcurrentHashMap.newKeySet(); + var scheduler = new ExportScheduler( + List.of(record -> {}), + (rec, exp) -> { + exporting.countDown(); + await(release, 10); + exported.add(rec); + }, + new CopyOnWriteArrayList()::add, + executor); + + String executionArn = arn(1); + InsightPlugin execution = Executions.plugin(scheduler, executionArn); + WorkflowInsightRecord terminal = record(executionArn, "SUCCEEDED"); + scheduler.schedule(execution, terminal); + assertEquals(1, executor.parked.size(), "the pump task was parked, so the record is still queued"); + + // A drainer picks the record up on the inline path and is now inside the exporter. + var inlineDrained = new CountDownLatch(1); + var inlineDrainer = new Thread( + () -> { + scheduler.drain(execution); + inlineDrained.countDown(); + }, + "inline-drainer"); + inlineDrainer.setDaemon(true); + inlineDrainer.start(); + assertTrue(exporting.await(5, TimeUnit.SECONDS), "the terminal record is inside the exporter"); + + // Now let the parked pump run to completion. It finds nothing queued and exits; its cleanup must leave the + // record that is mid-export alone. + executor.parked.poll().run(); + + var secondDrained = new CountDownLatch(1); + var secondDrainer = new Thread( + () -> { + scheduler.drain(execution); + secondDrained.countDown(); + }, + "second-drainer"); + secondDrainer.setDaemon(true); + secondDrainer.start(); + assertFalse( + secondDrained.await(500, TimeUnit.MILLISECONDS), + "drain returned while the execution's record was still inside the exporter"); + assertTrue(exported.isEmpty(), "the exporter has not finished with the record yet"); + + release.countDown(); + assertTrue(secondDrained.await(5, TimeUnit.SECONDS), "the drain returns once the export completes"); + assertTrue(inlineDrained.await(5, TimeUnit.SECONDS), "so does the drain that ran the export"); + assertEquals(Set.of(terminal), exported, "the terminal record was exported exactly once"); + } + + @Test + void aRecordForAnotherExecutionNeverDisplacesAPendingTerminalRecord() throws Exception { + String slowExecution = arn(1); + String otherExecution = arn(2); + var exporting = new CountDownLatch(1); + var release = new CountDownLatch(1); + var exporter = new CapturingExporter() { + @Override + public void export(WorkflowInsightRecord record) { + super.export(record); + if (records.size() == 1) { + exporting.countDown(); + await(release); + } + } + }; + var scheduler = scheduler(new CopyOnWriteArrayList<>(), exporter); + InsightPlugin slow = Executions.plugin(scheduler, slowExecution); + InsightPlugin other = Executions.plugin(scheduler, otherExecution); + + // One execution's export is in flight and blocked... + scheduler.schedule(slow, record(slowExecution, "RUNNING")); + assertTrue(exporting.await(5, TimeUnit.SECONDS), "the first export is in flight"); + // ...while a second execution's terminal record is queued, followed by an update for the first execution. + // The first execution's own update must coalesce only with its own slot, never over the second execution's. + scheduler.schedule(other, record(otherExecution, "SUCCEEDED")); + scheduler.schedule(slow, record(slowExecution, "SUCCEEDED")); + + var seenBySlowDrain = Collections.synchronizedList(new ArrayList()); + var seenByOtherDrain = Collections.synchronizedList(new ArrayList()); + var slowDrained = new CountDownLatch(1); + var otherDrained = new CountDownLatch(1); + var slowDrainer = new Thread( + () -> { + scheduler.drain(slow); + seenBySlowDrain.addAll(terminalArns(exporter)); + slowDrained.countDown(); + }, + "slow-drainer"); + var otherDrainer = new Thread( + () -> { + scheduler.drain(other); + seenByOtherDrain.addAll(terminalArns(exporter)); + otherDrained.countDown(); + }, + "other-drainer"); + slowDrainer.start(); + otherDrainer.start(); + + assertFalse(slowDrained.await(200, TimeUnit.MILLISECONDS), "a drain cannot return while its record is pending"); + assertFalse(otherDrained.await(50, TimeUnit.MILLISECONDS), "nor can the other execution's drain"); + + release.countDown(); + assertTrue(slowDrained.await(5, TimeUnit.SECONDS), "the blocked execution's drain completes"); + assertTrue(otherDrained.await(5, TimeUnit.SECONDS), "the other execution's drain completes"); + + assertTrue( + seenByOtherDrain.contains(otherExecution), + "drain returned only after this execution's own terminal record was exported"); + assertTrue( + seenBySlowDrain.contains(slowExecution), + "drain returned only after this execution's own terminal record was exported"); + List delivered = terminalArns(exporter); + assertEquals( + Set.of(slowExecution, otherExecution), + new HashSet<>(delivered), + "neither execution's terminal record was lost"); + assertEquals(2, delivered.size(), "and neither was exported twice"); + } + + @Test + void concurrentExecutionsDrivenThroughThePluginHooksAllDeliverTheirTerminalRecord() throws Exception { + int executions = 5; + var exporter = new CapturingExporter(); + // One factory — one environment, one scheduler, one set of exporters — and one plugin instance per invocation, + // which is how the SDK drives several concurrent executions through the same exporters. + var factory = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) + .addExporter(exporter) + .build()); + + var barrier = new CyclicBarrier(executions); + var plugins = new ArrayList(); + var threads = new ArrayList(); + for (int i = 0; i < executions; i++) { + String executionArn = arn(i); + InvocationInfo startInfo = start(executionArn); + var plugin = Executions.plugin(factory, startInfo); + plugins.add(plugin); + var thread = new Thread( + () -> { + awaitBarrier(barrier); + plugin.onInvocationStart(startInfo); + for (int c = 0; c < 3; c++) { + plugin.onOperationChange(new OperationChangeInfo( + "req", + executionArn, + ops(OperationStatus.SUCCEEDED), + ops(OperationStatus.SUCCEEDED))); + } + plugin.onInvocationEnd(end(executionArn, InvocationStatus.SUCCEEDED)); + }, + "execution-" + i); + threads.add(thread); + thread.start(); + } + for (Thread thread : threads) { + thread.join(30_000); + assertFalse(thread.isAlive(), "every invocation-end hook returned"); + } + + List delivered = terminalArns(exporter); + Set expected = new HashSet<>(); + for (int i = 0; i < executions; i++) { + expected.add(arn(i)); + } + assertEquals(expected, new HashSet<>(delivered), "every execution's terminal record arrived"); + assertEquals(executions, delivered.size(), "and none arrived twice"); + for (InsightPlugin plugin : plugins) { + assertFalse( + Executions.outstanding(plugin), + "the scheduler still owes this execution work after its invocation end: " + plugin); + } + } + + private static Map ops(OperationStatus status) { + Map operations = new LinkedHashMap<>(); + operations.put( + "op-1", + new OperationChangeItemInfo( + "op-1", + "greet", + "STEP", + "Step", + null, + START, + START.plusMillis(5), + status, + 1, + false, + null, + null)); + return operations; + } + + private static InvocationInfo start(String executionArn) { + return new InvocationInfo("req", executionArn, true, START, "in", ops(OperationStatus.STARTED), Map.of()); + } + + private static InvocationEndInfo end(String executionArn, InvocationStatus status) { + return new InvocationEndInfo( + "req", executionArn, true, START, ops(OperationStatus.SUCCEEDED), status, null, "in", "out"); + } + + private static void awaitBarrier(CyclicBarrier barrier) { + try { + barrier.await(30, TimeUnit.SECONDS); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + private static void await(CountDownLatch latch) { + await(latch, 5); + } + + private static void await(CountDownLatch latch, long timeoutSeconds) { + try { + if (!latch.await(timeoutSeconds, TimeUnit.SECONDS)) { + throw new AssertionError("latch not released"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } +} diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ErrorPrivacyGateTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ErrorPrivacyGateTest.java index 0cd5ede12..2e0a47af9 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ErrorPrivacyGateTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ErrorPrivacyGateTest.java @@ -14,7 +14,6 @@ import java.util.Map; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.lambda.model.OperationStatus; -import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.InvocationStatus; @@ -64,10 +63,15 @@ private Map failingOp() { } private WorkflowInsightRecord runFailedExecution(boolean includeErrors, CapturingExporter exporter) { - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .content(ContentConfig.builder().includeErrors(includeErrors).build()) - .addExporter(exporter) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .content(ContentConfig.builder() + .includeErrors(includeErrors) + .build()) + .addExporter(exporter) + .build()), + ARN, + START); plugin.onInvocationStart(new InvocationInfo("req", ARN, true, START, "in", failingOp(), Map.of())); plugin.onInvocationEnd(new InvocationEndInfo( "req", diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/Executions.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/Executions.java new file mode 100644 index 000000000..51d30e414 --- /dev/null +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/Executions.java @@ -0,0 +1,77 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight; + +import java.time.Instant; +import java.util.Map; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationInfo; + +/** + * Test helper: builds the per-invocation plugin instances the {@link ExportScheduler} schedules for. + * + *

The scheduler no longer resolves an execution ARN to anything — an invocation's state is its plugin + * instance, created from the {@link InvocationInfo} the SDK is about to hand the first hook — so tests hold the + * instance exactly as the SDK does. Everything here goes through the same constructor and the same factory production + * uses; nothing is a test-only back door into the scheduler. + */ +final class Executions { + + private static final Instant START = Instant.parse("2026-08-05T00:00:00Z"); + + private Executions() {} + + /** The invocation description the SDK would build for one execution, with no payload or operation snapshot. */ + static InvocationInfo info(String executionArn) { + return new InvocationInfo("req", executionArn, true, START, null, Map.of(), Map.of()); + } + + /** + * One invocation's plugin instance, bound to this scheduler and configured with the plugin's defaults. For tests + * that drive the scheduler directly and do not care how records are shaped. + */ + static InsightPlugin plugin(ExportScheduler scheduler, String executionArn) { + return new InsightPlugin( + new InsightSettings(WorkflowInsightConfig.builder().build()), scheduler, info(executionArn)); + } + + /** + * One invocation's plugin instance from the factory, for the identity the SDK would have built it with. The + * invocation's own {@code InvocationInfo} still goes to {@code onInvocationStart}; this is the same pair of facts + * that info carries, which is all an instance's identity is. + */ + static InsightPlugin plugin(DurableExecutionPluginFactory factory, String executionArn, Instant startTime) { + return plugin(factory, new InvocationInfo("req", executionArn, true, startTime, null, Map.of(), Map.of())); + } + + /** + * One invocation's plugin instance, exactly as the SDK creates it: from the factory, with that invocation's info. + */ + static InsightPlugin plugin(DurableExecutionPluginFactory factory, InvocationInfo info) { + return (InsightPlugin) factory.createPlugin(info); + } + + /** + * Whether the scheduler still owes this invocation anything: a queued record, a record inside the exporters, an + * uncompleted drain signal, or a drain waiting on it. Read under the monitor those fields are guarded by. + * + *

This is the question the plugin's {@code retainedStateCount()} seam used to answer for a whole registry. There + * is no registry to count now — an invocation's state is its plugin instance, and the SDK drops it — so the + * property worth asserting is that nothing the environment outlives keeps hold of it. + */ + static boolean outstanding(InsightPlugin plugin) { + synchronized (plugin.scheduler) { + return plugin.record != null || plugin.exporting || plugin.settled != null || plugin.drainWaiters > 0; + } + } + + /** + * The instance plus its first hook, in the order the SDK dispatches them: the factory is called with the very + * {@link InvocationInfo} that {@code onInvocationStart} then receives. + */ + static InsightPlugin started(DurableExecutionPluginFactory factory, InvocationInfo info) { + InsightPlugin plugin = plugin(factory, info); + plugin.onInvocationStart(info); + return plugin; + } +} diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFanOutReentryTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFanOutReentryTest.java new file mode 100644 index 000000000..973fe97d3 --- /dev/null +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFanOutReentryTest.java @@ -0,0 +1,199 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +/** + * A {@code flush()} or {@code drain()} issued from an exporter fan-out worker is refused and reported, not waited on. + * + *

With two or more exporters configured, the pump does not run the exporter callbacks itself: it submits one task + * per exporter and then waits for all of them. A callback therefore runs on a worker the pump is blocked on, and a wait + * for the pump issued from that worker is a wait-for cycle two threads wide — the worker parks on a future only the + * pump can complete, and the pump cannot resume its loop until that worker returns. The single-exporter case runs the + * callback on the pump thread itself and is covered by {@link ExportSchedulerReentrantFlushTest}; this covers the + * fan-out, which the pump-thread identity check alone does not recognize. + */ +class ExportSchedulerFanOutReentryTest { + + /** Longest a call that must return promptly may take before the property under test is considered broken. */ + private static final long DEADLINE_MILLIS = 5_000; + + private static String arn(int index) { + return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1"; + } + + private static WorkflowInsightRecord record(String executionArn, String status) { + var r = new WorkflowInsightRecord(); + r.executionArn = executionArn; + r.status = status; + return r; + } + + /** Unbounded thread-per-task executor, like the cached pool the plugin injects in production. */ + private static Executor sharedWorkers() { + return command -> { + var thread = new Thread(command, "fan-out-reentry-worker"); + thread.setDaemon(true); + thread.start(); + }; + } + + private static final class CountingExporter implements InsightExporter { + final AtomicInteger exports = new AtomicInteger(); + final AtomicInteger flushes = new AtomicInteger(); + + @Override + public void export(WorkflowInsightRecord record) { + exports.incrementAndGet(); + } + + @Override + public void flush() { + flushes.incrementAndGet(); + } + } + + @Test + void flushReenteredFromAFanOutWorkerIsRefusedReportedAndLosesNoWork() throws Exception { + var first = new CountingExporter(); + var second = new CountingExporter(); + var failures = new CopyOnWriteArrayList(); + var holder = new AtomicReference(); + var reentrantFlushReturned = new CountDownLatch(1); + var flushesSeenByTheRefusedCall = new AtomicInteger(-1); + var reentered = new AtomicInteger(); + + var scheduler = new ExportScheduler( + List.of(first, second), + (rec, exp) -> { + exp.export(rec); + // Only the first exporter re-enters, and only once, so exactly one refusal is expected. + if (exp == first && "SUCCEEDED".equals(rec.status()) && reentered.getAndIncrement() == 0) { + holder.get().flush(); + flushesSeenByTheRefusedCall.set(first.flushes.get() + second.flushes.get()); + reentrantFlushReturned.countDown(); + } + }, + failures::add, + sharedWorkers()); + holder.set(scheduler); + + var drainReturned = new CountDownLatch(1); + var firstExecution = Executions.plugin(scheduler, arn(0)); + var invocation = new Thread( + () -> { + scheduler.schedule(firstExecution, record(arn(0), "SUCCEEDED")); + scheduler.drain(firstExecution); + drainReturned.countDown(); + }, + "fan-out-reentry-invocation"); + invocation.setDaemon(true); + invocation.start(); + + assertTrue( + reentrantFlushReturned.await(DEADLINE_MILLIS, MILLISECONDS), + "flush() re-entered from an exporter fan-out worker never returned: the pump is waiting for that" + + " worker, so nothing can serve the request it made"); + assertTrue( + drainReturned.await(DEADLINE_MILLIS, MILLISECONDS), + "drain() never returned after the re-entrant flush"); + invocation.join(DEADLINE_MILLIS); + + assertEquals(1, failures.size(), "exactly one failure reported: " + failures); + assertTrue( + failures.get(0) instanceof IllegalStateException, + "the refusal is reported as an IllegalStateException: " + failures.get(0)); + assertTrue( + failures.get(0).getMessage().contains("flush()"), + "the report names the refused call: " + failures.get(0).getMessage()); + assertEquals(0, flushesSeenByTheRefusedCall.get(), "the refused request must not have reached an exporter"); + + // No work lost: the record that was in flight reached both exporters. + assertEquals(1, first.exports.get(), "the record reached the first exporter"); + assertEquals(1, second.exports.get(), "the record reached the second exporter"); + + // Still usable from a thread that is not pump-dependent. + var secondExecution = Executions.plugin(scheduler, arn(1)); + scheduler.schedule(secondExecution, record(arn(1), "SUCCEEDED")); + scheduler.drain(secondExecution); + scheduler.flush(); + + assertEquals(2, first.exports.get(), "both records reached the first exporter"); + assertEquals(2, second.exports.get(), "both records reached the second exporter"); + assertEquals(1, first.flushes.get(), "the later flush is served normally on the first exporter"); + assertEquals(1, second.flushes.get(), "the later flush is served normally on the second exporter"); + assertEquals(1, failures.size(), "no further failure after the refusal: " + failures); + } + + @Test + void drainReenteredFromAFanOutWorkerIsRefusedReportedAndLosesNoWork() throws Exception { + var first = new CountingExporter(); + var second = new CountingExporter(); + var failures = new CopyOnWriteArrayList(); + var holder = new AtomicReference(); + var pluginHolder = new AtomicReference(); + var reentrantDrainReturned = new CountDownLatch(1); + var reentered = new AtomicInteger(); + + var scheduler = new ExportScheduler( + List.of(first, second), + (rec, exp) -> { + exp.export(rec); + if (exp == first && "SUCCEEDED".equals(rec.status()) && reentered.getAndIncrement() == 0) { + holder.get().drain(pluginHolder.get()); + reentrantDrainReturned.countDown(); + } + }, + failures::add, + sharedWorkers()); + holder.set(scheduler); + + var execution = Executions.plugin(scheduler, arn(0)); + pluginHolder.set(execution); + + var drainReturned = new CountDownLatch(1); + var invocation = new Thread( + () -> { + scheduler.schedule(execution, record(arn(0), "SUCCEEDED")); + scheduler.drain(execution); + drainReturned.countDown(); + }, + "fan-out-reentry-drain-invocation"); + invocation.setDaemon(true); + invocation.start(); + + assertTrue( + reentrantDrainReturned.await(DEADLINE_MILLIS, MILLISECONDS), + "drain() re-entered from an exporter fan-out worker never returned: the pump is waiting for that" + + " worker, so nothing can settle the signal it waited for"); + assertTrue( + drainReturned.await(DEADLINE_MILLIS, MILLISECONDS), + "the invocation's own drain() never returned after the re-entrant drain"); + invocation.join(DEADLINE_MILLIS); + + assertEquals(1, failures.size(), "exactly one failure reported: " + failures); + assertTrue( + failures.get(0) instanceof IllegalStateException, + "the refusal is reported as an IllegalStateException: " + failures.get(0)); + assertTrue( + failures.get(0).getMessage().contains("drain"), + "the report names the refused call: " + failures.get(0).getMessage()); + + // No work lost by refusing the drain: the record still reached every exporter, and the invocation's own drain + // returned only once it had. + assertEquals(1, first.exports.get(), "the record reached the first exporter"); + assertEquals(1, second.exports.get(), "the record reached the second exporter"); + assertTrue(!Executions.outstanding(execution), "the scheduler owes the invocation nothing after its drain"); + } +} diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushCoalescingTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushCoalescingTest.java new file mode 100644 index 000000000..fc162981f --- /dev/null +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushCoalescingTest.java @@ -0,0 +1,311 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; + +/** + * Contract tests for the flush cadence: at most one flush per invocation end that asks for one, invocation ends that + * overlap may share a flush, and a request made while a flush is already running is never satisfied by that flush. + * + *

Coalescing is sound because every requester drains its own record before asking, so a flush that starts + * after the request was made has that record in the buffer. It is what stops N ends that ask together from paying for N + * serialized flush fan-outs — a 60 ms exporter flush cost the slowest of 8 ends ~520 ms before this change. + */ +class ExportSchedulerFlushCoalescingTest { + + /** A slow flush must not cost the caller more than a small multiple of the one flush it asked for. */ + private static final int PROMPTNESS_FACTOR = 3; + + private static String arn(int index) { + return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1"; + } + + private static WorkflowInsightRecord record(String executionArn, String status) { + var r = new WorkflowInsightRecord(); + r.executionArn = executionArn; + r.status = status; + return r; + } + + private static ExportScheduler scheduler(Executor executor, InsightExporter... exporters) { + return new ExportScheduler(List.of(exporters), (rec, exp) -> exp.export(rec), t -> {}, executor); + } + + /** Unbounded thread-per-task executor, like the cached pool the plugin injects in production. */ + private static Executor sharedWorkers() { + return command -> { + var thread = new Thread(command, "coalescing-worker"); + thread.setDaemon(true); + thread.start(); + }; + } + + private static final class SlowFlushExporter implements InsightExporter { + private final long flushMillis; + final AtomicInteger flushes = new AtomicInteger(); + + SlowFlushExporter(long flushMillis) { + this.flushMillis = flushMillis; + } + + @Override + public void export(WorkflowInsightRecord record) {} + + @Override + public void flush() { + flushes.incrementAndGet(); + sleep(flushMillis); + } + } + + /** + * N invocation ends whose records have already been delivered ask for their flush together: they share one flush, + * so the slowest pays a small multiple of one flush rather than N times one. + * + *

The records are drained before the requests are made on purpose: this is coalescing on its own, with every + * request already queued when the pump reaches its flush step. The harder case — ends that are still inside + * {@code drain()} when the first request is served, and so cannot have asked yet — is + * {@link #simultaneousDrainAndFlushEndsShareAFlushRatherThanOneEach()}. + */ + @Test + void invocationEndsAskingForAFlushTogetherShareOneFlush() { + long flushMillis = 60; + int executions = 8; + var exporter = new SlowFlushExporter(flushMillis); + var scheduler = scheduler(sharedWorkers(), exporter); + + var durations = Collections.synchronizedList(new java.util.ArrayList()); + var recordsDelivered = new CyclicBarrier(executions); + var done = new CountDownLatch(executions); + for (int i = 0; i < executions; i++) { + String executionArn = arn(i); + InsightPlugin execution = Executions.plugin(scheduler, executionArn); + start("end-" + i, () -> { + scheduler.schedule(execution, record(executionArn, "SUCCEEDED")); + scheduler.drain(execution); + awaitBarrier(recordsDelivered); + long began = System.nanoTime(); + scheduler.flush(); + durations.add((System.nanoTime() - began) / 1_000_000L); + done.countDown(); + }); + } + assertTrue(await(done, 60_000), "an invocation end never returned"); + + long slowest = Collections.max(durations); + int flushes = exporter.flushes.get(); + System.out.printf( + "COALESCING: %d invocation ends flushing together, %d ms exporter flush | slowest end returned after" + + " %d ms | flushes run: %d (one per end, %d, before coalescing)%n", + executions, flushMillis, slowest, flushes, executions); + + assertTrue(flushes >= 1, "every invocation end must be covered by a flush"); + assertTrue(flushes <= executions, "at most one flush per invocation end: " + flushes + " for " + executions); + assertTrue( + slowest <= flushMillis * PROMPTNESS_FACTOR, + "the slowest invocation end waited " + slowest + " ms for a " + flushMillis + " ms flush: ends that ask" + + " together must share a flush rather than serialize one fan-out each"); + } + + /** + * The realistic shape: schedule, drain, flush, all landing at once. An end cannot ask for its flush until its own + * record has been exported, so the pump exports the records a drain is waiting for before it spends a flush + * fan-out; without that the ends are staggered one record per flush and each pays for a flush of its own. + */ + @Test + void simultaneousDrainAndFlushEndsShareAFlushRatherThanOneEach() { + long flushMillis = 40; + int executions = 8; + var exporter = new SlowFlushExporter(flushMillis); + var scheduler = scheduler(sharedWorkers(), exporter); + + var durations = Collections.synchronizedList(new java.util.ArrayList()); + var barrier = new CyclicBarrier(executions); + var done = new CountDownLatch(executions); + for (int i = 0; i < executions; i++) { + String executionArn = arn(i); + InsightPlugin execution = Executions.plugin(scheduler, executionArn); + start("drain-and-flush-" + i, () -> { + awaitBarrier(barrier); + long began = System.nanoTime(); + scheduler.schedule(execution, record(executionArn, "SUCCEEDED")); + scheduler.drain(execution); + scheduler.flush(); + durations.add((System.nanoTime() - began) / 1_000_000L); + done.countDown(); + }); + } + assertTrue(await(done, 60_000), "an invocation end never returned"); + + int flushes = exporter.flushes.get(); + long slowest = Collections.max(durations); + System.out.printf( + "COALESCING (drain then flush): %d simultaneous ends, %d ms exporter flush | slowest end after %d ms |" + + " flushes run: %d (one per end, %d, before coalescing)%n", + executions, flushMillis, slowest, flushes, executions); + assertTrue(flushes >= 1, "every invocation end must be covered by a flush"); + assertTrue(flushes <= executions, "at most one flush per invocation end: " + flushes + " for " + executions); + assertTrue( + slowest <= flushMillis * (PROMPTNESS_FACTOR + 1), + "the slowest of " + executions + " simultaneous ends waited " + slowest + " ms for a " + flushMillis + + " ms flush: an end must not pay for one flush per end"); + } + + /** + * The exact counts, deterministically: five requests made while a flush is running are not satisfied by it — that + * flush cannot have seen their records — and they share the single flush that follows it. + */ + @Test + void requestsMadeWhileAFlushRunsShareTheNextFlushAndAreNeverSatisfiedByTheRunningOne() { + var insideFirstFlush = new CountDownLatch(1); + var releaseFirstFlush = new CountDownLatch(1); + var flushStarts = new AtomicInteger(); + var flushCompletions = new AtomicInteger(); + var exporter = new InsightExporter() { + @Override + public void export(WorkflowInsightRecord record) {} + + @Override + public void flush() { + if (flushStarts.incrementAndGet() == 1) { + insideFirstFlush.countDown(); + await(releaseFirstFlush, 30_000); + } + flushCompletions.incrementAndGet(); + } + }; + var scheduler = scheduler(sharedWorkers(), exporter); + + start("first-flusher", scheduler::flush); + assertTrue(await(insideFirstFlush, 5_000), "the pump never entered the first flush"); + + int latecomers = 5; + var returned = new CountDownLatch(latecomers); + var startsSeenOnReturn = new CopyOnWriteArrayList(); + for (int i = 0; i < latecomers; i++) { + start("latecomer-" + i, () -> { + scheduler.flush(); + startsSeenOnReturn.add(flushStarts.get()); + returned.countDown(); + }); + } + sleep(300); // every latecomer is queued while the first flush is still inside the exporter + + assertFalse( + await(returned, 200), + "a request made while a flush was already running was satisfied by that flush, which cannot have seen" + + " the requester's record"); + releaseFirstFlush.countDown(); + assertTrue(await(returned, 10_000), "a queued request was never served"); + + assertEquals( + 2, + flushStarts.get(), + "the five latecomers must share exactly one flush, taken as a batch after the first one ended"); + assertTrue( + startsSeenOnReturn.stream().allMatch(starts -> starts >= 2), + "each latecomer must be served by a flush that started after it was enqueued: " + startsSeenOnReturn); + assertEquals(2, flushCompletions.get(), "no flush ran twice for the same batch"); + } + + /** + * The same property under load, measured per request: when a request returns, a flush that started after the + * request was made must already have completed. Flushes are serialized by the pump, so counting completions is + * enough — a request satisfied by a flush that was already running would return with the completion count still at + * or below the value observed before it asked. + */ + @Test + void everyRequestIsSatisfiedByAFlushThatStartedAfterItWasMade() { + var flushStarts = new AtomicInteger(); + var flushCompletions = new AtomicInteger(); + var exporter = new InsightExporter() { + @Override + public void export(WorkflowInsightRecord record) {} + + @Override + public void flush() { + flushStarts.incrementAndGet(); + sleep(1); + flushCompletions.incrementAndGet(); + } + }; + var scheduler = scheduler(sharedWorkers(), exporter); + + int requests = 120; + var done = new CountDownLatch(requests); + var violations = new CopyOnWriteArrayList(); + for (int i = 0; i < requests; i++) { + String executionArn = arn(i); + InsightPlugin execution = Executions.plugin(scheduler, executionArn); + start("load-flusher-" + i, () -> { + scheduler.schedule(execution, record(executionArn, "SUCCEEDED")); + scheduler.drain(execution); + int startsBefore = flushStarts.get(); + scheduler.flush(); + if (flushCompletions.get() <= startsBefore) { + violations.add("returned with completions=" + flushCompletions.get() + " after observing starts=" + + startsBefore); + } + done.countDown(); + }); + } + assertTrue(await(done, 60_000), "a request was never served"); + scheduler.drainAll(); + + System.out.printf("COALESCING under load: %d requests satisfied by %d flushes%n", requests, flushStarts.get()); + assertEquals(List.of(), violations, "a request was credited to a flush that was already running"); + assertTrue(flushStarts.get() >= 1); + assertTrue( + flushStarts.get() <= requests, + "at most one flush per request: " + flushStarts.get() + " for " + requests); + } + + private static Thread start(String name, Runnable body) { + var thread = new Thread(body, name); + thread.setDaemon(true); + thread.start(); + return thread; + } + + private static boolean await(CountDownLatch latch, long timeoutMillis) { + try { + return latch.await(timeoutMillis, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + private static void awaitBarrier(CyclicBarrier barrier) { + try { + barrier.await(60, TimeUnit.SECONDS); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + private static void sleep(long millis) { + if (millis <= 0) { + return; + } + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } +} diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushSerializationTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushSerializationTest.java new file mode 100644 index 000000000..ff6254539 --- /dev/null +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushSerializationTest.java @@ -0,0 +1,434 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; + +/** + * Contract tests for the exporter-facing flush guarantee: {@code flush()} is served by the export pump, so it is never + * called concurrently with {@code export()} on the same plugin instance, and cannot be starved by a queue that keeps + * receiving records. + * + *

The cadence itself — at most one flush per request, requests that overlap sharing one flush — is covered by + * {@link ExportSchedulerFlushCoalescingTest}. + */ +class ExportSchedulerFlushSerializationTest { + + /** Longest a flush may take to be served before the property under test is considered broken. */ + private static final long FLUSH_DEADLINE_MILLIS = 2_000; + + private static String arn(int index) { + return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1"; + } + + private static WorkflowInsightRecord record(String executionArn, String status) { + var r = new WorkflowInsightRecord(); + r.executionArn = executionArn; + r.status = status; + return r; + } + + private static ExportScheduler scheduler( + Executor executor, List failures, InsightExporter... exporters) { + return new ExportScheduler(List.of(exporters), (rec, exp) -> exp.export(rec), failures::add, executor); + } + + /** Unbounded thread-per-task executor, like the cached pool the plugin injects in production. */ + private static Executor sharedWorkers() { + return command -> { + var thread = new Thread(command, "test-export-worker"); + thread.setDaemon(true); + thread.start(); + }; + } + + /** + * Records, per exporter instance, whether an {@code export()} and a {@code flush()} were ever inside the exporter + * at the same time, and how many of each ran concurrently. + */ + private static final class OverlapProbeExporter implements InsightExporter { + private final long exportMillis; + private final long flushMillis; + final AtomicInteger inExport = new AtomicInteger(); + final AtomicInteger inFlush = new AtomicInteger(); + final AtomicInteger maxConcurrentExports = new AtomicInteger(); + final AtomicInteger maxConcurrentFlushes = new AtomicInteger(); + final AtomicInteger flushes = new AtomicInteger(); + final List exported = new CopyOnWriteArrayList<>(); + final AtomicBoolean overlapped = new AtomicBoolean(); + + OverlapProbeExporter(long exportMillis, long flushMillis) { + this.exportMillis = exportMillis; + this.flushMillis = flushMillis; + } + + @Override + public void export(WorkflowInsightRecord record) { + trackMax(maxConcurrentExports, inExport.incrementAndGet()); + try { + checkOverlap(); + sleep(exportMillis); + checkOverlap(); + exported.add(record.status() + "@" + record.executionArn()); + } finally { + inExport.decrementAndGet(); + } + } + + @Override + public void flush() { + trackMax(maxConcurrentFlushes, inFlush.incrementAndGet()); + try { + checkOverlap(); + sleep(flushMillis); + checkOverlap(); + flushes.incrementAndGet(); + } finally { + inFlush.decrementAndGet(); + } + } + + private void checkOverlap() { + if (inExport.get() > 0 && inFlush.get() > 0) { + overlapped.set(true); + } + } + + private static void trackMax(AtomicInteger max, int observed) { + max.accumulateAndGet(observed, Math::max); + } + } + + @Test + void aFlushNeverOverlapsAnExportEvenWithManyExecutionsEndingAtOnce() throws Exception { + var first = new OverlapProbeExporter(15, 5); + var second = new OverlapProbeExporter(15, 5); + var failures = new CopyOnWriteArrayList(); + var scheduler = scheduler(sharedWorkers(), failures, first, second); + + int executions = 6; + var barrier = new CyclicBarrier(executions); + var threads = new ArrayList(); + for (int i = 0; i < executions; i++) { + String executionArn = arn(i); + InsightPlugin execution = Executions.plugin(scheduler, executionArn); + var thread = new Thread( + () -> { + awaitBarrier(barrier); + // What an invocation does: a few RUNNING snapshots, the terminal record, then drain + flush. + for (int change = 0; change < 3; change++) { + scheduler.schedule(execution, record(executionArn, "RUNNING")); + } + scheduler.schedule(execution, record(executionArn, "SUCCEEDED")); + scheduler.drain(execution); + scheduler.flush(); + }, + "invocation-" + i); + thread.setDaemon(true); + threads.add(thread); + thread.start(); + } + for (Thread thread : threads) { + thread.join(30_000); + assertFalse(thread.isAlive(), thread.getName() + " never returned from drain/flush"); + } + + for (OverlapProbeExporter exporter : List.of(first, second)) { + assertFalse(exporter.overlapped.get(), "flush() ran while an export was in flight on the same exporter"); + assertEquals(1, exporter.maxConcurrentExports.get(), "exports must stay serialized"); + assertEquals(1, exporter.maxConcurrentFlushes.get(), "one flush at a time, one fan-out per batch"); + // At most one flush per invocation end, and at least one: ends that land together share a flush, so the + // count is bounded by the number of ends rather than equal to it. + assertTrue(exporter.flushes.get() >= 1, "every invocation end must be covered by a flush"); + assertTrue( + exporter.flushes.get() <= executions, + "at most one flush per invocation end: " + exporter.flushes.get() + " for " + executions); + for (int i = 0; i < executions; i++) { + assertTrue( + exporter.exported.contains("SUCCEEDED@" + arn(i)), + "terminal record of " + arn(i) + " never reached the exporter"); + } + } + assertTrue(failures.isEmpty(), "no failure should be reported: " + failures); + } + + @Test + void twoFlushRequestsQueuedTogetherShareOneFlush() throws Exception { + var exporting = new CountDownLatch(1); + var release = new CountDownLatch(1); + var flushes = new AtomicInteger(); + var exporter = new InsightExporter() { + @Override + public void export(WorkflowInsightRecord record) { + exporting.countDown(); + await(release); + } + + @Override + public void flush() { + flushes.incrementAndGet(); + } + }; + var scheduler = scheduler(sharedWorkers(), new CopyOnWriteArrayList<>(), exporter); + + scheduler.schedule(Executions.plugin(scheduler, arn(0)), record(arn(0), "SUCCEEDED")); + assertTrue(exporting.await(5, TimeUnit.SECONDS), "the pump is inside the exporter"); + + var flushed = new CountDownLatch(2); + for (int i = 0; i < 2; i++) { + var flusher = new Thread( + () -> { + scheduler.flush(); + flushed.countDown(); + }, + "flusher-" + i); + flusher.setDaemon(true); + flusher.start(); + } + Thread.sleep(200); // let both requests queue up behind the in-flight export + + release.countDown(); + assertTrue(flushed.await(5, TimeUnit.SECONDS), "both flush requests must be served"); + assertEquals( + 1, + flushes.get(), + "two requests queued together are taken as one batch and share a single flush: both drained their own" + + " record before asking, so one flush covers both"); + } + + /** Distinct {@link Error} type so the test asserts on this exact failure rather than any Error. */ + private static final class FlushError extends Error { + FlushError() { + super("flush blew up with an Error"); + } + } + + @Test + void aFlushThatThrowsStillReleasesTheInvocationAndLetsTheOtherExportersFlush() throws Exception { + var flushed = new AtomicInteger(); + var throwsException = new InsightExporter() { + @Override + public void export(WorkflowInsightRecord record) {} + + @Override + public void flush() { + throw new IllegalStateException("flush blew up"); + } + }; + var throwsError = new InsightExporter() { + @Override + public void export(WorkflowInsightRecord record) {} + + @Override + public void flush() { + throw new FlushError(); + } + }; + var healthy = new InsightExporter() { + @Override + public void export(WorkflowInsightRecord record) {} + + @Override + public void flush() { + flushed.incrementAndGet(); + } + }; + var failures = new CopyOnWriteArrayList(); + var scheduler = scheduler(sharedWorkers(), failures, throwsException, throwsError, healthy); + + assertTrue(returnsWithin(scheduler::flush, FLUSH_DEADLINE_MILLIS), "a throwing flush stranded the invocation"); + + assertEquals(1, flushed.get(), "the healthy exporter still flushed"); + assertEquals(2, failures.size(), "both failures are reported, neither escapes: " + failures); + assertTrue( + failures.stream().anyMatch(t -> t instanceof IllegalStateException), + "the thrown exception is reported"); + assertTrue(failures.stream().anyMatch(t -> t instanceof FlushError), "the thrown Error is reported"); + } + + @Test + void anErrorFromTheOnlyExportersFlushStillReleasesTheInvocation() throws Exception { + var failures = new CopyOnWriteArrayList(); + var onlyExporter = new InsightExporter() { + @Override + public void export(WorkflowInsightRecord record) {} + + @Override + public void flush() { + throw new FlushError(); + } + }; + var scheduler = scheduler(sharedWorkers(), failures, onlyExporter); + + assertTrue(returnsWithin(scheduler::flush, FLUSH_DEADLINE_MILLIS), "an Error from flush() stranded the caller"); + assertEquals(1, failures.size(), "the Error is reported, not propagated: " + failures); + assertTrue(failures.get(0) instanceof FlushError); + } + + @Test + void aFlushIsServedWhileTheQueueKeepsReceivingRecords() throws Exception { + var exports = new AtomicInteger(); + var flushes = new AtomicInteger(); + var exporter = new InsightExporter() { + @Override + public void export(WorkflowInsightRecord record) { + sleep(2); + exports.incrementAndGet(); + } + + @Override + public void flush() { + flushes.incrementAndGet(); + } + }; + var scheduler = scheduler(sharedWorkers(), new CopyOnWriteArrayList<>(), exporter); + + // A producer that never lets the queue run dry: it keeps re-scheduling a fixed, rotating set of executions, so + // `pending` stays non-empty (and bounded, since records coalesce per execution) for as long as it runs. + var rotation = new ArrayList(); + for (int i = 0; i < 50; i++) { + rotation.add(Executions.plugin(scheduler, arn(i))); + } + var stop = new AtomicBoolean(); + var scheduled = new AtomicInteger(); + var producing = new CountDownLatch(1); + var producer = new Thread( + () -> { + int index = 0; + while (!stop.get()) { + InsightPlugin execution = rotation.get(index++ % rotation.size()); + scheduler.schedule(execution, record(execution.executionArn, "RUNNING")); + scheduled.incrementAndGet(); + producing.countDown(); + } + }, + "record-producer"); + producer.setDaemon(true); + producer.start(); + int exportsBefore; + int scheduledBefore; + boolean served; + var exportsWhenServed = new AtomicInteger(); + var scheduledWhenServed = new AtomicInteger(); + var flushReturned = new CountDownLatch(1); + try { + assertTrue(producing.await(5, TimeUnit.SECONDS), "the producer never started scheduling"); + exportsBefore = exports.get(); + scheduledBefore = scheduled.get(); + + // On its own thread with a deadline: a starved flush must fail this test, not hang it. + var flusher = new Thread( + () -> { + scheduler.flush(); + exportsWhenServed.set(exports.get()); + scheduledWhenServed.set(scheduled.get()); + flushReturned.countDown(); + }, + "flusher"); + flusher.setDaemon(true); + flusher.start(); + served = flushReturned.await(FLUSH_DEADLINE_MILLIS, TimeUnit.MILLISECONDS); + } finally { + // Stop the producer before asserting, so a starved flush is released and its thread does not leak. + stop.set(true); + producer.join(10_000); + } + + assertTrue( + served, + "the flush was not served within " + FLUSH_DEADLINE_MILLIS + " ms while the queue kept receiving" + + " records; it must be served between records rather than after the queue drains"); + assertTrue(flushReturned.await(5, TimeUnit.SECONDS)); + assertEquals(1, flushes.get(), "the flush was served exactly once"); + assertTrue( + exportsWhenServed.get() > exportsBefore, "the pump kept exporting: the flush did not stall the queue"); + assertTrue( + scheduledWhenServed.get() > scheduledBefore, + "the queue was still receiving records when the flush was served"); + } + + @Test + void theFlushHappensOnTheCallingThreadWhenNoWorkerCouldBeStarted() { + Executor rejecting = command -> { + throw new RejectedExecutionException("no worker"); + }; + var flushThreads = new CopyOnWriteArrayList(); + var exporter = new InsightExporter() { + @Override + public void export(WorkflowInsightRecord record) {} + + @Override + public void flush() { + flushThreads.add(Thread.currentThread()); + } + }; + var failures = new CopyOnWriteArrayList(); + var scheduler = scheduler(rejecting, failures, exporter); + + scheduler.flush(); + + assertEquals(1, flushThreads.size(), "the flush must still happen when no worker can be started"); + assertSame(Thread.currentThread(), flushThreads.get(0), "the invocation boundary flushes inline"); + assertFalse(failures.isEmpty(), "the rejected worker is reported"); + } + + /** Runs {@code action} on its own thread and reports whether it returned within the deadline. */ + private static boolean returnsWithin(Runnable action, long timeoutMillis) throws InterruptedException { + var returned = new CountDownLatch(1); + var thread = new Thread( + () -> { + action.run(); + returned.countDown(); + }, + "deadline-runner"); + thread.setDaemon(true); + thread.start(); + return returned.await(timeoutMillis, TimeUnit.MILLISECONDS); + } + + private static void awaitBarrier(CyclicBarrier barrier) { + try { + barrier.await(10, TimeUnit.SECONDS); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + private static void await(CountDownLatch latch) { + try { + if (!latch.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("latch not released"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } + + private static void sleep(long millis) { + if (millis <= 0) { + return; + } + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } +} diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerReentrantFlushTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerReentrantFlushTest.java new file mode 100644 index 000000000..bb0577ca2 --- /dev/null +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerReentrantFlushTest.java @@ -0,0 +1,131 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +/** + * A {@code flush()} issued from the thread currently serving the export pump is refused and reported, not waited on. + * + *

That thread is the only one able to serve the request it would be making — flush requests are served by the pump, + * between records — so waiting for it is a wait-for cycle one thread wide, and the invocation never returns. With a + * single exporter the fan-out runs on the pump thread, so anything an exporter's {@code export()} does synchronously is + * enough to reach it. + */ +class ExportSchedulerReentrantFlushTest { + + /** Longest a call that must return promptly may take before the property under test is considered broken. */ + private static final long DEADLINE_MILLIS = 5_000; + + private static String arn(int index) { + return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1"; + } + + private static WorkflowInsightRecord record(String executionArn, String status) { + var r = new WorkflowInsightRecord(); + r.executionArn = executionArn; + r.status = status; + return r; + } + + /** Unbounded thread-per-task executor, like the cached pool the plugin injects in production. */ + private static Executor sharedWorkers() { + return command -> { + var thread = new Thread(command, "reentrant-flush-worker"); + thread.setDaemon(true); + thread.start(); + }; + } + + private static final class CountingExporter implements InsightExporter { + final AtomicInteger exports = new AtomicInteger(); + final AtomicInteger flushes = new AtomicInteger(); + + @Override + public void export(WorkflowInsightRecord record) { + exports.incrementAndGet(); + } + + @Override + public void flush() { + flushes.incrementAndGet(); + } + } + + @Test + void flushReenteredFromThePumpThreadIsRefusedReportedAndLeavesTheSchedulerUsable() throws Exception { + var exporter = new CountingExporter(); + var failures = new CopyOnWriteArrayList(); + var holder = new AtomicReference(); + var reentrantFlushReturned = new CountDownLatch(1); + var flushesSeenByTheRefusedCall = new AtomicInteger(-1); + + var scheduler = new ExportScheduler( + List.of(exporter), + (rec, exp) -> { + exp.export(rec); + if (reentrantFlushReturned.getCount() > 0 && "SUCCEEDED".equals(rec.status())) { + // Re-entering the scheduler from inside the fan-out: this is the pump's own thread. + holder.get().flush(); + flushesSeenByTheRefusedCall.set(exporter.flushes.get()); + reentrantFlushReturned.countDown(); + } + }, + failures::add, + sharedWorkers()); + holder.set(scheduler); + + var drainReturned = new CountDownLatch(1); + var firstExecution = Executions.plugin(scheduler, arn(0)); + var invocation = new Thread( + () -> { + scheduler.schedule(firstExecution, record(arn(0), "SUCCEEDED")); + scheduler.drain(firstExecution); + drainReturned.countDown(); + }, + "reentrant-flush-invocation"); + invocation.setDaemon(true); + invocation.start(); + + assertTrue( + reentrantFlushReturned.await(DEADLINE_MILLIS, MILLISECONDS), + "flush() re-entered from the pump thread never returned: the only thread that can serve the request is" + + " the one waiting for it"); + assertTrue( + drainReturned.await(DEADLINE_MILLIS, MILLISECONDS), + "drain() never returned after the re-entrant flush"); + invocation.join(DEADLINE_MILLIS); + + // Reported, not silently swallowed, and nothing thrown into the caller. + assertEquals(1, failures.size(), "exactly one failure reported: " + failures); + assertTrue( + failures.get(0) instanceof IllegalStateException, + "the refusal is reported as an IllegalStateException: " + failures.get(0)); + assertTrue( + failures.get(0).getMessage().contains("flush()"), + "the report names the refused call: " + failures.get(0).getMessage()); + assertEquals(0, flushesSeenByTheRefusedCall.get(), "the refused request must not have reached an exporter"); + + // Still usable: the next invocation's record is exported and its flush — from a thread that is not the pump — + // is + // served exactly as before. + var secondExecution = Executions.plugin(scheduler, arn(1)); + scheduler.schedule(secondExecution, record(arn(1), "SUCCEEDED")); + scheduler.drain(secondExecution); + scheduler.flush(); + + assertEquals(2, exporter.exports.get(), "both records reached the exporter"); + assertEquals(1, exporter.flushes.get(), "the next invocation's flush is served normally"); + assertEquals(1, failures.size(), "no further failure after the refusal: " + failures); + } +} diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerTest.java index 4f2d8af42..2c3df1a3e 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerTest.java @@ -22,6 +22,13 @@ /** Contract tests for {@link ExportScheduler}: serial exports, latest-wins coalescing, drain, and exporter fan-out. */ class ExportSchedulerTest { + /** All single-execution cases below drive one invocation's plugin instance through the scheduler. */ + private static final String ARN = arn(0); + + private static String arn(int index) { + return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1"; + } + /** Runs submitted tasks only when the test asks, so pump timing is fully controlled. */ private static final class ManualExecutor implements Executor { final Deque tasks = new ArrayDeque<>(); @@ -57,7 +64,7 @@ public void export(WorkflowInsightRecord record) { } private static WorkflowInsightRecord record(String status) { - return record("arn:exec-a", status); + return record(ARN, status); } private static WorkflowInsightRecord record(String executionArn, String status) { @@ -72,9 +79,11 @@ void recordsOfDifferentExecutionsNeverDisplaceEachOther() { var executor = new ManualExecutor(); var exporter = new CapturingExporter(); var scheduler = scheduler(executor, new ArrayList<>(), exporter); + var executionA = Executions.plugin(scheduler, "arn:exec-a"); + var executionB = Executions.plugin(scheduler, "arn:exec-b"); - scheduler.schedule(record("arn:exec-a", "a-final")); - scheduler.schedule(record("arn:exec-b", "b-running")); + scheduler.schedule(executionA, record("arn:exec-a", "a-final")); + scheduler.schedule(executionB, record("arn:exec-b", "b-running")); executor.runAll(); assertEquals(List.of("a-final", "b-running"), statuses(exporter)); @@ -85,12 +94,16 @@ void coalescingStaysWithinOneExecutionAndExecutionsAreServedInFirstPendingOrder( var executor = new ManualExecutor(); var exporter = new CapturingExporter(); var scheduler = scheduler(executor, new ArrayList<>(), exporter); + var executionA = Executions.plugin(scheduler, "arn:exec-a"); + var executionB = Executions.plugin(scheduler, "arn:exec-b"); + + var executionC = Executions.plugin(scheduler, "arn:exec-c"); - scheduler.schedule(record("arn:exec-a", "a1")); - scheduler.schedule(record("arn:exec-b", "b1")); - scheduler.schedule(record("arn:exec-a", "a2")); // supersedes a1 but keeps a's place ahead of b - scheduler.schedule(record("arn:exec-b", "b2")); - scheduler.schedule(record("arn:exec-c", "c1")); + scheduler.schedule(executionA, record("arn:exec-a", "a1")); + scheduler.schedule(executionB, record("arn:exec-b", "b1")); + scheduler.schedule(executionA, record("arn:exec-a", "a2")); // supersedes a1 but keeps a's place ahead of b + scheduler.schedule(executionB, record("arn:exec-b", "b2")); + scheduler.schedule(executionC, record("arn:exec-c", "c1")); executor.runAll(); assertEquals(List.of("a2", "b2", "c1"), statuses(exporter)); @@ -111,14 +124,17 @@ public void export(WorkflowInsightRecord record) { } }; var scheduler = scheduler(sharedWorkers(), new ArrayList<>(), exporter); + var executionA = Executions.plugin(scheduler, "arn:exec-a"); + var executionB = Executions.plugin(scheduler, "arn:exec-b"); - scheduler.schedule(record("arn:exec-a", "a-running")); + scheduler.schedule(executionA, record("arn:exec-a", "a-running")); assertTrue(entered.await(5, TimeUnit.SECONDS), "a's first export is in flight"); - scheduler.schedule(record("arn:exec-a", "a-final")); - scheduler.schedule(record("arn:exec-b", "b-running")); - scheduler.schedule(record("arn:exec-b", "b-final")); + scheduler.schedule(executionA, record("arn:exec-a", "a-final")); + scheduler.schedule(executionB, record("arn:exec-b", "b-running")); + scheduler.schedule(executionB, record("arn:exec-b", "b-final")); release.countDown(); - scheduler.drain(); + scheduler.drain(executionA); + scheduler.drain(executionB); assertEquals(List.of("a-running", "a-final", "b-final"), statuses(exporter)); } @@ -133,8 +149,9 @@ void scheduleHandsTheRecordToAWorkerRatherThanExportingOnTheCallingThread() { var executor = new ManualExecutor(); var exporter = new CapturingExporter(); var scheduler = scheduler(executor, new ArrayList<>(), exporter); + var execution = Executions.plugin(scheduler, ARN); - scheduler.schedule(record("RUNNING")); + scheduler.schedule(execution, record("RUNNING")); assertTrue(exporter.records.isEmpty(), "nothing exported until a worker runs"); executor.runAll(); @@ -147,10 +164,11 @@ void updatesScheduledBeforeTheWorkerRunsCollapseIntoTheLatestRecord() { var executor = new ManualExecutor(); var exporter = new CapturingExporter(); var scheduler = scheduler(executor, new ArrayList<>(), exporter); + var execution = Executions.plugin(scheduler, ARN); - scheduler.schedule(record("r1")); - scheduler.schedule(record("r2")); - scheduler.schedule(record("r3")); + scheduler.schedule(execution, record("r1")); + scheduler.schedule(execution, record("r2")); + scheduler.schedule(execution, record("r3")); executor.runAll(); assertEquals(1, exporter.records.size(), "one pump, one latest record"); @@ -163,10 +181,11 @@ void recordScheduledAfterAPumpFinishesStartsANewPump() { var executor = new ManualExecutor(); var exporter = new CapturingExporter(); var scheduler = scheduler(executor, new ArrayList<>(), exporter); + var execution = Executions.plugin(scheduler, ARN); - scheduler.schedule(record("first")); + scheduler.schedule(execution, record("first")); executor.runAll(); - scheduler.schedule(record("second")); + scheduler.schedule(execution, record("second")); executor.runAll(); assertEquals(List.of("first", "second"), statuses(exporter)); @@ -187,14 +206,15 @@ public void export(WorkflowInsightRecord record) { } }; var scheduler = scheduler(sharedWorkers(), new ArrayList<>(), exporter); + var execution = Executions.plugin(scheduler, ARN); - scheduler.schedule(record("first")); + scheduler.schedule(execution, record("first")); assertTrue(entered.await(5, TimeUnit.SECONDS), "first export is in flight"); - scheduler.schedule(record("dropped-1")); - scheduler.schedule(record("dropped-2")); - scheduler.schedule(record("final")); + scheduler.schedule(execution, record("dropped-1")); + scheduler.schedule(execution, record("dropped-2")); + scheduler.schedule(execution, record("final")); release.countDown(); - scheduler.drain(); + scheduler.drain(execution); assertEquals(List.of("first", "final"), statuses(exporter)); } @@ -202,8 +222,9 @@ public void export(WorkflowInsightRecord record) { @Test void drainReturnsImmediatelyWhenIdle() { var scheduler = scheduler(new ManualExecutor(), new ArrayList<>(), new CapturingExporter()); - scheduler.drain(); - scheduler.drain(); + var execution = Executions.plugin(scheduler, ARN); + scheduler.drain(execution); + scheduler.drain(execution); } @Test @@ -221,14 +242,15 @@ public void export(WorkflowInsightRecord record) { } }; var scheduler = scheduler(sharedWorkers(), new ArrayList<>(), exporter); + var execution = Executions.plugin(scheduler, ARN); - scheduler.schedule(record("slow")); + scheduler.schedule(execution, record("slow")); assertTrue(entered.await(5, TimeUnit.SECONDS)); - scheduler.schedule(record("final")); + scheduler.schedule(execution, record("final")); var drained = new CountDownLatch(1); var drainer = new Thread(() -> { - scheduler.drain(); + scheduler.drain(execution); drained.countDown(); }); drainer.start(); @@ -243,9 +265,10 @@ public void export(WorkflowInsightRecord record) { void exportersRunOffTheSchedulingThread() { var exporter = new CapturingExporter(); var scheduler = scheduler(sharedWorkers(), new ArrayList<>(), exporter); + var execution = Executions.plugin(scheduler, ARN); - scheduler.schedule(record("RUNNING")); - scheduler.drain(); + scheduler.schedule(execution, record("RUNNING")); + scheduler.drain(execution); assertEquals(1, exporter.threads.size()); assertNotSame(Thread.currentThread(), exporter.threads.get(0)); @@ -259,9 +282,10 @@ void aFailingExporterNeverBlocksTheOthersForTheSameRecord() { throw new AssertionError("exporter blew up"); }; var scheduler = scheduler(sharedWorkers(), failures, bad, good); + var execution = Executions.plugin(scheduler, ARN); - scheduler.schedule(record("RUNNING")); - scheduler.drain(); + scheduler.schedule(execution, record("RUNNING")); + scheduler.drain(execution); assertEquals(1, good.records.size()); assertEquals(1, failures.size()); @@ -274,8 +298,9 @@ void exportersForOneRecordRunConcurrentlySoASlowExporterDoesNotDelayTheOthers() var fast = new CapturingExporter(); InsightExporter slow = record -> await(release); var scheduler = scheduler(sharedWorkers(), new ArrayList<>(), slow, fast); + var execution = Executions.plugin(scheduler, ARN); - scheduler.schedule(record("RUNNING")); + scheduler.schedule(execution, record("RUNNING")); long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); while (fast.records.isEmpty() && System.nanoTime() < deadline) { Thread.sleep(5); @@ -283,7 +308,7 @@ void exportersForOneRecordRunConcurrentlySoASlowExporterDoesNotDelayTheOthers() assertEquals(1, fast.records.size(), "fast exporter received the record while the slow one is still blocked"); release.countDown(); - scheduler.drain(); + scheduler.drain(execution); } @Test @@ -293,12 +318,13 @@ void drainExportsThePendingRecordInlineWhenNoWorkerCouldBeStarted() { var failures = new ArrayList(); var exporter = new CapturingExporter(); var scheduler = scheduler(executor, failures, exporter); + var execution = Executions.plugin(scheduler, ARN); - scheduler.schedule(record("final")); + scheduler.schedule(execution, record("final")); assertTrue(exporter.records.isEmpty(), "the hook thread does not export"); assertEquals(1, failures.size(), "the worker failure is reported"); - scheduler.drain(); + scheduler.drain(execution); assertEquals(List.of("final"), statuses(exporter)); assertSame(Thread.currentThread(), exporter.threads.get(0), "the invocation boundary delivers it"); @@ -310,14 +336,15 @@ void aLaterScheduleRetriesTheWorkerAfterARejection() { executor.reject = true; var exporter = new CapturingExporter(); var scheduler = scheduler(executor, new ArrayList<>(), exporter); + var execution = Executions.plugin(scheduler, ARN); - scheduler.schedule(record("older")); + scheduler.schedule(execution, record("older")); executor.reject = false; - scheduler.schedule(record("newer")); + scheduler.schedule(execution, record("newer")); executor.runAll(); assertEquals(List.of("newer"), statuses(exporter), "the retry exports the latest record"); - scheduler.drain(); + scheduler.drain(execution); assertEquals(1, exporter.records.size()); } @@ -333,15 +360,16 @@ void aDrainThatObservedTheHandleBeforeTheWorkerWasRejectedStillCompletesInline() var failures = new CopyOnWriteArrayList(); var exporter = new CapturingExporter(); var scheduler = scheduler(blockingRejector, failures, exporter); + var execution = Executions.plugin(scheduler, ARN); - var scheduling = new Thread(() -> scheduler.schedule(record("final")), "scheduling"); + var scheduling = new Thread(() -> scheduler.schedule(execution, record("final")), "scheduling"); scheduling.start(); assertTrue(submitted.await(5, TimeUnit.SECONDS), "the pump handle is published before execute rejects"); var drained = new CountDownLatch(1); var drainer = new Thread( () -> { - scheduler.drain(); + scheduler.drain(execution); drained.countDown(); }, "drainer"); @@ -357,7 +385,7 @@ void aDrainThatObservedTheHandleBeforeTheWorkerWasRejectedStillCompletesInline() } @Test - void flushAllRunsExporterFlushesConcurrentlySoASlowFlushDoesNotDelayTheOthers() throws Exception { + void flushRunsExporterFlushesConcurrentlySoASlowFlushDoesNotDelayTheOthers() throws Exception { var release = new CountDownLatch(1); var fastFlushed = new CountDownLatch(1); var slow = new InsightExporter() { @@ -382,19 +410,19 @@ public void flush() { var flushed = new CountDownLatch(1); new Thread(() -> { - scheduler.flushAll(); + scheduler.flush(); flushed.countDown(); }) .start(); assertTrue(fastFlushed.await(5, TimeUnit.SECONDS), "fast exporter flushed while the slow one is blocked"); - assertFalse(flushed.await(100, TimeUnit.MILLISECONDS), "flushAll waits for every exporter"); + assertFalse(flushed.await(100, TimeUnit.MILLISECONDS), "flush waits for every exporter"); release.countDown(); assertTrue(flushed.await(5, TimeUnit.SECONDS)); } @Test - void flushAllIsolatesAFailingFlush() { + void flushIsolatesAFailingFlush() { var failures = new CopyOnWriteArrayList(); var flushed = new CountDownLatch(1); var bad = new InsightExporter() { @@ -417,7 +445,7 @@ public void flush() { }; var scheduler = scheduler(sharedWorkers(), failures, bad, good); - scheduler.flushAll(); + scheduler.flush(); assertEquals(0, flushed.getCount(), "the healthy exporter still flushed"); assertEquals(1, failures.size()); diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExporterIsolationTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExporterIsolationTest.java index 61fe8ae68..4a297c100 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExporterIsolationTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExporterIsolationTest.java @@ -69,14 +69,17 @@ public void export(WorkflowInsightRecord record) { void firstExporterMutationsDoNotLeakIntoLaterExporter() { var mutating = new MutatingExporter(); var good = new CapturingExporter(); - var plugin = (WorkflowInsight.InsightPlugin) WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) - .content(ContentConfig.builder() - .addOverride(OperationOverride.withResult("compute", r -> r)) - .build()) - .addExporter(mutating) - .addExporter(good) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) + .content(ContentConfig.builder() + .addOverride(OperationOverride.withResult("compute", r -> r)) + .build()) + .addExporter(mutating) + .addExporter(good) + .build()), + ARN, + START); Map input = new LinkedHashMap<>(); input.put("k", "v"); diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InputSnapshotTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InputSnapshotTest.java index abd5f89f7..4af7d55fb 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InputSnapshotTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InputSnapshotTest.java @@ -13,7 +13,6 @@ import java.util.function.Function; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.lambda.model.OperationStatus; -import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.InvocationStatus; @@ -64,8 +63,11 @@ private Map ops() { @Test void handlerMutationAfterStartDoesNotCorruptCachedInputSnapshot() { var exporter = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight( - WorkflowInsightConfig.builder().addExporter(exporter).build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight( + WorkflowInsightConfig.builder().addExporter(exporter).build()), + ARN, + START); // A mutable input whose nested list the handler mutates after the invocation has started. List items = new ArrayList<>(); @@ -99,13 +101,16 @@ void mutatingInputTransformDoesNotAccumulateAcrossEmissions() { return v; }; var exporter = new CapturingExporter(); - var plugin = (WorkflowInsight.InsightPlugin) WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) - .content(ContentConfig.builder() - .inputTransform(mutatingTransform) - .build()) - .addExporter(exporter) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) + .content(ContentConfig.builder() + .inputTransform(mutatingTransform) + .build()) + .addExporter(exporter) + .build()), + ARN, + START); List items = new ArrayList<>(); items.add("a"); diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InsightFatalBoundaryTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InsightFatalBoundaryTest.java new file mode 100644 index 000000000..e282028e9 --- /dev/null +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InsightFatalBoundaryTest.java @@ -0,0 +1,237 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.UndeclaredThrowableException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.lambda.durable.insight.internal.FatalErrors; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.plugin.OperationChangeInfo; +import software.amazon.lambda.durable.plugin.PluginRunner; + +class InsightFatalBoundaryTest { + private static final String ARN = "arn:aws:lambda:us-west-2:1:function:f:$LATEST/durable-execution/e/i"; + + public static final class FatalPayload { + private final Error failure; + + FatalPayload(Error failure) { + this.failure = failure; + } + + public String getValue() { + throw failure; + } + } + + private static final class TestVmError extends VirtualMachineError {} + + @SuppressWarnings("removal") + private static Error fatal(String kind) { + return "vm".equals(kind) ? new TestVmError() : new ThreadDeath(); + } + + private static RuntimeException transport(Error error) { + return new CompletionException( + new ExecutionException(new UndeclaredThrowableException(new InvocationTargetException(error)))); + } + + @ParameterizedTest + @CsvSource({"vm,false", "vm,true", "thread,false", "thread,true"}) + void resultRedactorPropagatesFatalWithoutReplacingIdentity(String kind, boolean wrapped) { + var failure = fatal(kind); + assertSame( + failure, + assertThrows( + Error.class, + () -> WorkflowInsight.applyResultOverride( + value -> { + if (wrapped) throw transport(failure); + throw failure; + }, + "{\"private\":true}"))); + } + + @ParameterizedTest + @CsvSource({"vm", "thread"}) + void jacksonGetterFatalEscapesSnapshotAndSerialization(String kind) { + var failure = fatal(kind); + var value = new FatalPayload(failure); + assertSame(failure, assertThrows(Error.class, () -> Json.deepCopyContent(value))); + assertSame(failure, assertThrows(Error.class, () -> Json.stringify(value))); + assertSame(failure, assertThrows(Error.class, () -> Json.prettyStringify(value))); + assertSame(failure, assertThrows(Error.class, () -> Json.byteSize(value))); + var info = new InvocationInfo("r", ARN, true, Instant.now(), value, Map.of(), Map.of()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder().build()), info); + assertSame(failure, assertThrows(Error.class, () -> plugin.onInvocationStart(info))); + assertFalse(Executions.outstanding(plugin)); + } + + @ParameterizedTest + @CsvSource({"vm", "thread"}) + void rendererFatalReachesTheBoundary(String kind) { + var failure = fatal(kind); + var exporter = new InsightExporter() { + @Override + public void export(WorkflowInsightRecord record) {} + + @Override + public Integer maxRecordSizeBytes() { + return 1; + } + + @Override + public Object render(WorkflowInsightRecord record) { + throw transport(failure); + } + }; + assertSame( + failure, + assertThrows(Error.class, () -> WorkflowInsight.exportRecord(new WorkflowInsightRecord(), exporter))); + } + + @Test + void finishedFatalPumpLeavesExceptionalSignalAndCannotAcceptMoreWork() { + var failure = new TestVmError(); + var tasks = new ArrayList(); + InsightExporter exporter = record -> { + throw transport(failure); + }; + var scheduler = + new ExportScheduler(List.of(exporter), WorkflowInsight::exportRecord, ignored -> {}, tasks::add); + var owner = Executions.plugin(scheduler, ARN); + scheduler.schedule(owner, new WorkflowInsightRecord()); + var signal = owner.settled; + assertSame(failure, assertThrows(Error.class, () -> tasks.remove(0).run())); + assertTrue(signal.isCompletedExceptionally()); + assertSame( + failure, assertThrows(CompletionException.class, signal::join).getCause()); + assertFalse(Executions.outstanding(owner)); + assertSame(failure, assertThrows(Error.class, () -> scheduler.drain(owner))); + var later = Executions.plugin(scheduler, ARN + "later"); + assertSame(failure, assertThrows(Error.class, () -> scheduler.schedule(later, new WorkflowInsightRecord()))); + assertFalse(Executions.outstanding(later)); + assertEquals(0, scheduler.retainedInvocationCount()); + } + + @Test + void fatalExecutorSubmissionReleasesQueuedOwner() { + var failure = new TestVmError(); + var scheduler = new ExportScheduler(List.of(), (record, exporter) -> {}, ignored -> {}, command -> { + throw transport(failure); + }); + var owner = Executions.plugin(scheduler, ARN); + assertSame(failure, assertThrows(Error.class, () -> scheduler.schedule(owner, new WorkflowInsightRecord()))); + assertFalse(Executions.outstanding(owner)); + assertEquals(0, scheduler.retainedInvocationCount()); + } + + @Test + void fatalFailureHandlerCannotTurnExporterFailureIntoSuccess() { + var failure = new TestVmError(); + var tasks = new ArrayList(); + var scheduler = new ExportScheduler( + List.of(record -> {}), + (record, exporter) -> { + throw new AssertionError("ordinary exporter error"); + }, + ignored -> { + throw transport(failure); + }, + tasks::add); + var owner = Executions.plugin(scheduler, ARN); + scheduler.schedule(owner, new WorkflowInsightRecord()); + assertSame(failure, assertThrows(Error.class, () -> tasks.remove(0).run())); + assertSame(failure, assertThrows(Error.class, scheduler::flush)); + assertFalse(Executions.outstanding(owner)); + } + + @Test + void transportCyclesAreBoundedAndBusinessCausesAreNotUnwrapped() { + class Cycle extends CompletionException {} + var left = new Cycle(); + var right = new Cycle(); + left.initCause(right); + right.initCause(left); + assertTimeoutPreemptively(Duration.ofSeconds(1), () -> assertNull(FatalErrors.find(left))); + assertNull(FatalErrors.find(new IllegalStateException("business wrapper", new TestVmError()))); + assertDoesNotThrow( + () -> WorkflowInsight.logSafely("ordinary linkage error", new NoClassDefFoundError("optional"))); + } + + @Test + void observedBackgroundFatalPreventsLaterHooksFromRunningTransforms() { + var failure = new TestVmError(); + var outputCalls = new AtomicInteger(); + InsightExporter exporter = record -> { + throw failure; + }; + var info = new InvocationInfo("r", ARN, true, Instant.now(), null, Map.of(), Map.of()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) + .content(ContentConfig.builder() + .outputTransform(value -> { + outputCalls.incrementAndGet(); + return value; + }) + .build()) + .addExporter(exporter) + .build()), + info); + plugin.onInvocationStart(info); + assertSame(failure, assertThrows(Error.class, plugin::drainExports)); + assertSame( + failure, + assertThrows( + Error.class, + () -> plugin.onOperationChange(new OperationChangeInfo("r", ARN, Map.of(), Map.of())))); + var end = new InvocationEndInfo( + "r", ARN, true, info.executionStartTime(), Map.of(), InvocationStatus.SUCCEEDED, null, null, "output"); + assertSame(failure, assertThrows(Error.class, () -> plugin.onInvocationEnd(end))); + assertEquals(0, outputCalls.get()); + assertFalse(Executions.outstanding(plugin)); + } + + @ParameterizedTest + @CsvSource({"vm", "thread"}) + void sdkPluginRunnerObservesTheAsynchronousExporterFatal(String kind) { + var failure = fatal(kind); + InsightExporter exporter = record -> { + throw transport(failure); + }; + var factory = WorkflowInsight.workflowInsight( + WorkflowInsightConfig.builder().addExporter(exporter).build()); + var runner = new PluginRunner(List.of(factory)); + var start = new InvocationInfo("r", ARN, true, Instant.now(), null, Map.of(), Map.of()); + runner.onInvocationStart(start); + var end = new InvocationEndInfo( + "r", ARN, true, start.executionStartTime(), Map.of(), InvocationStatus.SUCCEEDED, null, null, "result"); + assertTimeoutPreemptively( + Duration.ofSeconds(3), + () -> assertSame(failure, assertThrows(Error.class, () -> runner.onInvocationEnd(end)))); + } +} diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InsightFatalFailureTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InsightFatalFailureTest.java new file mode 100644 index 000000000..c62d91232 --- /dev/null +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InsightFatalFailureTest.java @@ -0,0 +1,172 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; + +class InsightFatalFailureTest { + private static final String ARN = "arn:aws:lambda:us-west-2:1:function:f:$LATEST/durable-execution/e/i"; + private static final Instant START = Instant.parse("2026-10-06T00:00:00Z"); + + private static final class TestVmError extends VirtualMachineError { + TestVmError() { + super("deterministic VM failure"); + } + } + + @SuppressWarnings("removal") + private static Error fatal(String kind) { + return "vm".equals(kind) ? new TestVmError() : new ThreadDeath(); + } + + private static RuntimeException wrapped(Error fatal) { + return new CompletionException(new ExecutionException(fatal)); + } + + private static InvocationEndInfo end() { + return new InvocationEndInfo("r", ARN, true, START, Map.of(), InvocationStatus.SUCCEEDED, null, null, "output"); + } + + @ParameterizedTest + @CsvSource({"vm,false", "vm,true", "thread,false", "thread,true"}) + void outputTransformFatalEscapesInvocationEnd(String kind, boolean wrap) { + var failure = fatal(kind); + Function transform = value -> { + if (wrap) throw wrapped(failure); + throw failure; + }; + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .content(ContentConfig.builder() + .outputTransform(transform) + .build()) + .build()), + ARN, + START); + plugin.onInvocationStart(Executions.info(ARN)); + assertSame(failure, assertThrows(Error.class, () -> plugin.onInvocationEnd(end()))); + assertFalse(Executions.outstanding(plugin)); + } + + @ParameterizedTest + @CsvSource({"vm,false", "vm,true", "thread,false", "thread,true"}) + void asynchronousExporterFatalReachesInvocationEnd(String kind, boolean wrap) { + var failure = fatal(kind); + InsightExporter exporter = record -> { + if (wrap) throw wrapped(failure); + throw failure; + }; + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight( + WorkflowInsightConfig.builder().addExporter(exporter).build()), + ARN, + START); + plugin.onInvocationStart(Executions.info(ARN)); + assertTimeoutPreemptively( + Duration.ofSeconds(3), + () -> assertSame(failure, assertThrows(Error.class, () -> plugin.onInvocationEnd(end())))); + assertFalse(Executions.outstanding(plugin)); + assertSame(failure, assertThrows(Error.class, plugin::drainExports), "a finished worker cannot hide its fatal"); + } + + @ParameterizedTest + @CsvSource({"vm,false", "vm,true", "thread,false", "thread,true"}) + void flushFatalReachesInvocationEnd(String kind, boolean wrap) { + var failure = fatal(kind); + var exporter = new InsightExporter() { + @Override + public void export(WorkflowInsightRecord record) {} + + @Override + public void flush() { + if (wrap) throw wrapped(failure); + throw failure; + } + }; + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight( + WorkflowInsightConfig.builder().addExporter(exporter).build()), + ARN, + START); + assertTimeoutPreemptively( + Duration.ofSeconds(3), + () -> assertSame(failure, assertThrows(Error.class, () -> plugin.onInvocationEnd(end())))); + assertFalse(Executions.outstanding(plugin)); + } + + @Test + void fatalFanOutReleasesCallerAndQueuedInvocationWithoutWaitingForBlockedPeer() throws Exception { + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + var failure = new TestVmError(); + InsightExporter blocked = record -> { + entered.countDown(); + try { + assertTrue(release.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + }; + InsightExporter throwing = record -> { + try { + assertTrue(entered.await(2, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + throw wrapped(failure); + }; + var scheduler = new ExportScheduler(List.of(blocked, throwing), WorkflowInsight::exportRecord, ignored -> {}); + var first = Executions.plugin(scheduler, ARN); + var queued = Executions.plugin(scheduler, ARN + "2"); + var record = new WorkflowInsightRecord(); + record.executionArn = ARN; + try { + synchronized (scheduler) { + scheduler.schedule(first, record); + scheduler.schedule(queued, record); + } + assertTimeoutPreemptively( + Duration.ofSeconds(2), + () -> assertSame(failure, assertThrows(Error.class, () -> scheduler.drain(first)))); + assertFalse(Executions.outstanding(first)); + assertFalse(Executions.outstanding(queued)); + assertEquals(0, scheduler.retainedInvocationCount()); + assertSame(failure, assertThrows(Error.class, scheduler::flush)); + } finally { + release.countDown(); + } + } + + @Test + void ordinaryApplicationExceptionWithFatalAsDataRemainsFailOpen() { + var record = new WorkflowInsightRecord(); + assertEquals(null, WorkflowInsight.applyDataContent("input", "value", true, value -> { + throw new IllegalStateException("business error", new TestVmError()); + })); + WorkflowInsight.exportRecord(record, value -> { + throw new AssertionError("ordinary plugin error"); + }); + } +} diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/JsonJavaTimeTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/JsonJavaTimeTest.java index e57366bba..91d5ec9da 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/JsonJavaTimeTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/JsonJavaTimeTest.java @@ -55,10 +55,13 @@ public void export(WorkflowInsightRecord record) { @Test void pluginOutputWithInstantInInputSerializesInsteadOfDropping() { var exporter = new CapturingExporter(); - var plugin = (WorkflowInsight.InsightPlugin) WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) - .addExporter(exporter) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) + .addExporter(exporter) + .build()), + ARN, + START); Map input = new LinkedHashMap<>(); input.put("startedAt", TS); diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/MutableNumberIsolationTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/MutableNumberIsolationTest.java index 19f906209..e9191ee8f 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/MutableNumberIsolationTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/MutableNumberIsolationTest.java @@ -113,11 +113,14 @@ public void export(WorkflowInsightRecord record) { } }; var good = new CapturingExporter(); - var plugin = (WorkflowInsight.InsightPlugin) WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) - .addExporter(mutating) - .addExporter(good) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) + .addExporter(mutating) + .addExporter(good) + .build()), + ARN, + START); AtomicInteger topLevel = new AtomicInteger(3); List list = new ArrayList<>(); diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/OperationErrorIdentityTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/OperationErrorIdentityTest.java index 5e67cd554..116823363 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/OperationErrorIdentityTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/OperationErrorIdentityTest.java @@ -15,7 +15,6 @@ import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.lambda.durable.exception.DurableOperationException; -import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationStatus; import software.amazon.lambda.durable.plugin.OperationChangeItemInfo; @@ -72,8 +71,11 @@ private Map failedOps(Throwable opError) { @Test void operationAndExecutionErrorUseCheckpointedErrorObjectIdentity() { var exporter = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight( - WorkflowInsightConfig.builder().addExporter(exporter).build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight( + WorkflowInsightConfig.builder().addExporter(exporter).build()), + ARN, + START); Throwable opError = wrapped("CustomerValidationError", "invalid postal code"); Throwable execError = wrapped("OrchestrationFailure", "workflow aborted"); @@ -98,8 +100,11 @@ void operationAndExecutionErrorUseCheckpointedErrorObjectIdentity() { @Test void fallsBackToThrowableFieldsWhenErrorObjectFieldsMissing() { var exporter = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight( - WorkflowInsightConfig.builder().addExporter(exporter).build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight( + WorkflowInsightConfig.builder().addExporter(exporter).build()), + ARN, + START); // ErrorObject present but errorType null: name falls back to the throwable's simple class name. ErrorObject partial = diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/OperationOrderingTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/OperationOrderingTest.java index d1ca11b03..fb5e9a94c 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/OperationOrderingTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/OperationOrderingTest.java @@ -44,10 +44,13 @@ private static OperationChangeItemInfo item( private WorkflowInsightRecord emitStart(Map ops) { var exporter = new CapturingExporter(); - var plugin = (WorkflowInsight.InsightPlugin) WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) - .addExporter(exporter) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) + .addExporter(exporter) + .build()), + ARN, + START); plugin.onInvocationStart(new InvocationInfo("req", ARN, true, START, "in", ops, Map.of())); plugin.drainExports(); return exporter.records.get(0); diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/PluginThrowableContainmentTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/PluginThrowableContainmentTest.java index 4a13268be..5b7958c37 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/PluginThrowableContainmentTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/PluginThrowableContainmentTest.java @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.insight; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -13,18 +14,18 @@ import java.util.Map; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.lambda.model.OperationStatus; -import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.InvocationStatus; import software.amazon.lambda.durable.plugin.OperationChangeItemInfo; +import software.amazon.lambda.durable.plugin.PluginRunner; /** - * Fix 2 — plugin {@link Throwable} containment. A plugin fault at any plugin-owned boundary (record construction, input + * Non-fatal plugin containment. An ordinary plugin fault at any plugin-owned boundary (record construction, input * snapshotting, transforms, and each exporter's render/export/flush) must be caught — including {@link Error}s such as * an optional exporter's {@code NoClassDefFoundError} — so one failing exporter never blocks the others and no plugin * fault disrupts durable execution. Tests use deterministic {@link AssertionError}/{@link Error} throwers rather than - * inducing a real {@code StackOverflowError}. + * inducing a real {@code StackOverflowError}; VM/thread-termination propagation has separate tests. */ class PluginThrowableContainmentTest { @@ -83,6 +84,39 @@ private InvocationEndInfo end(Object input) { "req", ARN, true, START, ops("compute"), InvocationStatus.SUCCEEDED, null, input, "out"); } + @Test + void aNullExecutionArnEscapesNoHook() { + // The SDK's contract is that a plugin fault never disrupts durable execution, so an invocation whose execution + // ARN the plugin cannot use must be contained rather than thrown back. It is contained one step earlier now: + // identity is taken when the instance is built, so the failure happens in the factory and no hook is ever + // dispatched. That containment belongs to the SDK, so it is asserted through the SDK's own runner — which is + // also what makes the old worst case ("the state removal runs last, in a finally, and a ConcurrentHashMap + // cannot remove a null key") unreachable: there is no map and no removal. + var exporter = new CapturingExporter(); + var environment = WorkflowInsight.workflowInsight( + WorkflowInsightConfig.builder().addExporter(exporter).build()); + + InvocationInfo nullStart = new InvocationInfo("req", null, true, START, "in", ops("compute"), Map.of()); + InvocationEndInfo nullEnd = new InvocationEndInfo( + "req", null, true, START, ops("compute"), InvocationStatus.SUCCEEDED, null, "in", "out"); + + var runner = new PluginRunner(List.of(environment)); + assertDoesNotThrow(() -> runner.onInvocationStart(nullStart), "onInvocationStart must contain a null ARN"); + assertDoesNotThrow( + () -> runner.onOperationChange(new software.amazon.lambda.durable.plugin.OperationChangeInfo( + "req", null, ops("compute"), ops("compute"))), + "onOperationChange must contain a null ARN"); + assertDoesNotThrow(() -> runner.onInvocationEnd(nullEnd), "onInvocationEnd must contain a null ARN"); + assertEquals(0, exporter.records.size(), "an invocation with no usable ARN emits nothing"); + + // The environment is still usable afterwards: a well-formed invocation still emits and flushes. + var plugin = Executions.plugin(environment, ARN, START); + plugin.onInvocationStart(start("in")); + plugin.onInvocationEnd(end("in")); + assertEquals(1, exporter.records.size(), "the environment still works after a null-ARN invocation"); + assertTrue(exporter.flushes > 0); + } + @Test void exporterThrowingErrorIsIsolatedAndLaterExportersStillReceiveAndFlush() { var throwing = new InsightExporter() { @@ -92,10 +126,13 @@ public void export(WorkflowInsightRecord record) { } }; var good = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .addExporter(throwing) - .addExporter(good) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .addExporter(throwing) + .addExporter(good) + .build()), + ARN, + START); plugin.onInvocationStart(start("in")); plugin.onInvocationEnd(end("in")); @@ -107,8 +144,11 @@ public void export(WorkflowInsightRecord record) { @Test void inputSnapshotErrorOmitsInputButDoesNotDisruptExecution() { var exporter = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight( - WorkflowInsightConfig.builder().addExporter(exporter).build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight( + WorkflowInsightConfig.builder().addExporter(exporter).build()), + ARN, + START); // Snapshotting the input fails with an Error; the hook must not propagate it. plugin.onInvocationStart(start(new ExplodingPayload())); @@ -123,14 +163,17 @@ void inputSnapshotErrorOmitsInputButDoesNotDisruptExecution() { @Test void throwingInputTransformOmitsInputWithoutFailure() { var exporter = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .content(ContentConfig.builder() - .inputTransform(v -> { - throw new AssertionError("redactor blew up"); - }) - .build()) - .addExporter(exporter) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .content(ContentConfig.builder() + .inputTransform(v -> { + throw new AssertionError("redactor blew up"); + }) + .build()) + .addExporter(exporter) + .build()), + ARN, + START); plugin.onInvocationStart(start("in")); plugin.onInvocationEnd(end("in")); @@ -143,14 +186,17 @@ void throwingInputTransformOmitsInputWithoutFailure() { @Test void throwingResultTransformOmitsResultWithoutFailure() { var exporter = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .content(ContentConfig.builder() - .addOverride(OperationOverride.withResult("compute", r -> { - throw new AssertionError("result redactor blew up"); - })) - .build()) - .addExporter(exporter) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .content(ContentConfig.builder() + .addOverride(OperationOverride.withResult("compute", r -> { + throw new AssertionError("result redactor blew up"); + })) + .build()) + .addExporter(exporter) + .build()), + ARN, + START); plugin.onInvocationStart(start("in")); plugin.onInvocationEnd(end("in")); @@ -174,11 +220,14 @@ public void export(WorkflowInsightRecord record) { } }; var third = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .addExporter(first) - .addExporter(throwing) - .addExporter(third) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .addExporter(first) + .addExporter(throwing) + .addExporter(third) + .build()), + ARN, + START); plugin.onInvocationStart(start("in")); plugin.onInvocationEnd(end("in")); diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/RecordSupersessionTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/RecordSupersessionTest.java new file mode 100644 index 000000000..faf7dbaf4 --- /dev/null +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/RecordSupersessionTest.java @@ -0,0 +1,282 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.plugin.OperationChangeInfo; +import software.amazon.lambda.durable.plugin.OperationChangeItemInfo; + +/** + * Build order, not hand-off order, decides which record an invocation exports. + * + *

Customer code runs while a record is being built, on the hook thread, before anything is scheduled: the input and + * output content transforms, an operation's result transform, and any Jackson serializer registered for a customer + * type. That code can call back into a hook of the same plugin instance, which builds and hands over a newer record + * while the outer build is still running. The outer build then hands over an older snapshot last, and the scheduler's + * per-invocation slot takes the last hand-off with no comparison of record ages. + * + *

Two outcomes follow if nothing orders the two records. With the pump held, the newer record is coalesced away and + * only the older snapshot is exported. With the pump running immediately, the exporter sees the newer record and then + * the older one, so the last state a destination records for the execution is stale. + * + *

The plugin takes a build revision before each build and the scheduler queues the record only while that revision + * is still the newest, so an overtaken build's record is dropped. The final record is exempt from that check and is + * ordered by the invocation's {@code closed} flag instead, so a newer RUNNING build started from inside the final + * record's own transforms cannot drop it. + * + *

The SDK serializes change hooks for one execution today, so the re-entrant hook here is forced rather than + * observed in production. The plugin must not depend on that: nothing in the SDK pins it, and the three language SDKs + * carry the same guard. + */ +class RecordSupersessionTest { + + private static final String ARN = + "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-1/invocation-1"; + private static final Instant START = Instant.parse("2026-08-05T00:00:00Z"); + private static final String INPUT = "payload"; + + /** Records every record handed to an exporter, in the order the exporter saw them. */ + private static final class RecordingExporter implements InsightExporter { + final List exported = Collections.synchronizedList(new ArrayList<>()); + + @Override + public void export(WorkflowInsightRecord record) { + exported.add(record); + } + } + + /** Holds every pump the scheduler starts until the test runs it, so the coalescing window is under test control. */ + private static final class HeldExecutor implements Executor { + private final Queue pending = new ConcurrentLinkedQueue<>(); + + @Override + public void execute(Runnable command) { + pending.add(command); + } + + void runPending() { + Runnable task; + while ((task = pending.poll()) != null) { + task.run(); + } + } + } + + /** + * One invocation's plugin, wired to a scheduler whose pump the test controls, with an input transform that can be + * armed to re-enter a hook of that same plugin. Re-entry through a content transform is the reachable path: + * {@code buildRecord} calls the transform before it returns the record to be scheduled. + */ + private static final class Fixture { + final RecordingExporter exporter = new RecordingExporter(); + final List failures = Collections.synchronizedList(new ArrayList<>()); + final AtomicReference armed = new AtomicReference<>(); + final ExportScheduler scheduler; + final InsightPlugin plugin; + + Fixture(Executor executor) { + var config = WorkflowInsightConfig.builder() + .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) + .content(ContentConfig.builder() + .inputTransform(value -> { + Runnable reentry = armed.getAndSet(null); + if (reentry != null) { + reentry.run(); + } + return value; + }) + .build()) + .addExporter(exporter) + .build(); + scheduler = new ExportScheduler( + List.of(exporter), (record, target) -> target.export(record), failures::add, executor); + plugin = new InsightPlugin(new InsightSettings(config), scheduler, Executions.info(ARN)); + } + + /** Arms the next build's transform to run this once, before the build that triggered it finishes. */ + void arm(Runnable reentry) { + armed.set(reentry); + } + + boolean isDraining() { + synchronized (scheduler) { + return plugin.drainWaiters > 0; + } + } + } + + // --- The two outcomes an unordered hand-off produces. --- + + @Test + void anOvertakenBuildDoesNotOverwriteTheNewerRecordInTheSlot() { + var executor = new HeldExecutor(); + var fixture = new Fixture(executor); + + // The start record claims the pump, which this executor holds, so every record below coalesces into the one + // slot this invocation has. + fixture.plugin.onInvocationStart(startInfo(operations(1))); + // The change hook this arms builds a two-operation record and hands it over while the outer build below is + // still inside its transform. The outer build then hands over its one-operation record last. + fixture.arm(() -> fixture.plugin.onOperationChange(changeInfo(operations(2)))); + fixture.plugin.onOperationChange(changeInfo(operations(1))); + + executor.runPending(); + + assertEquals( + List.of(2), + operationCounts(fixture), + "the slot must hold the newest build's record; the overtaken build's older snapshot is dropped"); + assertEquals(List.of(), fixture.failures, "no scheduler failure was reported"); + } + + @Test + void anOvertakenBuildIsNotExportedAfterTheRecordThatOvertookIt() { + // Runnable::run makes the pump drain inside schedule(), so each record reaches the exporter before the next + // hand-off. Nothing is coalesced, and a superseded record shows up as a stale export rather than a lost one. + var fixture = new Fixture(Runnable::run); + + fixture.plugin.onInvocationStart(startInfo(operations(1))); + fixture.arm(() -> fixture.plugin.onOperationChange(changeInfo(operations(2)))); + fixture.plugin.onOperationChange(changeInfo(operations(1))); + + assertEquals( + List.of(1, 2), + operationCounts(fixture), + "the overtaken build's record must not be exported after the record that overtook it"); + assertEquals(List.of(), fixture.failures, "no scheduler failure was reported"); + } + + // --- The failure mode a revision check can introduce. --- + + @Test + void theFinalRecordSurvivesANewerBuildStartedInsideIt() { + var fixture = new Fixture(Runnable::run); + + fixture.plugin.onInvocationStart(startInfo(operations(1))); + // Re-entered from the final record's own build, so the final record's revision is no longer the newest by the + // time it is handed over. Dropping it would leave a RUNNING snapshot as this execution's last exported state. + fixture.arm(() -> fixture.plugin.onOperationChange(changeInfo(operations(2)))); + fixture.plugin.onInvocationEnd(endInfo(operations(2))); + + var statuses = statuses(fixture); + assertTrue(statuses.contains("SUCCEEDED"), "the final record must be exported; exported: " + statuses); + assertEquals( + "SUCCEEDED", + statuses.get(statuses.size() - 1), + "no RUNNING record may be exported after the final one; exported: " + statuses); + assertEquals(List.of(), fixture.failures, "no scheduler failure was reported"); + } + + @Test + void theFinalRecordIsTheOnlyRecordExportedWhenThePumpRunsAfterTheInvocationEnds() throws Exception { + var executor = new HeldExecutor(); + var fixture = new Fixture(executor); + + // Runs the held pump only once the invocation end is inside its drain. Every record that end built is in the + // slot by then, so which record is exported is decided by the slot rather than by when this thread wakes up. + var invocationEnded = new AtomicBoolean(); + var pumper = new Thread(() -> { + while (!fixture.isDraining()) { + Thread.onSpinWait(); + } + // Kept pumping until the hook returns: the flush the end requests after its drain needs a pump too. + while (!invocationEnded.get()) { + executor.runPending(); + Thread.onSpinWait(); + } + executor.runPending(); + }); + pumper.setDaemon(true); + pumper.start(); + + fixture.plugin.onInvocationStart(startInfo(operations(1))); + fixture.arm(() -> fixture.plugin.onOperationChange(changeInfo(operations(2)))); + fixture.plugin.onInvocationEnd(endInfo(operations(2))); + invocationEnded.set(true); + pumper.join(30_000); + assertFalse(pumper.isAlive(), "the invocation end never completed its drain and flush"); + + assertEquals( + List.of("SUCCEEDED"), + statuses(fixture), + "the final record supersedes both RUNNING records in the slot and is the one exported"); + assertEquals(List.of(), fixture.failures, "no scheduler failure was reported"); + } + + // --- Fixture helpers. --- + + private static List operationCounts(Fixture fixture) { + List counts = new ArrayList<>(); + synchronized (fixture.exporter.exported) { + for (WorkflowInsightRecord record : fixture.exporter.exported) { + counts.add(record.operations().size()); + } + } + return counts; + } + + private static List statuses(Fixture fixture) { + List statuses = new ArrayList<>(); + synchronized (fixture.exporter.exported) { + for (WorkflowInsightRecord record : fixture.exporter.exported) { + statuses.add(record.status()); + } + } + return statuses; + } + + private static InvocationInfo startInfo(Map operations) { + return new InvocationInfo("req", ARN, true, START, INPUT, operations, Map.of()); + } + + private static OperationChangeInfo changeInfo(Map operations) { + return new OperationChangeInfo("req", ARN, operations, operations); + } + + private static InvocationEndInfo endInfo(Map operations) { + return new InvocationEndInfo( + "req", ARN, true, START, operations, InvocationStatus.SUCCEEDED, null, INPUT, "result"); + } + + /** A snapshot of {@code count} completed steps; the count is how a test tells two records apart. */ + private static Map operations(int count) { + Map snapshot = new LinkedHashMap<>(); + for (int i = 1; i <= count; i++) { + snapshot.put( + "op-" + i, + new OperationChangeItemInfo( + "op-" + i, + "step-" + i, + "STEP", + "Step", + null, + START.plusMillis(i), + START.plusMillis(i + 1), + OperationStatus.SUCCEEDED, + 1, + false, + null, + null)); + } + return snapshot; + } +} diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/StateCleanupLifecycleTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/StateCleanupLifecycleTest.java index da2768602..e52504867 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/StateCleanupLifecycleTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/StateCleanupLifecycleTest.java @@ -3,30 +3,55 @@ package software.amazon.lambda.durable.insight; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import java.lang.ref.WeakReference; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.InvocationStatus; import software.amazon.lambda.durable.plugin.OperationChangeItemInfo; /** - * Finding {@code arf_v1_qh6xoafzze3z3ccgrbppucmunr} ([P2] remove retained suspended execution state): per-execution - * state must be removed on every {@code onInvocationEnd}, including non-terminal PENDING/RETRYING suspends, so a warm - * container never leaks one entry per suspended execution. A resume re-seeds identical stable start time and input. + * Finding {@code arf_v1_qh6xoafzze3z3ccgrbppucmunr} ([P2] remove retained suspended execution state): a warm container + * must never accumulate per-execution state, including for executions that suspend (PENDING/RETRYING) and never + * terminate in that container. A resume re-seeds identical stable start time and input. + * + *

The plugin used to keep that state in an ARN-keyed map and remove the entry at every invocation end, so the test + * counted the entries left behind. There is no map now — an invocation's state is its plugin instance, which + * the SDK creates per invocation and drops when it returns — so what is left to prove is about the one object that does + * outlive invocations: the factory's {@link ExportScheduler}. Two things are asserted, both read directly out of that + * scheduler under the monitor its fields are guarded by. First, that it owes a finished invocation nothing: no queued + * record, nothing inside the exporters, no uncompleted drain signal, no drain waiting. Second, that it holds no + * reference to the instance: {@link ExportScheduler#queue} is the only collection of per-invocation objects it has, so + * an empty queue after every invocation has ended is "the environment retains nothing", and a retained entry + * of any kind would fail it — which the old count could not do, because it could only count the entries the plugin knew + * it had. + * + *

Reachability from the scheduler is what determines whether state accumulates, and that is a fact about the + * scheduler's own fields, not about the collector. Whether the JVM has actually reclaimed a finished instance is + * reported below as a diagnostic and never asserted: {@link System#gc()} is a request the JVM is free to ignore, so an + * implementation that retains nothing can still leave every weak reference set, and asserting on it would fail the + * build on garbage-collector behaviour rather than on this plugin's. */ class StateCleanupLifecycleTest { private static final Instant START = Instant.parse("2026-08-05T00:00:00Z"); private static final class CapturingExporter implements InsightExporter { - final List records = new ArrayList<>(); + /** Written on pump threads, read on the test thread after a drain; synchronized so the reads are sound. */ + final List records = Collections.synchronizedList(new ArrayList<>()); @Override public void export(WorkflowInsightRecord record) { @@ -34,6 +59,18 @@ public void export(WorkflowInsightRecord record) { } } + /** + * An execution environment that emits on every change, so each invocation below really does put records through the + * scheduler. With the default {@code ON_COMPLETE} mode a suspending invocation emits nothing, and "the environment + * retains nothing" would hold trivially because nothing was ever queued. + */ + private static DurableExecutionPluginFactory emittingEnvironment(CapturingExporter exporter) { + return WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) + .addExporter(exporter) + .build()); + } + private static String arn(int i) { return "arn:aws:lambda:us-west-2:1:function:f:$LATEST/durable-execution/exec-" + i + "/invocation-1"; } @@ -58,57 +95,192 @@ private static Map ops() { return m; } + private static InvocationInfo start(int i) { + return new InvocationInfo("req", arn(i), true, START, "in-" + i, ops(), Map.of()); + } + + private static InvocationEndInfo end(int i, InvocationStatus status) { + return new InvocationEndInfo("req", arn(i), true, START, ops(), status, null, "in-" + i, null); + } + + /** What a finished invocation leaves behind: the environment that served it, and a way to observe reclamation. */ + private record Finished(ExportScheduler environment, WeakReference instance) {} + + /** + * Runs one whole invocation in the given environment and returns the environment's scheduler plus a weak reference + * to the instance that served it, keeping no strong reference of its own — so whatever that reference still points + * at afterwards is retained by the environment, not by this test. + * + *

Both assertions are made here, while the instance is still in hand: the scheduler's per-invocation fields for + * this instance are all clear, and the scheduler's queue does not contain it. Those are the two halves of one + * documented invariant — an invocation is in the queue exactly while its record is non-null — so checking both + * catches a state that satisfies one and not the other. + */ + private static Finished runInvocation(DurableExecutionPluginFactory environment, int i, InvocationStatus status) { + InsightPlugin plugin = Executions.started(environment, start(i)); + plugin.onInvocationEnd(end(i, status)); + ExportScheduler scheduler = plugin.scheduler; + assertFalse(Executions.outstanding(plugin), "the scheduler still owes execution " + i + " work"); + assertFalse(scheduler.retains(plugin), "the environment still holds a reference to execution " + i); + return new Finished(scheduler, new WeakReference<>(plugin)); + } + + /** + * Diagnostic only, never an assertion: how many finished instances the JVM has reclaimed after being asked to. Kept + * because it is the observation that first exposed the retained-state finding, and printed so a regression is + * visible in the build log without a nondeterministic failure. + */ + private static void reportReclamation(String scenario, List finished) { + System.gc(); + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + long reclaimed = + finished.stream().filter(f -> f.instance().get() == null).count(); + System.out.printf( + "DIAGNOSTIC %s: %d of %d finished plugin instances reclaimed after a System.gc() request%n", + scenario, reclaimed, finished.size()); + } + + /** + * Every plugin instance the scheduler still reaches through any of its fields, described as {@code field -> + * plugin}. + * + *

The seams above answer the same question for the one collection the scheduler is known to keep. This finds the + * collection it is not known to keep: a registry reintroduced under any name, keyed by execution ARN or + * otherwise, shows up here as soon as it holds an instance. That is what makes reachability, rather than + * collection, the thing this test asserts — and it is deterministic, unlike asking the collector. + * + *

Read under the scheduler's monitor, which is the monitor its per-invocation fields are guarded by. + */ + private static List pluginsReachableFrom(ExportScheduler scheduler) { + var reachable = new ArrayList(); + synchronized (scheduler) { + for (Field field : ExportScheduler.class.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers())) { + continue; + } + field.setAccessible(true); + Object value; + try { + value = field.get(scheduler); + } catch (ReflectiveOperationException e) { + throw new AssertionError("could not read ExportScheduler." + field.getName(), e); + } + for (Object element : elementsOf(value)) { + if (element instanceof InsightPlugin plugin) { + reachable.add(field.getName() + " -> " + plugin); + } + } + } + } + return reachable; + } + + /** The elements a field value exposes, so a collection or map of any shape can be inspected uniformly. */ + private static Collection elementsOf(Object value) { + if (value instanceof Collection collection) { + return new ArrayList(collection); + } + if (value instanceof Map map) { + var elements = new ArrayList(map.keySet()); + elements.addAll(map.values()); + return elements; + } + return List.of(); + } + @Test void nDistinctPendingExecutionsLeaveNoRetainedState() { - var plugin = (WorkflowInsight.InsightPlugin) - WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder().build()); + var exporter = new CapturingExporter(); + var environment = emittingEnvironment(exporter); int n = 25; + var finished = new ArrayList(); for (int i = 0; i < n; i++) { - String arn = arn(i); - plugin.onInvocationStart(new InvocationInfo("req", arn, true, START, "in-" + i, ops(), Map.of())); // Each execution suspends (PENDING) and never terminates in this container. - plugin.onInvocationEnd(new InvocationEndInfo( - "req", arn, true, START, ops(), InvocationStatus.PENDING, null, "in-" + i, null)); + finished.add(runInvocation(environment, i, InvocationStatus.PENDING)); } - assertEquals(0, plugin.retainedStateCount(), "no per-execution state retained for suspended executions"); + ExportScheduler scheduler = finished.get(0).environment(); + // Quiesce: returns once nothing is queued and no pump owns the scheduler, so the count below is read at a point + // where a still-running pump cannot be mistaken for retained state. + scheduler.drainAll(); + + // Every invocation really did put records through the scheduler, so the assertions below are about state that + // existed and was released, not state that was never created. Counted by distinct execution rather than by + // record: a RUNNING snapshot that the end record supersedes before any pump takes it is coalesced away by + // design, so the number of records is not fixed, but every invocation drains its own final record. + assertEquals( + n, + exporter.records.stream() + .map(WorkflowInsightRecord::executionArn) + .distinct() + .count(), + "every invocation delivered at least one record through the environment's scheduler"); + assertEquals( + 0, + scheduler.retainedInvocationCount(), + "the environment still holds per-invocation state after all " + n + " invocations ended"); + assertEquals( + List.of(), + pluginsReachableFrom(scheduler), + "the environment still reaches plugin instances after all " + n + " invocations ended"); + reportReclamation(n + " pending executions", finished); } @Test void retryingSuspendAlsoLeavesNoRetainedState() { - var plugin = (WorkflowInsight.InsightPlugin) - WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder().build()); - String arn = arn(0); - plugin.onInvocationStart(new InvocationInfo("req", arn, true, START, "in", ops(), Map.of())); - plugin.onInvocationEnd( - new InvocationEndInfo("req", arn, true, START, ops(), InvocationStatus.RETRYING, null, "in", null)); - assertEquals(0, plugin.retainedStateCount(), "RETRYING suspend also clears state"); + var exporter = new CapturingExporter(); + var environment = emittingEnvironment(exporter); + + var finished = runInvocation(environment, 0, InvocationStatus.RETRYING); + + finished.environment().drainAll(); + assertFalse(exporter.records.isEmpty(), "the invocation put at least one record through the scheduler"); + assertEquals( + 0, + finished.environment().retainedInvocationCount(), + "a RETRYING suspend leaves the environment holding per-invocation state"); + assertEquals( + List.of(), + pluginsReachableFrom(finished.environment()), + "a RETRYING suspend leaves the environment reaching its plugin instance"); + reportReclamation("one retrying execution", List.of(finished)); } @Test void resumeReSeedsStableStartTimeAndInput() { var exporter = new CapturingExporter(); - var plugin = (WorkflowInsight.InsightPlugin) WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) - .addExporter(exporter) - .build()); - String arn = arn(0); + var environment = emittingEnvironment(exporter); - // First invocation with input "alpha", then suspend (state removed). - plugin.onInvocationStart(new InvocationInfo("req", arn, true, START, "alpha", ops(), Map.of())); - plugin.onInvocationEnd( - new InvocationEndInfo("req", arn, true, START, ops(), InvocationStatus.PENDING, null, "alpha", null)); + // First invocation with input "alpha", then suspend. Its instance is dropped with it. + var first = Executions.started( + environment, new InvocationInfo("req", arn(0), true, START, "alpha", ops(), Map.of())); + first.onInvocationEnd(new InvocationEndInfo( + "req", arn(0), true, START, ops(), InvocationStatus.PENDING, null, "alpha", null)); - // Resume invocation: onInvocationStart re-seeds state from hook data (same START, same input). - plugin.onInvocationStart(new InvocationInfo("req", arn, false, START, "alpha", ops(), Map.of())); - plugin.onInvocationEnd(new InvocationEndInfo( - "req", arn, true, START, ops(), InvocationStatus.SUCCEEDED, null, "alpha", "out")); + // Resume invocation: a new instance, seeded from the resume's own hook data (same START, same input). + var resumed = Executions.started( + environment, new InvocationInfo("req", arn(0), false, START, "alpha", ops(), Map.of())); + resumed.onInvocationEnd(new InvocationEndInfo( + "req", arn(0), true, START, ops(), InvocationStatus.SUCCEEDED, null, "alpha", "out")); var terminal = exporter.records.get(exporter.records.size() - 1); assertEquals("SUCCEEDED", terminal.status()); assertEquals(START.toString(), terminal.startTime(), "stable start time recreated across the suspend boundary"); assertEquals("alpha", terminal.input, "input re-seeded from resume onInvocationStart"); - assertEquals(0, plugin.retainedStateCount(), "terminal end also clears state"); + assertFalse(Executions.outstanding(resumed), "the terminal end leaves the scheduler owing nothing"); + assertFalse(resumed.scheduler.retains(resumed), "the environment holds no reference to the resumed invocation"); + assertEquals( + 0, + resumed.scheduler.retainedInvocationCount(), + "neither the suspended invocation nor the resumed one is retained by the environment"); + assertEquals( + List.of(), + pluginsReachableFrom(resumed.scheduler), + "the environment reaches neither the suspended invocation nor the resumed one"); } } diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/TransformContractTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/TransformContractTest.java index ee8219bc7..b4c426964 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/TransformContractTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/TransformContractTest.java @@ -17,7 +17,6 @@ import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.model.ExecutionStatus; -import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.InvocationStatus; @@ -81,10 +80,15 @@ private Map ops() { private WorkflowInsightRecord runOnce(Object input, Function inputTransform) { var exporter = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .content(ContentConfig.builder().inputTransform(inputTransform).build()) - .addExporter(exporter) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .content(ContentConfig.builder() + .inputTransform(inputTransform) + .build()) + .addExporter(exporter) + .build()), + ARN, + START); plugin.onInvocationStart(new InvocationInfo("req", ARN, true, START, input, ops(), Map.of())); plugin.onInvocationEnd( new InvocationEndInfo("req", ARN, true, START, ops(), InvocationStatus.SUCCEEDED, null, input, "out")); @@ -126,11 +130,15 @@ void eachTransformInvocationReceivesAFreshDetachedCopy() { m.put("injected-" + m.size(), Boolean.TRUE); // mutate the argument in place return m; }; - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) - .content(ContentConfig.builder().inputTransform(mutating).build()) - .addExporter(exporter) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) + .content( + ContentConfig.builder().inputTransform(mutating).build()) + .addExporter(exporter) + .build()), + ARN, + START); Map input = new LinkedHashMap<>(); input.put("a", 1); @@ -150,7 +158,7 @@ void eachTransformInvocationReceivesAFreshDetachedCopy() { @Test void throwingTransformOmitsInputWithoutFailingExecution() { var exporter = new CapturingExporter(); - var plugin = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + var factory = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() .content(ContentConfig.builder() .inputTransform(v -> { throw new AssertionError("redactor blew up"); @@ -161,7 +169,7 @@ void throwingTransformOmitsInputWithoutFailingExecution() { var runner = LocalDurableTestRunner.create( String.class, (input, context) -> context.step("greet", String.class, sc -> "hi"), - DurableConfig.builder().withPlugins(plugin).build()); + DurableConfig.builder().withPlugins(factory).build()); var result = runner.runUntilComplete("World"); diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/UnrecoverableErrorUnwrapTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/UnrecoverableErrorUnwrapTest.java index ccd32782f..972862170 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/UnrecoverableErrorUnwrapTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/UnrecoverableErrorUnwrapTest.java @@ -14,7 +14,6 @@ import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; -import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationStatus; import software.amazon.lambda.durable.plugin.OperationChangeItemInfo; @@ -71,8 +70,11 @@ private static Map ops(OperationStatus status) @Test void failedExecutionUnwrapsUnrecoverableErrorObject() { var exporter = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight( - WorkflowInsightConfig.builder().addExporter(exporter).build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight( + WorkflowInsightConfig.builder().addExporter(exporter).build()), + ARN, + START); Throwable execError = unrecoverable("PoisonPayload", "cannot deserialize checkpoint"); plugin.onInvocationEnd(new InvocationEndInfo( @@ -88,10 +90,13 @@ void failedExecutionUnwrapsUnrecoverableErrorObject() { @Test void retryingExecutionUnwrapsUnrecoverableErrorObjectInOnChangeMode() { var exporter = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) - .addExporter(exporter) - .build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) + .addExporter(exporter) + .build()), + ARN, + START); Throwable execError = unrecoverable("TransientBackendError", "retry scheduled"); // RETRYING maps to a non-terminal RUNNING status but still emits in ON_CHANGE mode. @@ -117,8 +122,11 @@ void retryingExecutionUnwrapsUnrecoverableErrorObjectInOnChangeMode() { @Test void fallsBackToThrowableFieldsWhenUnrecoverableErrorTypeMissing() { var exporter = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight( - WorkflowInsightConfig.builder().addExporter(exporter).build()); + var plugin = Executions.plugin( + WorkflowInsight.workflowInsight( + WorkflowInsightConfig.builder().addExporter(exporter).build()), + ARN, + START); ErrorObject partial = ErrorObject.builder().errorMessage("only a message").build(); diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/WorkflowInsightFlushCadenceTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/WorkflowInsightFlushCadenceTest.java new file mode 100644 index 000000000..43f5b7e57 --- /dev/null +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/WorkflowInsightFlushCadenceTest.java @@ -0,0 +1,230 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.insight; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.plugin.OperationChangeInfo; +import software.amazon.lambda.durable.plugin.OperationChangeItemInfo; + +/** + * Pins the flush cadence at the plugin boundary: every invocation end that reaches the exporters flushes them — at most + * once, and exactly once when ends do not overlap — including the ends that emit no record, while a sampled-out + * execution flushes not at all. Ends that overlap may share one flush; no end's record is ever left unflushed. + */ +class WorkflowInsightFlushCadenceTest { + + private static final Instant START = Instant.parse("2026-08-05T00:00:00Z"); + + private static String arn(int index) { + return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1"; + } + + /** Counts exports and flushes, and remembers how many exports had happened when each flush ran. */ + private static final class CountingExporter implements InsightExporter { + final List exported = new CopyOnWriteArrayList<>(); + final AtomicInteger flushes = new AtomicInteger(); + final List exportsAtFlush = new CopyOnWriteArrayList<>(); + + @Override + public void export(WorkflowInsightRecord record) { + exported.add(record.status() + "@" + record.executionArn()); + } + + @Override + public void flush() { + flushes.incrementAndGet(); + exportsAtFlush.add(exported.size()); + } + } + + /** The environment: one factory, one scheduler, one set of exporters, however many invocations follow. */ + private static DurableExecutionPluginFactory environment( + WorkflowInsightConfig.EmitMode mode, Double samplingRate, CountingExporter... exporters) { + var builder = WorkflowInsightConfig.builder().emitMode(mode); + for (CountingExporter exporter : exporters) { + builder = builder.addExporter(exporter); + } + if (samplingRate != null) { + builder = builder.samplingRate(samplingRate); + } + return WorkflowInsight.workflowInsight(builder.build()); + } + + @Test + void anInvocationEndThatEmitsARecordFlushesEveryExporterExactlyOnce() { + var first = new CountingExporter(); + var second = new CountingExporter(); + var environment = environment(WorkflowInsightConfig.EmitMode.ON_COMPLETE, null, first, second); + + var plugin = Executions.started(environment, start(arn(0))); + plugin.onInvocationEnd(end(arn(0), InvocationStatus.SUCCEEDED)); + + for (CountingExporter exporter : List.of(first, second)) { + assertEquals(List.of("SUCCEEDED@" + arn(0)), exporter.exported); + assertEquals(1, exporter.flushes.get(), "exactly one flush per invocation end"); + assertEquals(List.of(1), exporter.exportsAtFlush, "the flush follows the record it is meant to flush"); + } + } + + @Test + void anInvocationEndThatEmitsNothingStillFlushesEveryExporterExactlyOnce() { + // ON_COMPLETE + a non-terminal suspend, and ON_FAILURE + a success: both are sampled in, both emit no record, + // and both must still flush — a buffering exporter's earlier records depend on it. + record Case(String name, WorkflowInsightConfig.EmitMode mode, InvocationStatus status) {} + List cases = List.of( + new Case("ON_COMPLETE + PENDING", WorkflowInsightConfig.EmitMode.ON_COMPLETE, InvocationStatus.PENDING), + new Case( + "ON_COMPLETE + RETRYING", + WorkflowInsightConfig.EmitMode.ON_COMPLETE, + InvocationStatus.RETRYING), + new Case( + "ON_FAILURE + SUCCEEDED", + WorkflowInsightConfig.EmitMode.ON_FAILURE, + InvocationStatus.SUCCEEDED)); + + for (Case scenario : cases) { + var exporter = new CountingExporter(); + var plugin = Executions.started(environment(scenario.mode(), null, exporter), start(arn(1))); + plugin.onInvocationEnd(end(arn(1), scenario.status())); + + assertEquals(List.of(), exporter.exported, scenario.name() + ": no record should be emitted"); + assertEquals(1, exporter.flushes.get(), scenario.name() + ": the flush must happen anyway"); + } + } + + @Test + void everyInvocationEndOfAWarmEnvironmentFlushesExactlyOnce() { + var exporter = new CountingExporter(); + var environment = environment(WorkflowInsightConfig.EmitMode.ON_CHANGE, null, exporter); + + int invocations = 5; + for (int i = 0; i < invocations; i++) { + // A warm environment: each invocation is served by its own instance from the same factory. + var plugin = Executions.started(environment, start(arn(i))); + plugin.onOperationChange(change(arn(i))); + plugin.onInvocationEnd(end(arn(i), InvocationStatus.SUCCEEDED)); + // Sequential ends have nothing to share a flush with, so the cadence bound is tight here. + assertEquals(i + 1, exporter.flushes.get(), "one flush per invocation end, never skipped"); + } + assertEquals(invocations, exporter.flushes.get()); + assertTrue(exporter.exported.size() >= invocations, "each execution's terminal record was exported"); + } + + @Test + void invocationEndsThatOverlapMayShareAFlushButNoneIsLeftUnflushed() { + var exporter = new CountingExporter(); + var environment = environment(WorkflowInsightConfig.EmitMode.ON_COMPLETE, null, exporter); + + int executions = 8; + var barrier = new CyclicBarrier(executions); + var done = new CountDownLatch(executions); + for (int i = 0; i < executions; i++) { + String executionArn = arn(100 + i); + var plugin = Executions.started(environment, start(executionArn)); + var thread = new Thread( + () -> { + try { + barrier.await(60, TimeUnit.SECONDS); + } catch (Exception e) { + throw new AssertionError(e); + } + plugin.onInvocationEnd(end(executionArn, InvocationStatus.SUCCEEDED)); + done.countDown(); + }, + "overlapping-end-" + i); + thread.setDaemon(true); + thread.start(); + } + try { + assertTrue(done.await(60, TimeUnit.SECONDS), "an invocation end never returned"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + + assertEquals(executions, exporter.exported.size(), "every execution's terminal record must be exported"); + assertTrue(exporter.flushes.get() >= 1, "the ends must be covered by at least one flush"); + assertTrue( + exporter.flushes.get() <= executions, + "at most one flush per invocation end: " + exporter.flushes.get() + " for " + executions); + // Every record must be followed by a flush: each end's own request is served by a flush that starts after its + // record was exported, so the last flush cannot precede the last export. + assertEquals( + executions, + exporter.exportsAtFlush.get(exporter.exportsAtFlush.size() - 1), + "the last flush ran after every terminal record: " + exporter.exportsAtFlush); + } + + @Test + void aSampledOutExecutionFlushesNothing() { + // Unchanged by the move of flush onto the export pump: a sampled-out end never schedules a record, so it + // neither drains nor flushes. + var exporter = new CountingExporter(); + var environment = environment(WorkflowInsightConfig.EmitMode.ON_CHANGE, 0.0, exporter); + + var plugins = new ArrayList(); + for (int i = 0; i < 10; i++) { + var plugin = Executions.started(environment, start(arn(i))); + plugins.add(plugin); + plugin.onOperationChange(change(arn(i))); + plugin.onInvocationEnd(end(arn(i), InvocationStatus.SUCCEEDED)); + } + + assertEquals(List.of(), exporter.exported); + assertEquals(0, exporter.flushes.get(), "a sampled-out invocation end neither drains nor flushes"); + for (InsightPlugin plugin : plugins) { + assertFalse(Executions.outstanding(plugin), "a sampled-out invocation leaves the scheduler owing nothing"); + } + } + + private static Map ops() { + Map operations = new LinkedHashMap<>(); + operations.put( + "op-1", + new OperationChangeItemInfo( + "op-1", + "greet", + "STEP", + "Step", + null, + START, + START.plusMillis(5), + OperationStatus.SUCCEEDED, + 1, + false, + null, + null)); + return operations; + } + + private static InvocationInfo start(String executionArn) { + return new InvocationInfo("req", executionArn, true, START, "in", ops(), Map.of()); + } + + private static OperationChangeInfo change(String executionArn) { + return new OperationChangeInfo("req", executionArn, ops(), ops()); + } + + private static InvocationEndInfo end(String executionArn, InvocationStatus status) { + return new InvocationEndInfo("req", executionArn, true, START, ops(), status, null, "in", "out"); + } +} diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/WorkflowInsightHookTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/WorkflowInsightHookTest.java index acd3a98b9..fc3e3e3da 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/WorkflowInsightHookTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/WorkflowInsightHookTest.java @@ -15,7 +15,7 @@ import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.lambda.model.OperationStatus; -import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.InvocationStatus; @@ -97,26 +97,22 @@ public void export(WorkflowInsightRecord record) { } } }; - var plugin = (WorkflowInsight.InsightPlugin) WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) - .addExporter(exporter) - .build()); - - plugin.onInvocationStart(start(arnA, true)); + var environment = onChangeEnvironment(exporter); + var pluginA = Executions.started(environment, start(arnA, true)); assertTrue(exportingA.await(5, TimeUnit.SECONDS)); - plugin.onInvocationStart(start(arnB, true)); + var pluginB = Executions.started(environment, start(arnB, true)); // a ends: its SUCCEEDED record is pending behind the blocked export, and a's end hook blocks in the drain. - var endingA = new Thread(() -> plugin.onInvocationEnd(end(arnA, InvocationStatus.SUCCEEDED, "out", null))); + var endingA = new Thread(() -> pluginA.onInvocationEnd(end(arnA, InvocationStatus.SUCCEEDED, "out", null))); endingA.start(); Thread.sleep(100); // b's change hook arrives while a's final record is still pending; it must not displace it. - plugin.onOperationChange(new OperationChangeInfo( + pluginB.onOperationChange(new OperationChangeInfo( "req", arnB, ops("greet", OperationStatus.SUCCEEDED), ops("greet", OperationStatus.SUCCEEDED))); release.countDown(); endingA.join(5_000); assertFalse(endingA.isAlive()); - plugin.onInvocationEnd(end(arnB, InvocationStatus.SUCCEEDED, "out", null)); + pluginB.onInvocationEnd(end(arnB, InvocationStatus.SUCCEEDED, "out", null)); var aStatuses = exporter.records.stream() .filter(r -> arnA.equals(r.executionArn())) @@ -131,17 +127,21 @@ public void export(WorkflowInsightRecord record) { assertTrue(bStatuses.subList(0, bStatuses.size() - 1).stream().allMatch("RUNNING"::equals)); } - @Test - void onChangeEmitsAtStartChangeAndEnd() { - var exporter = new CapturingExporter(); - var plugin = (WorkflowInsight.InsightPlugin) WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + /** The environment one or more invocations are then served in: one factory, one scheduler, one exporter set. */ + private static DurableExecutionPluginFactory onChangeEnvironment(InsightExporter exporter) { + return WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) .addExporter(exporter) .build()); + } + + @Test + void onChangeEmitsAtStartChangeAndEnd() { + var exporter = new CapturingExporter(); + var plugin = Executions.started(onChangeEnvironment(exporter), start(true)); // Let each scheduled export land before the next hook so all three snapshots are observable; back-to-back // hooks may otherwise coalesce into the latest record (covered separately below). - plugin.onInvocationStart(start(true)); plugin.drainExports(); plugin.onOperationChange(new OperationChangeInfo( "req", ARN, ops("greet", OperationStatus.SUCCEEDED), ops("greet", OperationStatus.SUCCEEDED))); @@ -158,12 +158,8 @@ void onChangeEmitsAtStartChangeAndEnd() { @Test void onChangeExportsOffTheHookThreadAndCoalescesBurstsIntoTheLatestRecord() { var exporter = new CapturingExporter(); - var plugin = (WorkflowInsight.InsightPlugin) WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) - .addExporter(exporter) - .build()); + var plugin = Executions.started(onChangeEnvironment(exporter), start(true)); - plugin.onInvocationStart(start(true)); for (int i = 0; i < 20; i++) { plugin.onOperationChange(new OperationChangeInfo( "req", ARN, ops("greet", OperationStatus.SUCCEEDED), ops("greet", OperationStatus.SUCCEEDED))); @@ -200,12 +196,7 @@ public void export(WorkflowInsightRecord record) { } } }; - var plugin = (WorkflowInsight.InsightPlugin) WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) - .addExporter(exporter) - .build()); - - plugin.onInvocationStart(start(true)); + var plugin = Executions.started(onChangeEnvironment(exporter), start(true)); plugin.drainExports(); // The end hook blocks in its drain while the final record is being exported; the change hook arrives then, @@ -228,10 +219,10 @@ public void export(WorkflowInsightRecord record) { @Test void invocationEndFlushesExportersEvenWhenNothingWasEmitted() { var exporter = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight( - WorkflowInsightConfig.builder().addExporter(exporter).build()); - - plugin.onInvocationStart(start(true)); + var plugin = Executions.started( + WorkflowInsight.workflowInsight( + WorkflowInsightConfig.builder().addExporter(exporter).build()), + start(true)); plugin.onInvocationEnd(end(InvocationStatus.PENDING, null, null)); assertTrue(exporter.records.isEmpty(), "on-complete emits nothing for a suspend"); @@ -241,10 +232,10 @@ void invocationEndFlushesExportersEvenWhenNothingWasEmitted() { @Test void onCompleteSkipsNonTerminalAndEmitsTerminalOnly() { var exporter = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight( - WorkflowInsightConfig.builder().addExporter(exporter).build()); - - plugin.onInvocationStart(start(true)); + var plugin = Executions.started( + WorkflowInsight.workflowInsight( + WorkflowInsightConfig.builder().addExporter(exporter).build()), + start(true)); plugin.onOperationChange(new OperationChangeInfo( "req", ARN, ops("greet", OperationStatus.SUCCEEDED), ops("greet", OperationStatus.SUCCEEDED))); assertTrue(exporter.records.isEmpty(), "no record before terminal in on-complete mode"); @@ -256,17 +247,16 @@ void onCompleteSkipsNonTerminalAndEmitsTerminalOnly() { @Test void suspendResumeKeepsStableStartTimeAndLeavesNoRetainedState() { var exporter = new CapturingExporter(); - var plugin = (WorkflowInsight.InsightPlugin) WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE) - .addExporter(exporter) - .build()); + var environment = onChangeEnvironment(exporter); - plugin.onInvocationStart(start(true)); // first invocation - plugin.drainExports(); - plugin.onInvocationEnd(end(InvocationStatus.PENDING, null, null)); // suspend -> state removed - plugin.onInvocationStart(start(false)); // resume invocation re-seeds state - plugin.drainExports(); - plugin.onInvocationEnd(end(InvocationStatus.SUCCEEDED, "out", null)); // resume + terminal + // The suspend and the resume are two invocations of the same execution in one warm environment, so the SDK + // serves them with two instances: nothing is carried over in the plugin, and nothing has to be cleaned up. + var first = Executions.started(environment, start(true)); + first.drainExports(); + first.onInvocationEnd(end(InvocationStatus.PENDING, null, null)); // suspend + var resumed = Executions.started(environment, start(false)); // resume re-seeds from its own InvocationInfo + resumed.drainExports(); + resumed.onInvocationEnd(end(InvocationStatus.SUCCEEDED, "out", null)); // resume + terminal // start(RUNNING) + pending(RUNNING) + resume-start(RUNNING) + terminal(SUCCEEDED); all share the stable // startTime recreated from InvocationInfo.executionStartTime() across the suspend boundary. @@ -274,7 +264,8 @@ void suspendResumeKeepsStableStartTimeAndLeavesNoRetainedState() { String startTime = exporter.records.get(0).startTime(); assertTrue(exporter.records.stream().allMatch(r -> startTime.equals(r.startTime()))); assertEquals(START.toString(), startTime); - assertEquals(0, plugin.retainedStateCount(), "no per-execution state retained after invocation end"); + assertFalse(Executions.outstanding(first), "the suspended invocation left the scheduler owing nothing"); + assertFalse(Executions.outstanding(resumed), "nor did the resumed one"); } @Test @@ -286,12 +277,12 @@ public void export(WorkflowInsightRecord record) { } }; var good = new CapturingExporter(); - DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() - .addExporter(throwing) - .addExporter(good) - .build()); - - plugin.onInvocationStart(start(true)); + var plugin = Executions.started( + WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder() + .addExporter(throwing) + .addExporter(good) + .build()), + start(true)); plugin.onInvocationEnd(end(InvocationStatus.SUCCEEDED, "out", null)); assertEquals(1, good.records.size(), "failing exporter never blocks the others"); diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/WorkflowInsightPluginTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/WorkflowInsightPluginTest.java index 8d5cf0fff..98531b6e6 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/WorkflowInsightPluginTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/WorkflowInsightPluginTest.java @@ -44,8 +44,8 @@ public void export(WorkflowInsightRecord record) { 2, Duration.ofSeconds(1), Duration.ofSeconds(1), 2.0, JitterStrategy.NONE); private DurableConfig configWith(CapturingExporter exporter, WorkflowInsightConfig.Builder cfg) { - var plugin = WorkflowInsight.workflowInsight(cfg.addExporter(exporter).build()); - return DurableConfig.builder().withPlugins(plugin).build(); + var factory = WorkflowInsight.workflowInsight(cfg.addExporter(exporter).build()); + return DurableConfig.builder().withPlugins(factory).build(); } private OperationRecord op(WorkflowInsightRecord rec, String name) { diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/exporters/LazyClientTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/exporters/LazyClientTest.java index bf79cadb7..7ea4edec5 100644 --- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/exporters/LazyClientTest.java +++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/exporters/LazyClientTest.java @@ -43,4 +43,18 @@ void missingArtifactFailsWithAMessageNamingIt() { assertTrue(e.getMessage().contains("software.amazon.awssdk:redshiftdata"), e.getMessage()); assertInstanceOf(NoClassDefFoundError.class, e.getCause()); } + + public static final class FatalBuilder { + static final VirtualMachineError FAILURE = new VirtualMachineError("builder VM failure") {}; + + public static Object builder() { + throw FAILURE; + } + } + + @Test + void reflectiveBuilderFatalKeepsItsOriginalIdentity() { + var client = LazyClient.forSdkClient(null, "test", FatalBuilder.class.getName(), null); + assertSame(FatalBuilder.FAILURE, assertThrows(VirtualMachineError.class, client::get)); + } } diff --git a/otel-plugin/README.md b/otel-plugin/README.md index c51032aaa..eaf0ffd98 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -10,9 +10,41 @@ OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK fo - **Span-per-Operation**: Each durable operation (step, wait, map, etc.) gets its own span with accurate timing - **Attempt Spans**: Each user function execution (step attempt, child context run) gets a span, including retries - **Log Correlation**: Injects `traceId`, `spanId`, and `otelTraceSampled` into SLF4J MDC for end-to-end observability -- **ADOT Java Agent Integration**: `new InvocationOtelPlugin()` late-binds the ADOT Java agent's global provider with no handler-side OpenTelemetry initialization +- **ADOT Java Agent Integration**: `InvocationOtelPlugin.factory()` binds the ADOT Java agent's global provider on first use, with no handler-side OpenTelemetry initialization - **Lambda Layer Discovery**: `DURABLE_EXECUTION_PLUGINS` loads either OTel plugin from a JAR under a layer's `java/lib` directory +## One durable tracing view + +Choose either the invocation or execution factory. Their `durable-otel-view` exclusive group is validated across +explicit and environment-selected factories before any invocation instance or span is created. Unrelated factories +remain allowed. `DurableConfig.toBuilder()` and `LocalDurableTestRunner` preserve the resolved factory registrations +without rediscovering environment plugins; factory identity retains environment-owned provider resources. + +## Root handler context + +With a core that supports the explicit `@HandlerScoped` capability, user instrumentation in the root handler joins +its canonical execution trace. A valid same-trace ambient Lambda span stays current. If ambient context is absent or +belongs to another trace, invocation view activates `Invocation`; execution view activates the deterministic +`Workflow` context, including its unsampled non-recording form. Nested operations retain their existing contexts. + +The core opens the scope after invocation startup and closes it on the same handler thread on success, failure, or +suspension. On suspension or termination, an opened scope gets up to 500ms to unwind before finalization, capped by +remaining Lambda time after reserving five seconds per configured plugin and one second for shutdown/response. +This is a best-effort reserve, not a total deadline bound: existing plugin callbacks and checkpoint draining may exceed it. The invocation caller waits; the owning handler worker remains free to exit. +If cleanup is blocked, the SDK logs the timeout and preserves the original PENDING/RETRYING outcome. +An observed `VirtualMachineError` or `ThreadDeath` from the new scope callbacks instead escapes the invocation caller. +When observed before invocation finalization, the3.x lifecycle first calls invocation-end hooks once with RETRYING. +A fatal arriving during resource shutdown is checked after closure without repeating already completed end hooks; +ordinary cleanup failures and legacy body failures retain the original outcome. Fatals reported only after the response +cannot retroactively change it. The scope still +closes on its owning thread when the handler eventually exits; late cleanup telemetry is best effort. A compatible +ambient span uses a no-op scope and participates in the same bounded handoff. Invocation-end hooks can execute on a +different thread and do not own this scope. The explicit JDK-only opener requires explicit `@HandlerScoped` opt-in; +unannotated application methods are not invoked. The factory migration follows the3.x boundary in the migration guide. +Root-handler context requires the updated core/plugin pair. The new scope boundary isolates ordinary exceptions and nonfatal linkage +errors during open and close, continues earlier scope cleanup, and preserves the handler outcome. JVM fatal errors +remain outside that containment, including when wrapped by asynchronous completion/future exceptions. + ## Installation ```xml @@ -23,7 +55,7 @@ OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK fo ``` -For the no-arg constructor (`new InvocationOtelPlugin()`), no additional OpenTelemetry dependencies are needed — the ADOT Java agent layer provides them. +For the agent path (`InvocationOtelPlugin.factory()`), no additional OpenTelemetry dependencies are needed — the ADOT Java agent layer provides them. If you configure your own `SdkTracerProviderBuilder`, add the OpenTelemetry SDK and an exporter: @@ -40,6 +72,33 @@ If you configure your own `SdkTracerProviderBuilder`, add the OpenTelemetry SDK ``` +## Fallback execution roots + +When the backend supplies no complete remote parent, both views export a `DurableExecutionRoot` anchor before the first +invocation returns, including when it suspends with `PENDING` or fails with `RETRYING`. +The anchor is marked `durable.execution.synthetic_root=true`. Its trace and span IDs are deterministic and its start +and end timestamps are the checkpointed execution start. It does not report execution status or duration; `Workflow` +continues to report those at terminal completion. Complete remote parents remain externally owned and are never exported. +Java's `InvocationInfo` requires a non-null execution start timestamp from the initial checkpointed execution operation. +Missing timestamps are rejected before the plugin runs; anchor timestamps never fall back to the current wall clock. + +Every invocation may re-export the anchor to recover from an earlier interrupted or lost export. Re-exports retain the +same span fields under stable sampling. The existing provider resource still applies: if a later invocation runs in another +execution environment, resource attributes such as `faas.instance` can differ. Backends that deduplicate by span identity +may retain either copy's resource. Each execution ARN owns its own anchor even when multiple executions share a +propagated trace ID without a parent; they are not collapsed into one +execution. Upstream sampling and configured fallback sampling apply to anchors and their descendants together. + +After the first invocation's export and flush succeed, its anchor remains available even if the execution is stopped +or times out while suspended and never invokes the plugin again. An invocation killed before its end hook or flush can +still lose its spans; a later invocation, including terminal completion, attempts to export the anchor again. Export +and flush failures do not provide a delivery guarantee. + +For a consistent hierarchy across invocations, preserve an explicit upstream sampling decision or use a deterministic +sampling policy based on the stable trace ID. A non-deterministic sampler can export an anchor without a Workflow, or +a Workflow without its anchor. Any sampler-supplied attributes and trace state must also stay stable for identical +anchor re-exports. + ## Quick Start using X-Ray/CloudWatch Tracing (ADOT Java Agent) 1. Add the ADOT Lambda Layer to your function @@ -50,7 +109,7 @@ If you configure your own `SdkTracerProviderBuilder`, add the OpenTelemetry SDK ### 1. ADOT Lambda Layer -This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. The `new InvocationOtelPlugin()` constructor resolves the global provider initialized by the ADOT Java agent at invocation start, with deterministic span ID generation installed through the plugin's `AutoConfigurationCustomizerProvider` SPI. If the provider is not ready, the plugin emits no telemetry for that invocation and retries provider resolution on the next invocation. +This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. `InvocationOtelPlugin.factory()` resolves the global provider initialized by the ADOT Java agent when the first invocation's plugin instance is created, with deterministic span ID generation installed through the plugin's `AutoConfigurationCustomizerProvider` SPI. If the provider is not ready, that invocation's instance emits no telemetry and the next invocation's instance resolves the provider again. The layer ARN follows the format: @@ -90,6 +149,26 @@ aws lambda update-function-configuration \ Build the plugin layer ZIP with the OTel plugin JAR at `java/lib/aws-durable-execution-sdk-java-plugin-otel-.jar`. Lambda adds JARs in this directory to the Java class path. Set `OTEL_JAVAAGENT_EXTENSIONS` to the deployed JAR so the ADOT Java agent also loads its `AutoConfigurationCustomizerProvider`, and set `DURABLE_EXECUTION_PLUGINS=otel-invocation` so the Durable Execution SDK loads its `InvocationOtelPluginProvider`. +### Invocation-local headers on Lambda Managed Instances + +The SDK captures `Context.getXrayTraceId()` on the runtime thread before dispatching user work when an actual +runtime implementation overrides the accessor. It stores the immutable snapshot in `InvocationInfo.xRayTraceId()`. +The same information object reaches the ordinary `createPlugin(info)` factory and `onInvocationStart(info)` hook, +so both built-in and custom factories can read the header before creating spans. Context extraction uses `extract(info)`; +existing no-argument extractor implementations retain their default delegation. + +An override returning null/empty is represented by an empty String and is authoritative absence: deterministic fallback +is used without reading stale global context. An unavailable accessor, or only Lambda Core1.4's inherited null-returning +default, is represented by null and retains ordinary system-property then environment fallback. A malformed captured +header also never adopts the global carrier. No global header is modified. + +The header is the eighth `InvocationInfo` record component, carrying forward the 2.x metadata fix. Existing 4-, 5-, 6- +and 7-argument constructors remain and default the optional header to null. Existing compiled constructor/accessor calls +and seven-component record patterns remain compatible; recompiling a seven-component record pattern requires an eighth +binding. Reflection sees eight components, and record equality/hash calculation includes the header. `toString()` omits it. +Core/plugin artifacts still follow the explicit 3.x factory migration boundary: rebuild providers and layers against 3.x +as described in the migration guide; 2.x and 3.x plugin artifacts are not interchangeable. + ### 2. AWS X-Ray Active Tracing Enable active tracing on your Lambda function so the `_X_AMZN_TRACE_ID` environment variable is populated at invocation time. The plugin uses this header both to parent Invocation spans to the ambient Lambda/X-Ray trace and to anchor the execution trace on the propagated context when it carries a complete parent and an explicit sampling decision. @@ -130,7 +209,8 @@ public class MyHandler extends DurableHandler { @Override protected DurableConfig createConfiguration() { - return DurableConfig.builder().withPlugins(new InvocationOtelPlugin()).build(); + // A factory, not a plugin instance: the SDK creates one plugin instance per invocation from it. + return DurableConfig.builder().withPlugins(InvocationOtelPlugin.factory()).build(); } @Override @@ -193,7 +273,7 @@ The plugin decides sampling once per invocation and applies that single decision 1. **Backend decision** — `Sampled=1` / `Sampled=0` in the propagated header is authoritative and always preserved, regardless of the configured sampler. 2. **Same-trace ambient span** — when the header carries no usable `Sampled` value but a valid ambient span (for example an auto-instrumentation Lambda handler span) is already on the execution's trace, the plugin follows that span's decision: sampled → sampled; unsampled but still recording → `RECORD_ONLY`; unsampled and not recording → dropped. -3. **Configured sampler (application-owned provider)** — when you pass a `SdkTracerProvider` to the plugin, its sampler is read directly and evaluated once with the trace ID, span name, and attributes. A trace-ID-ratio sampler therefore produces a stable decision across reinvocations (the trace ID is stable). +3. **Configured sampler (application-owned provider)** — when you pass a `SdkTracerProviderBuilder` to `factory(...)`, the sampler of the provider it builds is read directly and evaluated once with the trace ID, span name, and attributes. A trace-ID-ratio sampler therefore produces a stable decision across reinvocations (the trace ID is stable). 4. **Installed sampler (Java-agent path)** — when the agent owns the provider, it is behind a classloader boundary and its *effective* sampler (which another agent extension may have wrapped or replaced) cannot be reliably read at decision time. Rather than guess, the plugin **defers**: it installs a delegating sampler through the agent's autoconfiguration and lets that wrapper consult the agent's real sampler. The delegate's decision is honored in full — if your configured policy is `always_off`, a rate limiter, or a remote sampler (`xray`, `jaeger_remote`) that returns drop, the durable spans are dropped; they are **not** force-sampled. To avoid consuming a stateful or quota-based sampler once per span, the wrapper consults the delegate once per execution (keyed by trace ID) and reuses that decision for the execution's remaining durable spans within the invocation. For precise, provider-independent control, set an explicit `Sampled` value upstream (for example by enabling X-Ray active tracing) — that backend decision takes precedence over everything else. @@ -255,21 +335,27 @@ With Lambda's `LoggingConfig: JSON` (required for durable functions), CloudWatch ## Configuration -Both plugins take a required `SdkTracerProviderBuilder` (your exporter/processor pipeline) plus an optional -`OtelPluginConfig` built with a named-field builder. This replaces the older telescoping constructors, giving readable, -type-safe call sites, and matches the `OtelPluginConfig` object in the JavaScript and Python SDKs. +Each plugin is registered as a `DurableExecutionPluginFactory` obtained from its static `factory(...)` methods, because +a plugin instance serves exactly one invocation: the SDK calls the factory once per invocation and drops the instance +when the invocation returns. The factory holds what belongs to the execution environment — your tracer provider (built +once) or the ADOT global provider binding, plus the deterministic ID generator — while each instance holds only its own +invocation's spans. + +The `factory(...)` overloads take an optional `SdkTracerProviderBuilder` (your exporter/processor pipeline) plus an +optional `OtelPluginConfig` built with a named-field builder, which matches the `OtelPluginConfig` object in the +JavaScript and Python SDKs. ### InvocationOtelPlugin ```java // Default: ADOT Java agent global provider, X-Ray context extraction, MDC enabled -new InvocationOtelPlugin(); +InvocationOtelPlugin.factory(); // Custom tracer provider pipeline, all other options defaulted -new InvocationOtelPlugin(tracerProviderBuilder); +InvocationOtelPlugin.factory(tracerProviderBuilder); // Full configuration via the builder -new InvocationOtelPlugin( +InvocationOtelPlugin.factory( tracerProviderBuilder, OtelPluginConfig.builder() .contextExtractor(new XRayContextExtractor()) @@ -282,18 +368,18 @@ new InvocationOtelPlugin( ### ExecutionOtelPlugin The `ExecutionOtelPlugin` renders the Workflow span as the durable trace root with operations beneath it. Invocation -spans remain in the ambient Lambda trace, and operations link to the Invocation that ran them. It takes the same -`(SdkTracerProviderBuilder, OtelPluginConfig)` constructor: +spans remain in the ambient Lambda trace, and operations link to the Invocation that ran them. It exposes the same +`factory(SdkTracerProviderBuilder, OtelPluginConfig)` methods: ```java // Default: ADOT Java agent global provider, X-Ray context extraction, MDC enabled -new ExecutionOtelPlugin(); +ExecutionOtelPlugin.factory(); // Custom tracer provider pipeline, all other options defaulted -new ExecutionOtelPlugin(tracerProviderBuilder); +ExecutionOtelPlugin.factory(tracerProviderBuilder); // Full configuration via the builder -new ExecutionOtelPlugin( +ExecutionOtelPlugin.factory( tracerProviderBuilder, OtelPluginConfig.builder() .enableMdc(false) @@ -310,9 +396,9 @@ new ExecutionOtelPlugin( | `workflowSpanName(...)` | Name for the Workflow span | `"Workflow"` | | `instrumentationName(...)` | Instrumentation scope name registered with the tracer | `"aws-durable-execution-sdk-java"` | -> The `tracerProviderBuilder` argument is not used by the no-arg `new InvocationOtelPlugin()` / -> `new ExecutionOtelPlugin()` constructors; those resolve the ADOT Java agent's global provider at invocation start. -> If it is not ready, all telemetry is disabled for that invocation and resolution is retried on the next invocation. +> The no-builder `InvocationOtelPlugin.factory()` / `ExecutionOtelPlugin.factory()` forms resolve the ADOT Java agent's +> global provider instead, when the first invocation's instance needs it. If it is not ready, all telemetry is disabled +> for that invocation and the next invocation's instance resolves it again. > A `null` passed to any `OtelPluginConfig` builder setter falls back to that option's default. ## Known Limitations @@ -357,7 +443,7 @@ For local testing, use a logging exporter to print spans to stdout: ```java import io.opentelemetry.exporter.logging.LoggingSpanExporter; -var otelPlugin = new InvocationOtelPlugin( +var otelPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create()))); ``` @@ -367,8 +453,21 @@ var otelPlugin = new InvocationOtelPlugin( - Java 17+ - AWS Durable Execution SDK for Java 2.0.0+ - OpenTelemetry SDK 1.65.0+ (only for custom TracerProvider path) -- ADOT Lambda Layer `AWSOpenTelemetryDistroJava` (for the no-arg constructor path) +- ADOT Lambda Layer `AWSOpenTelemetryDistroJava` (for the agent path, `factory()` without a builder) ## License Apache-2.0 + +### Installed core/plugin layer compatibility checks + +The retained `src/test/compatibility/run_matrix.py` is the2.x regression harness for the additive minor fixes. It +verifies released/current2.x artifacts in separate plugin-layer loaders without dependency-floor rejection. It is not +a3.x compatibility bridge: the factory/provider migration deliberately requires3.x artifacts and rebuilt provider +layers. The major's provider migration tests validate clear rejection of selected legacy providers; the minor PRs +retain their independently tested2.x compatibility guarantees. + +The inherited `@HandlerScoped` annotation names an explicit JDK `Function` opener. The core passes the plugin +instance to it without discovering a plugin method by name. Bundled openers call private SDK implementation code. +Custom opener classes need a public no-argument constructor accessible to the core. The factory/provider migration +still requires3.x artifacts and intentionally final view classes; this is not a cross-major subclass compatibility bridge. diff --git a/otel-plugin/pom.xml b/otel-plugin/pom.xml index c7ecb73d1..e44b503f0 100644 --- a/otel-plugin/pom.xml +++ b/otel-plugin/pom.xml @@ -7,7 +7,7 @@ software.amazon.lambda.durable aws-durable-execution-sdk-java-parent - 2.2.2-SNAPSHOT + 3.0.0-SNAPSHOT aws-durable-execution-sdk-java-plugin-otel diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ContextExtractor.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ContextExtractor.java index 0abb0c26d..cad5477c6 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ContextExtractor.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ContextExtractor.java @@ -2,15 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.otel; +import software.amazon.lambda.durable.plugin.InvocationInfo; + /** * Extracts the durable execution's propagated trace context from the Lambda runtime environment. * *

Implementations read trace context from various sources (X-Ray trace header, W3C traceparent, etc.) and return an * {@link ExtractedContext} containing the trace ID and optional parent span ID. * - *

When it is called: the plugin invokes {@link #extract()} once at the start of every invocation, - * unconditionally — including when an ambient OpenTelemetry span is active. The extracted context is the durable - * execution's identity and is resolved with the following precedence: + *

When it is called: the plugin invokes {@link #extract(InvocationInfo)} once at the start of every + * invocation, unconditionally — including when an ambient OpenTelemetry span is active. The extracted context is the + * durable execution's identity and is resolved with the following precedence: * *

    *
  1. a valid extracted backend context anchors the execution trace (this is what makes the durable spans share one @@ -34,4 +36,15 @@ public interface ContextExtractor { * @return the extracted context, or {@code null} if no context is available */ ExtractedContext extract(); + + /** + * Extracts context using the invocation and its immutable X-Ray header snapshot. Existing extractors retain their + * no-argument behavior; implementations that need invocation-local carriers can override this method. + * + *

    A null snapshot denotes an unavailable runtime accessor and permits legacy fallback. An empty string denotes + * an available accessor with no invocation header; the built-in extractor treats it as authoritative absence. + */ + default ExtractedContext extract(InvocationInfo info) { + return extract(); + } } diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java index 7ce304a54..34e222ab3 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSampler.java @@ -113,7 +113,7 @@ public SamplingResult shouldSample( SpanKind spanKind, Attributes attributes, List parentLinks) { - var intent = DurableSamplingDecision.get(parentContext); + var intent = DurableSamplingDecision.consume(parentContext); if (intent == null) { // Not a durable span: the customer's sampler governs it unchanged. return delegate.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java index c43d8eb02..9c72c39bd 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java @@ -6,6 +6,7 @@ import io.opentelemetry.context.ContextKey; import io.opentelemetry.sdk.trace.samplers.SamplingDecision; import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.util.concurrent.atomic.AtomicReference; /** * Carries the durable execution's sampling intent to {@link DurableSampler} for one durable span. @@ -32,15 +33,16 @@ * the full {@link SamplingResult}, including any attributes a custom sampler attached. *

  2. Thread-scoped system property — a cross-class-loader fallback modelled on * {@link DeterministicIdGenerator}'s scoped-ID bridge. The intent is published on the thread that creates the - * durable span for the synchronous duration of {@code startSpan()} (the sampler runs on that same thread), keyed - * by thread ID under a bootstrap-visible {@link System} property so both class loaders read the same value. A - * resolved decision bridges its {@link SamplingDecision} name (the three built-in decisions carry no attributes, - * so reconstructing them is faithful); a deferral bridges a sentinel plus the canonical trace ID. + * durable span until its sampler consumes the value (before synchronous span processors run), keyed by thread ID + * under a bootstrap-visible {@link System} property so both class loaders read the same value. A resolved + * decision bridges its {@link SamplingDecision} name (the three built-in decisions carry no attributes, so + * reconstructing them is faithful); a deferral bridges a sentinel plus the canonical trace ID. *
* *

The scope is opened immediately around each durable {@code startSpan()} call and closed right after, so the - * property never leaks beyond the span it applies to. Nothing is persisted across invocations; cross-invocation - * consistency comes from recomputing the intent from stable inputs, not from sharing state. + * property is consumed once by the sampler before callbacks can create unrelated spans. Nothing is persisted across + * invocations; cross-invocation consistency comes from recomputing the intent from stable inputs, not from sharing + * state. */ final class DurableSamplingDecision { @@ -65,7 +67,7 @@ boolean isDeferred() { } } - private static final ContextKey KEY = + private static final ContextKey> KEY = ContextKey.named("software.amazon.lambda.durable.otel.durable-sampling-decision"); private static final String SCOPED_PROPERTY_PREFIX = "software.amazon.lambda.durable.otel.scopedSamplingDecision."; @@ -74,9 +76,9 @@ boolean isDeferred() { private DurableSamplingDecision() {} - /** Returns a context carrying the durable sampling intent (same-class-loader carrier), derived from the given. */ + /** Returns a context carrying a one-shot intent for one span, preserving the full SamplingResult. */ static Context store(Context context, Intent intent) { - return context.with(KEY, intent); + return context.with(KEY, new AtomicReference<>(intent)); } /** @@ -98,17 +100,30 @@ static Scope openScope(Intent intent) { } /** - * Returns the durable sampling intent for a span, or {@code null} when none is present. Prefers the full-fidelity - * context key (same class loader) and falls back to the thread-scoped system property (cross class loader). + * Inspects the unconsumed durable sampling intent for a span, or {@code null} when none is present. Prefers the + * full-fidelity context key (same class loader) and falls back to the thread-scoped system property (cross class + * loader). */ static Intent get(Context context) { - var fromContext = context.get(KEY); + var holder = context.get(KEY); + var fromContext = holder != null ? holder.get() : null; if (fromContext != null) { return fromContext; } return fromScopedProperty(); } + /** Consumes both carriers before callbacks can reuse the parent context or cross-loader bridge. */ + static Intent consume(Context context) { + try { + var holder = context.get(KEY); + var fromContext = holder != null ? holder.getAndSet(null) : null; + return fromContext != null ? fromContext : fromScopedProperty(); + } finally { + System.clearProperty(scopedProperty()); + } + } + private static String encode(Intent intent) { return intent.isDeferred() ? DEFERRED_PREFIX + intent.deferredTraceId() diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index 09214055c..dcc82cf51 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -21,10 +21,13 @@ import java.time.Instant; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.HandlerScoped; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.OperationEndInfo; @@ -62,10 +65,20 @@ * *

The Workflow and Invocation spans share one execution trace, anchored at the execution ancestor resolved at * invocation start: a valid propagated remote server span becomes that ancestor directly, otherwise a synthetic - * execution root anchors the trace. The trace ID is stable across invocations of the same execution. When using - * {@link #ExecutionOtelPlugin()}, the plugin resolves the global provider at invocation start. If the OpenTelemetry - * Java agent is not initialized yet, telemetry is disabled for that entire invocation and provider resolution is - * retried on the next invocation. + * execution root anchors the trace. The trace ID is stable across invocations of the same execution, because it is + * derived from the execution ARN and start time rather than carried in the plugin. + * + *

Lifetime. One instance serves exactly one Lambda invocation: {@link #factory()} and its overloads return a + * {@link DurableExecutionPluginFactory} that the SDK calls once per invocation, and the instance is dropped when the + * invocation returns. Everything about the invocation — the execution ARN, the resolved execution trace and ancestor, + * the sampling intent, the Invocation span, the deferred Workflow span context — is therefore a {@code final} field, + * resolved in the constructor from the {@link InvocationInfo} the factory receives. Nothing is reset between + * invocations because nothing is carried between them. + * + *

What belongs to the execution environment stays in the factory's {@link OtelPluginEnvironment}: the configuration, + * the ID generator, and either the application-owned tracer provider (built once) or the lazily resolved ADOT global + * provider. An invocation whose instance cannot resolve the global provider emits no telemetry at all, and the next + * invocation's instance resolves it again. * *

Status mapping (parity with the Python/JS references): * @@ -85,43 +98,62 @@ * current, so {@code Span.current()} enrichment is not recorded on the final operation span. The placeholder uses the * Invocation span's resolved sampling metadata when available. * - *

Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple - * threads. + *

Thread-safe within its invocation: the SDK runs user code on multiple threads, so the open-span registries are + * {@link ConcurrentHashMap}s. The invocation's identity needs no such protection — it is final state written before the + * SDK publishes the instance to those threads. */ -public class ExecutionOtelPlugin implements DurableExecutionPlugin { +@HandlerScoped(ExecutionOtelPlugin.HandlerScopeOpener.class) +public final class ExecutionOtelPlugin implements DurableExecutionPlugin { private static final Logger logger = LoggerFactory.getLogger(ExecutionOtelPlugin.class); - private volatile SdkTracerProvider sdkTracerProvider; - private volatile Tracer tracer; + // ─── Environment lifetime (shared with every other invocation's instance) ───────────── + private final DeterministicIdGenerator idGenerator; - private final ContextExtractor contextExtractor; private final boolean enableMdc; private final String workflowSpanName; - private final String instrumentationName; - - // Per-invocation state - private volatile boolean tracingEnabled; - private volatile Span invocationSpan; - private volatile String durableExecutionArn; - - // Trace ID and flags of the execution trace, published together as one snapshot so readers never pair a trace ID - // with mismatched flags. - private volatile ExecutionTrace executionTrace; - // The execution's single sampling intent for this invocation, computed once at onInvocationStart and attached to - // every durable span's parent context so DurableSampler applies it (a resolved decision verbatim, or a deferral to - // its own delegate) without re-invoking the configured sampler per span. - private volatile DurableSamplingDecision.Intent samplingIntent; - - /** Immutable snapshot of the resolved execution trace, read atomically through a single volatile reference. */ - private record ExecutionTrace(String traceId, TraceFlags flags) {} - // Between invocations the Workflow span exists only as a deterministic context that operations parent onto; the - // recording span is started and ended in a single call on the terminal invocation, so it is never left open. The - // execution ancestor and start time are retained so that span can be built at invocation end. - private volatile SpanContext workflowSpanContext; - private volatile SpanContext executionAncestor; - private volatile Instant executionStartTime; + // ─── This invocation, all resolved in the constructor from its InvocationInfo ───────── + + /** The provider used to flush before Lambda freezes; null when it is not visible to the application. */ + private final SdkTracerProvider sdkTracerProvider; + + /** Null when telemetry is disabled for this invocation, which makes every hook on this instance a no-op. */ + private final Tracer tracer; + + private final String durableExecutionArn; + private final Instant executionStartTime; + + /** Trace ID and flags of the execution trace, resolved together so they can never be paired mismatched. */ + private final ExecutionTrace executionTrace; + + /** + * The execution's single sampling intent for this invocation, resolved once and attached to every durable span's + * parent context so DurableSampler applies it (a resolved decision verbatim, or a deferral to its own delegate) + * without re-invoking the configured sampler per span. + */ + private final DurableSamplingDecision.Intent samplingIntent; + + private final Span invocationSpan; + + /** + * The Workflow span exists as a deterministic context that operations parent onto; the recording span is started + * and ended in a single call on the terminal invocation, so it is never left open. The execution ancestor and start + * time are held so that span can be built at invocation end. + */ + private final SpanContext workflowSpanContext; + + private final SpanContext executionAncestor; + + /** + * Set when this invocation ends; never cleared, because an instance is never reused. Read by the operation and user + * function hooks, which may run on other threads of this invocation, so that a straggler hook arriving after the + * spans have been ended does not open a new one — volatile for that publication. + */ + private volatile boolean ended; + + /** Immutable snapshot of the resolved execution trace. */ + private record ExecutionTrace(String traceId, TraceFlags flags) {} // Thread-safe storage for attempt spans/scopes (keyed by operationId + "-" + attempt) private final ConcurrentHashMap attemptSpans = new ConcurrentHashMap<>(); @@ -136,89 +168,127 @@ private record ExecutionTrace(String traceId, TraceFlags flags) {} private final ConcurrentHashMap operationStartTimes = new ConcurrentHashMap<>(); /** - * Creates a Workflow-rooted OTel plugin with default settings: X-Ray context extraction, MDC enabled, root span - * named {@code "Workflow"}. + * Returns a factory that creates one plugin instance per invocation against the ADOT Java agent's global provider, + * with default settings: X-Ray context extraction, MDC enabled, root span named {@code "Workflow"}. * - *

Uses the provided tracer provider builder. For ADOT Java agent usage, prefer {@link #ExecutionOtelPlugin()} - * with the plugin jar configured through {@code OTEL_JAVAAGENT_EXTENSIONS}. + * 

{@code
+     * DurableConfig.builder().withPlugins(ExecutionOtelPlugin.factory()).build();
+     * }
* - * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped) + * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins} */ - public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { - this(tracerProviderBuilder, OtelPluginConfig.defaults()); + public static DurableExecutionPluginFactory factory() { + return factory(OtelPluginConfig.defaults()); } /** - * Creates a Workflow-rooted OTel plugin with default settings: X-Ray context extraction and MDC enabled. + * Returns a factory that creates one plugin instance per invocation against the ADOT Java agent's global provider. * - *

Resolves {@code GlobalOpenTelemetry} at invocation start. If the ADOT Java agent has not initialized it yet, - * telemetry is disabled for that invocation and resolution is retried on the next invocation. + *

The global provider is resolved when the first invocation's instance needs it. If the agent has not + * initialized it yet, that invocation emits no telemetry and the next invocation's instance resolves it again. + * + * @param config the plugin configuration + * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins} */ - public ExecutionOtelPlugin() { - this(OtelPluginConfig.defaults()); + public static DurableExecutionPluginFactory factory(OtelPluginConfig config) { + var environment = OtelPluginEnvironment.forGlobalProvider(config); + return invocationFactory(environment); } /** - * Creates a Workflow-rooted OTel plugin from the given tracer provider builder and configuration. + * Returns a factory that creates one plugin instance per invocation against an application-owned tracer provider, + * with default settings: X-Ray context extraction, MDC enabled, root span named {@code "Workflow"}. + * + *

Customers configure exporters and span processors on the builder — the plugin handles ID generation. The + * provider is built once, here, and shared by every invocation's instance. + * + * @param tracerProviderBuilder the tracer provider builder (its ID generator and sampler will be wrapped) + * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins} + */ + public static DurableExecutionPluginFactory factory(SdkTracerProviderBuilder tracerProviderBuilder) { + return factory(tracerProviderBuilder, OtelPluginConfig.defaults()); + } + + /** + * Returns a factory that creates one plugin instance per invocation against an application-owned tracer provider. * *

Customers configure exporters and span processors on the builder; all other tunables (context extractor, MDC - * toggle, Workflow span name, instrumentation scope name) come from {@link OtelPluginConfig}. Use - * {@link OtelPluginConfig#builder()} for readable, named configuration: + * toggle, Workflow span name, instrumentation scope name) come from {@link OtelPluginConfig}: * * 

{@code
-     * var plugin = new ExecutionOtelPlugin(
+     * var factory = ExecutionOtelPlugin.factory(
      *     SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
      *     OtelPluginConfig.builder().enableMdc(false).workflowSpanName("Workflow").build());
      * }
* - * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped) + * @param tracerProviderBuilder the tracer provider builder (its ID generator and sampler will be wrapped) * @param config the plugin configuration + * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins} */ - public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) { - this.idGenerator = DeterministicIdGenerator.installOn(tracerProviderBuilder); - // Wrap the configured sampler so durable spans use the execution's single precomputed decision. - DurableSampler.installOn(tracerProviderBuilder); - - this.sdkTracerProvider = tracerProviderBuilder.build(); - this.tracer = sdkTracerProvider.get(config.instrumentationName()); - this.contextExtractor = config.contextExtractor(); - this.enableMdc = config.enableMdc(); - this.workflowSpanName = config.workflowSpanName(); - this.instrumentationName = config.instrumentationName(); + public static DurableExecutionPluginFactory factory( + SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) { + var environment = OtelPluginEnvironment.forProviderBuilder(tracerProviderBuilder, config); + return invocationFactory(environment); + } + + private static DurableExecutionPluginFactory invocationFactory(OtelPluginEnvironment environment) { + return new DurableExecutionPluginFactory() { + @Override + public String getExclusiveGroup() { + return "durable-otel-view"; + } + + @Override + public String toString() { + return "ExecutionOtelPlugin"; + } + + @Override + public DurableExecutionPlugin createPlugin(InvocationInfo info) { + return new ExecutionOtelPlugin(environment, info); + } + }; } /** - * Creates a Workflow-rooted OTel plugin from configuration alone (no caller-supplied tracer provider builder). + * Creates the instance that serves one invocation. * - *

The config-only constructor uses the ADOT/global provider. Supply a {@code SdkTracerProviderBuilder} via the - * two-arg constructor for an application-owned provider. + *

Everything this invocation's spans are keyed by is resolved here, from the {@code info} the factory received: + * the tracer binding, the extracted context, the canonical execution trace and its ancestor, the single sampling + * intent, the Invocation span, and the deferred Workflow span context. Resolving them in the constructor — before + * the SDK publishes this instance to the operation and user function threads — is what lets them be {@code final} + * rather than volatile per-invocation state. * - * @param config the plugin configuration + *

When the tracer cannot be bound, telemetry is disabled for this invocation: the span fields stay null and + * every hook returns immediately. The next invocation gets a new instance, which binds again. */ - public ExecutionOtelPlugin(OtelPluginConfig config) { - this.contextExtractor = config.contextExtractor(); + private ExecutionOtelPlugin(OtelPluginEnvironment environment, InvocationInfo info) { + var config = environment.config(); + this.idGenerator = environment.idGenerator(); this.enableMdc = config.enableMdc(); this.workflowSpanName = config.workflowSpanName(); - this.instrumentationName = config.instrumentationName(); - this.idGenerator = OtelPluginSupport.createDefaultIdGenerator(); - } - - // ─── Invocation hooks ──────────────────────────────────────────────── - - @Override - public void onInvocationStart(InvocationInfo info) { - tracingEnabled = false; - if (!bindTracer()) { + this.durableExecutionArn = info.durableExecutionArn(); + this.executionStartTime = info.executionStartTime(); + + var setup = environment.bind("ExecutionOtelPlugin"); + if (setup == null) { + this.sdkTracerProvider = null; + this.tracer = null; + this.samplingIntent = null; + this.executionTrace = null; + this.executionAncestor = null; + this.invocationSpan = null; + this.workflowSpanContext = null; return; } - - this.durableExecutionArn = info.durableExecutionArn(); + this.sdkTracerProvider = setup.sdkTracerProvider(); + this.tracer = setup.tracer(); // Resolve the one execution ancestor both spans parent onto, so they share a stable-per-execution trace and a // sampling decision. - var extracted = contextExtractor.extract(); + var extracted = config.contextExtractor().extract(info); var canonicalTraceId = - ExecutionTraceContext.canonicalTraceId(extracted, arn(), info.executionStartTime(), idGenerator); + ExecutionTraceContext.canonicalTraceId(extracted, arn(), executionStartTime, idGenerator); // Resolve the execution's sampling decision once for this invocation as a full SamplingResult, then apply it to // every durable span via DurableSampler. The execution ancestor's trace flags are derived from the same // decision so a parent-based sampler stays consistent with it. @@ -231,14 +301,13 @@ public void onInvocationStart(InvocationInfo info) { Attributes.of(DURABLE_EXECUTION_ARN, arn())); // A null decision is unresolved on the agent path: defer to DurableSampler's own delegate (keyed by trace ID), // rather than fabricating a decision that would bypass an installed drop/rate-limit policy. - samplingIntent = decision != null + this.samplingIntent = decision != null ? DurableSamplingDecision.Intent.resolved(decision) : DurableSamplingDecision.Intent.deferred(canonicalTraceId); var sampled = OtelPluginSupport.isSampled(decision); var execCtx = ExecutionTraceContext.resolve(extracted, canonicalTraceId, arn(), idGenerator, () -> sampled); - executionTrace = new ExecutionTrace(canonicalTraceId, execCtx.traceFlags()); - executionAncestor = execCtx.executionAncestor(); - executionStartTime = info.executionStartTime(); + this.executionTrace = new ExecutionTrace(canonicalTraceId, execCtx.traceFlags()); + this.executionAncestor = execCtx.executionAncestor(); // Invocation span — child of the ambient Lambda span when it is on the execution trace, otherwise a child of // the execution ancestor so it stays within the same trace. @@ -246,69 +315,91 @@ public void onInvocationStart(InvocationInfo info) { var spanBuilder = tracer.spanBuilder("Invocation") .setSpanKind(SpanKind.INTERNAL) .setParent(invocationParent) - .setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn()) + .setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn) .setAttribute(DURABLE_FIRST_INVOCATION, info.isFirstInvocation()); if (info.requestId() != null) { spanBuilder.setAttribute(AttributeKey.stringKey("faas.invocation_id"), info.requestId()); } - invocationSpan = startDurableSpan(spanBuilder); + this.invocationSpan = startDurableSpan(spanBuilder); // Defer the recording Workflow span until terminal completion. The placeholder uses the Invocation span's // resolved sampling metadata so operation parents/links match the span that is eventually exported. - var workflowSpanId = idGenerator.generateWorkflowSpanId(info.durableExecutionArn()); var invocationContext = invocationSpan.getSpanContext(); - workflowSpanContext = SpanContext.create( - canonicalTraceId, workflowSpanId, invocationContext.getTraceFlags(), invocationContext.getTraceState()); + this.workflowSpanContext = SpanContext.create( + canonicalTraceId, + idGenerator.generateWorkflowSpanId(durableExecutionArn), + invocationContext.getTraceFlags(), + invocationContext.getTraceState()); + } - // Inject MDC on the handler thread so handler-level logs (between steps) have trace context. - if (enableMdc) { - MDC.put( - MdcSpanEnricher.MDC_TRACE_ID, - invocationSpan.getSpanContext().getTraceId()); + // ─── Invocation hooks ──────────────────────────────────────────────── + + @Override + public void onInvocationStart(InvocationInfo info) { + // This invocation's identity, its Invocation span and its Workflow span context were resolved in the + // constructor, from the very InvocationInfo this hook receives. What is left is the MDC injection, which + // belongs + // here because it must run on the handler thread so handler-level logs between steps carry trace context. + if (invocationSpan == null || !enableMdc) { + return; + } + MDC.put(MdcSpanEnricher.MDC_TRACE_ID, invocationSpan.getSpanContext().getTraceId()); + } + + /** JDK-only scope bridge; does not dispatch to coincidental subclass methods. */ + public static final class HandlerScopeOpener implements Function { + @Override + public AutoCloseable apply(ExecutionOtelPlugin plugin) { + return plugin.activateHandlerContext(); } - tracingEnabled = true; + } + + private AutoCloseable activateHandlerContext() { + var trace = executionTrace; + if (ended || tracer == null || trace == null) return null; + var ambient = Span.current().getSpanContext(); + // Preserve a compatible ambient Lambda span. An absent or unrelated ambient span must not leave + // handler instrumentation outside the durable execution's canonical trace. + if (ambient.isValid() && trace.traceId().equals(ambient.getTraceId())) return Scope.noop(); + return Span.wrap(workflowSpanContext).makeCurrent(); } @Override public void onInvocationEnd(InvocationEndInfo info) { - if (!tracingEnabled) { + if (disabled()) { return; } - tracingEnabled = false; + // Set before the spans are ended, so a straggler hook from another thread of this invocation cannot open a span + // under one that is already closed. Never cleared: this instance serves no second invocation. + ended = true; // Clear invocation-level MDC if (enableMdc) { MdcSpanEnricher.clear(); } - // Drop placeholder state. Open operations have no recording span to abandon. - operationContexts.clear(); - operationStartTimes.clear(); - // Release OTel context on worker threads, then end any attempt spans still open so no recording span is // abandoned. Attempt spans normally start and end within one user-function call, so this is a safeguard. for (var scope : attemptScopes.values()) { scope.close(); } - attemptScopes.clear(); for (var span : attemptSpans.values()) { span.end(); } - attemptSpans.clear(); + // The placeholder and attempt registries are not emptied: an operation that never completed has no recording + // span to abandon, every attempt span above has been ended, and this instance is dropped when the invocation + // returns, so there is nothing to recycle them for. // End the invocation span every invocation. - if (invocationSpan != null) { - invocationSpan.setAttribute( - DURABLE_INVOCATION_STATUS, info.invocationStatus().name()); - applyInvocationStatus(invocationSpan, info); - invocationSpan.end(); - invocationSpan = null; - } + invocationSpan.setAttribute( + DURABLE_INVOCATION_STATUS, info.invocationStatus().name()); + applyInvocationStatus(invocationSpan, info); + invocationSpan.end(); // Materialize the Workflow span only on terminal status. - if (isTerminal(info) && workflowSpanContext != null && executionAncestor != null) { + if (isTerminal(info)) { var workflowSpanBuilder = tracer.spanBuilder(workflowSpanName) .setSpanKind(SpanKind.INTERNAL) .setParent(withDurableDecision(Context.root().with(Span.wrap(executionAncestor)))) @@ -332,10 +423,8 @@ public void onInvocationEnd(InvocationEndInfo info) { } workflowSpan.end(); } - workflowSpanContext = null; - executionAncestor = null; - executionStartTime = null; - samplingIntent = null; + OtelPluginSupport.exportExecutionRoot( + tracer, idGenerator, executionAncestor, durableExecutionArn, executionStartTime, samplingIntent); // Flush spans before Lambda freezes if (sdkTracerProvider != null) { @@ -350,7 +439,7 @@ public void onInvocationEnd(InvocationEndInfo info) { @Override public void onOperationStart(OperationInfo info) { - if (!tracingEnabled) return; + if (disabled()) return; if (info.id() == null) return; // Retain only a deterministic placeholder. Its flags/state come from the Invocation span's resolved sampling @@ -368,7 +457,7 @@ public void onOperationStart(OperationInfo info) { @Override public void onOperationEnd(OperationEndInfo info) { - if (!tracingEnabled) return; + if (disabled()) return; if (info.id() == null) return; // Start and end the operation's single span here, using its deterministic span ID and linking to the @@ -428,7 +517,7 @@ public void onOperationEnd(OperationEndInfo info) { @Override public void onUserFunctionStart(UserFunctionStartInfo info) { - if (!tracingEnabled) return; + if (disabled()) return; // Skip attempt spans for CONTEXT operations — they are a scoping construct, not a retriable unit of work. Still // make the operation's context current so auto-instrumented calls become children of the (deferred) operation @@ -486,7 +575,7 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { @Override public void onUserFunctionEnd(UserFunctionEndInfo info) { - if (!tracingEnabled) return; + if (disabled()) return; var key = attemptKey(info.id(), info.attempt()); @@ -524,22 +613,12 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) { // ─── Helpers ───────────────────────────────────────────────────────── - private boolean bindTracer() { - if (tracer != null) { - return true; - } - synchronized (this) { - if (tracer != null) { - return true; - } - var setup = OtelPluginSupport.tryResolveGlobalProvider(instrumentationName, "ExecutionOtelPlugin"); - if (setup == null) { - return false; - } - sdkTracerProvider = setup.sdkTracerProvider(); - tracer = setup.tracer(); - return true; - } + /** + * True when this instance emits no telemetry: either the tracer could not be bound for this invocation, or the + * invocation has already ended and its spans are closed. + */ + private boolean disabled() { + return invocationSpan == null || ended; } private void applyInvocationStatus(Span span, InvocationEndInfo info) { @@ -636,17 +715,11 @@ private Context withDurableDecision(Context context) { } private TraceFlags effectiveTraceFlags() { - var invocation = invocationSpan; - if (invocation != null) { - return invocation.getSpanContext().getTraceFlags(); - } - var trace = executionTrace; - return trace != null ? trace.flags() : TraceFlags.getDefault(); + return invocationSpan.getSpanContext().getTraceFlags(); } private TraceState effectiveTraceState() { - var invocation = invocationSpan; - return invocation != null ? invocation.getSpanContext().getTraceState() : TraceState.getDefault(); + return invocationSpan.getSpanContext().getTraceState(); } /** diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginProvider.java index 011e1dc02..01aee63c3 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginProvider.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginProvider.java @@ -3,30 +3,32 @@ package software.amazon.lambda.durable.otel; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider; +import software.amazon.lambda.durable.plugin.InvocationInfo; /** * Dynamically loads {@link ExecutionOtelPlugin} when {@code DURABLE_EXECUTION_PLUGINS} contains {@code otel-execution}. + * + *

The provider is itself the per-invocation factory: it holds the environment-lifetime state (the ADOT global + * provider binding, the ID generator) once and creates one plugin instance per invocation from it. */ public final class ExecutionOtelPluginProvider implements DurableExecutionPluginProvider { + private final DurableExecutionPluginFactory factory = ExecutionOtelPlugin.factory(); + @Override public String getName() { return "otel-execution"; } @Override - public int getApiVersion() { - return API_VERSION; - } - - @Override - public Class getPluginType() { - return ExecutionOtelPlugin.class; + public DurableExecutionPlugin createPlugin(InvocationInfo invocationInfo) { + return factory.createPlugin(invocationInfo); } @Override - public DurableExecutionPlugin createPlugin() { - return new ExecutionOtelPlugin(); + public String getExclusiveGroup() { + return factory.getExclusiveGroup(); } } diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionTraceContext.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionTraceContext.java index cf8a3dc4b..b3e7eaea5 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionTraceContext.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionTraceContext.java @@ -39,8 +39,9 @@ * so a remote parent left unsampled would make a parent-based sampler drop every child span; deferring to the sampler * avoids that. * - *

The ancestor is a non-recording context: it is either the external backend server span or a synthetic root the SDK - * does not export. + *

The ancestor is a non-recording context: it is either the external backend server span or an SDK-owned synthetic + * root. Both plugins export the synthetic root before each invocation returns, including suspension, with its stable + * execution-start timestamp and zero duration. They never export a remote parent. * * @param executionAncestor the common parent context for the Workflow and Invocation spans */ diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index e89804bdc..9da13db78 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -22,10 +22,13 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.HandlerScoped; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.OperationEndInfo; @@ -61,9 +64,18 @@ *

  • Tracing: Active (to populate {@code _X_AMZN_TRACE_ID}) * * - *

    When using {@link #InvocationOtelPlugin()}, the plugin resolves the global provider at invocation start. If the - * OpenTelemetry Java agent is not initialized yet, telemetry is disabled for that entire invocation and provider - * resolution is retried on the next invocation. + *

    Lifetime. One instance serves exactly one Lambda invocation: {@link #factory()} and its overloads return a + * {@link DurableExecutionPluginFactory} that the SDK calls once per invocation, and the instance is dropped when the + * invocation returns. Everything about the invocation — the execution ARN, the resolved execution trace and ancestor, + * the sampling intent, the Invocation span, the deferred Workflow span context — is therefore a {@code final} field, + * resolved in the constructor from the {@link InvocationInfo} the factory receives (the same instance + * {@link #onInvocationStart(InvocationInfo)} then receives). Nothing is reset between invocations because nothing is + * carried between them. + * + *

    What belongs to the execution environment stays in the factory's {@link OtelPluginEnvironment}: the configuration, + * the ID generator, and either the application-owned tracer provider (built once) or the lazily resolved ADOT global + * provider. On the agent path, an invocation whose instance cannot resolve the global provider emits no telemetry at + * all, and the next invocation's instance resolves it again. * *

    X-Ray console limitation: In the X-Ray "Segments Timeline" ungrouped view, the plugin's spans (Invocation, * operation, attempt) do not appear as nested subsegments of the Lambda platform segment. This is a known limitation of @@ -72,39 +84,57 @@ * view to inspect parent-child relationships within the shared execution trace and the links between operation spans * and the Workflow span. * - *

    Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple - * threads. + *

    Thread-safe within its invocation: the SDK runs user code on multiple threads, so the open-span registries are + * {@link ConcurrentHashMap}s. The invocation's identity needs no such protection — it is final state written before the + * SDK publishes the instance to those threads. */ -public class InvocationOtelPlugin implements DurableExecutionPlugin { +@HandlerScoped(InvocationOtelPlugin.HandlerScopeOpener.class) +public final class InvocationOtelPlugin implements DurableExecutionPlugin { private static final Logger logger = LoggerFactory.getLogger(InvocationOtelPlugin.class); - private volatile SdkTracerProvider sdkTracerProvider; - private volatile Tracer tracer; + // ─── Environment lifetime (shared with every other invocation's instance) ───────────── + private final DeterministicIdGenerator idGenerator; - private final ContextExtractor contextExtractor; private final boolean enableMdc; private final String workflowSpanName; - private final String instrumentationName; - - // Per-invocation state - private volatile boolean tracingEnabled; - private volatile Span invocationSpan; - private volatile String durableExecutionArn; - // Trace ID and flags of the execution trace, published together as one snapshot so readers never pair a trace ID - // with mismatched flags. - private volatile ExecutionTrace executionTrace; - // The execution's single sampling intent for this invocation, computed once at onInvocationStart and attached to - // every durable span's parent context so DurableSampler applies it (a resolved decision verbatim, or a deferral to - // its own delegate) without re-invoking the configured sampler per span. - private volatile DurableSamplingDecision.Intent samplingIntent; - - // Deferred Workflow placeholder; the recording span is emitted only on terminal invocation. - private volatile SpanContext workflowSpanContext; - private volatile SpanContext executionAncestor; - private volatile Instant executionStartTime; - - /** Immutable snapshot of the resolved execution trace, read atomically through a single volatile reference. */ + + // ─── This invocation, all resolved in the constructor from its InvocationInfo ───────── + + /** The provider used to flush before Lambda freezes; null when it is not visible to the application. */ + private final SdkTracerProvider sdkTracerProvider; + + /** Null when telemetry is disabled for this invocation, which makes every hook on this instance a no-op. */ + private final Tracer tracer; + + private final String durableExecutionArn; + private final Instant executionStartTime; + + /** Trace ID and flags of the execution trace, resolved together so they can never be paired mismatched. */ + private final ExecutionTrace executionTrace; + + /** + * The execution's single sampling intent for this invocation, resolved once and attached to every durable span's + * parent context so DurableSampler applies it (a resolved decision verbatim, or a deferral to its own delegate) + * without re-invoking the configured sampler per span. + */ + private final DurableSamplingDecision.Intent samplingIntent; + + private final SpanContext executionAncestor; + + private final Span invocationSpan; + + /** Deferred Workflow placeholder; the recording span is emitted only on terminal invocation. */ + private final SpanContext workflowSpanContext; + + /** + * Set when this invocation ends; never cleared, because an instance is never reused. Read by the operation and user + * function hooks, which may run on other threads of this invocation, so that a straggler hook arriving after the + * spans have been ended does not open a new one — volatile for that publication. + */ + private volatile boolean ended; + + /** Immutable snapshot of the resolved execution trace. */ private record ExecutionTrace(String traceId, TraceFlags flags) {} // Thread-safe storage for operation spans (keyed by operationId) — open spans that need ending @@ -121,97 +151,133 @@ private record ExecutionTrace(String traceId, TraceFlags flags) {} private final ConcurrentLinkedDeque operationStartOrder = new ConcurrentLinkedDeque<>(); /** - * Creates an OTel plugin with default settings: X-Ray context extraction, MDC enabled. - * - *

    Uses the provided tracer provider builder. Customers configure exporters and span processors on the builder — - * the plugin handles ID generation. - * - *

    For ADOT Java agent usage, prefer {@link #InvocationOtelPlugin()} with the plugin jar configured through - * {@code OTEL_JAVAAGENT_EXTENSIONS}. Use this builder constructor when you want to own the exporter pipeline: + * Returns a factory that creates one plugin instance per invocation against the ADOT Java agent's global provider, + * with default settings: X-Ray context extraction and MDC enabled. * * 

    {@code
-     * var exporter = LoggingSpanExporter.create();
-     * var plugin = new InvocationOtelPlugin(
-     *     SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)));
+     * DurableConfig.builder().withPlugins(InvocationOtelPlugin.factory()).build();
      * }
    * - * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped) + * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins} + */ + public static DurableExecutionPluginFactory factory() { + return factory(OtelPluginConfig.defaults()); + } + + /** + * Returns a factory that creates one plugin instance per invocation against the ADOT Java agent's global provider. + * + *

    The global provider is resolved when the first invocation's instance needs it. If the agent has not + * initialized it yet, that invocation emits no telemetry and the next invocation's instance resolves it again. + * + * @param config the plugin configuration + * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins} */ - public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) { - this(tracerProviderBuilder, OtelPluginConfig.defaults()); + public static DurableExecutionPluginFactory factory(OtelPluginConfig config) { + var environment = OtelPluginEnvironment.forGlobalProvider(config); + return invocationFactory(environment); } /** - * Creates an OTel plugin with default settings: X-Ray context extraction and MDC enabled. + * Returns a factory that creates one plugin instance per invocation against an application-owned tracer provider, + * with default settings: X-Ray context extraction and MDC enabled. + * + *

    Customers configure exporters and span processors on the builder — the plugin handles ID generation. The + * provider is built once, here, and shared by every invocation's instance: + * + * 

    {@code
+     * var exporter = LoggingSpanExporter.create();
+     * var factory = InvocationOtelPlugin.factory(
+     *     SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)));
+     * }
    * - *

    Resolves {@code GlobalOpenTelemetry} at invocation start. If the ADOT Java agent has not initialized it yet, - * telemetry is disabled for that invocation and resolution is retried on the next invocation. + * @param tracerProviderBuilder the tracer provider builder (its ID generator and sampler will be wrapped) + * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins} */ - public InvocationOtelPlugin() { - this(OtelPluginConfig.defaults()); + public static DurableExecutionPluginFactory factory(SdkTracerProviderBuilder tracerProviderBuilder) { + return factory(tracerProviderBuilder, OtelPluginConfig.defaults()); } /** - * Creates an OTel plugin from the given tracer provider builder and configuration. + * Returns a factory that creates one plugin instance per invocation against an application-owned tracer provider. * *

    Customers configure exporters and span processors on the builder; all other tunables (context extractor, MDC - * toggle, Workflow span name, instrumentation scope name) come from {@link OtelPluginConfig}. Use - * {@link OtelPluginConfig#builder()} for readable, named configuration: + * toggle, Workflow span name, instrumentation scope name) come from {@link OtelPluginConfig}: * * 

    {@code
-     * var plugin = new InvocationOtelPlugin(
+     * var factory = InvocationOtelPlugin.factory(
      *     SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)),
      *     OtelPluginConfig.builder().enableMdc(false).workflowSpanName("Workflow").build());
      * }
    * - * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped) + * @param tracerProviderBuilder the tracer provider builder (its ID generator and sampler will be wrapped) * @param config the plugin configuration + * @return the per-invocation plugin factory to hand to {@code DurableConfig.Builder.withPlugins} */ - public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) { - this.idGenerator = DeterministicIdGenerator.installOn(tracerProviderBuilder); - // Wrap the configured sampler so durable spans use the execution's single precomputed decision. - DurableSampler.installOn(tracerProviderBuilder); - - this.sdkTracerProvider = tracerProviderBuilder.build(); - this.tracer = sdkTracerProvider.get(config.instrumentationName()); - this.contextExtractor = config.contextExtractor(); - this.enableMdc = config.enableMdc(); - this.workflowSpanName = config.workflowSpanName(); - this.instrumentationName = config.instrumentationName(); + public static DurableExecutionPluginFactory factory( + SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) { + var environment = OtelPluginEnvironment.forProviderBuilder(tracerProviderBuilder, config); + return invocationFactory(environment); + } + + private static DurableExecutionPluginFactory invocationFactory(OtelPluginEnvironment environment) { + return new DurableExecutionPluginFactory() { + @Override + public String getExclusiveGroup() { + return "durable-otel-view"; + } + + @Override + public String toString() { + return "InvocationOtelPlugin"; + } + + @Override + public DurableExecutionPlugin createPlugin(InvocationInfo info) { + return new InvocationOtelPlugin(environment, info); + } + }; } /** - * Creates an OTel plugin from configuration alone (no caller-supplied tracer provider builder). + * Creates the instance that serves one invocation. * - *

    The config-only constructor uses the ADOT/global provider. Supply a {@code SdkTracerProviderBuilder} via the - * two-arg constructor for an application-owned provider. + *

    Everything this invocation's spans are keyed by is resolved here, from the {@code info} the factory received: + * the tracer binding, the extracted context, the canonical execution trace and its ancestor, the single sampling + * intent, the Invocation span, and the deferred Workflow span context. Resolving them in the constructor — before + * the SDK publishes this instance to the operation and user function threads — is what lets them be {@code final} + * rather than volatile per-invocation state. * - * @param config the plugin configuration + *

    When the tracer cannot be bound, telemetry is disabled for this invocation: the span fields stay null and + * every hook returns immediately. The next invocation gets a new instance, which binds again. */ - public InvocationOtelPlugin(OtelPluginConfig config) { - this.contextExtractor = config.contextExtractor(); + private InvocationOtelPlugin(OtelPluginEnvironment environment, InvocationInfo info) { + var config = environment.config(); + this.idGenerator = environment.idGenerator(); this.enableMdc = config.enableMdc(); this.workflowSpanName = config.workflowSpanName(); - this.instrumentationName = config.instrumentationName(); - this.idGenerator = OtelPluginSupport.createDefaultIdGenerator(); - } - - // ─── Invocation hooks ──────────────────────────────────────────────── - - @Override - public void onInvocationStart(InvocationInfo info) { - tracingEnabled = false; - if (!bindTracer()) { + this.durableExecutionArn = info.durableExecutionArn(); + this.executionStartTime = info.executionStartTime(); + + var setup = environment.bind("InvocationOtelPlugin"); + if (setup == null) { + this.sdkTracerProvider = null; + this.tracer = null; + this.samplingIntent = null; + this.executionTrace = null; + this.executionAncestor = null; + this.invocationSpan = null; + this.workflowSpanContext = null; return; } + this.sdkTracerProvider = setup.sdkTracerProvider(); + this.tracer = setup.tracer(); - this.durableExecutionArn = info.durableExecutionArn(); - - var extracted = contextExtractor.extract(); + var extracted = config.contextExtractor().extract(info); // Resolve the execution ancestor the Workflow span parents onto so it joins the stable-per-execution trace. - var canonicalTraceId = ExecutionTraceContext.canonicalTraceId( - extracted, info.durableExecutionArn(), info.executionStartTime(), idGenerator); + var canonicalTraceId = + ExecutionTraceContext.canonicalTraceId(extracted, durableExecutionArn, executionStartTime, idGenerator); // Resolve the execution's sampling decision once for this invocation as a full SamplingResult, then apply it to // every durable span via DurableSampler (see below). The execution ancestor's trace flags are derived from the // same decision so a parent-based sampler stays consistent with it. @@ -221,18 +287,17 @@ public void onInvocationStart(InvocationInfo info) { Span.current(), canonicalTraceId, workflowSpanName, - Attributes.of(DURABLE_EXECUTION_ARN, info.durableExecutionArn())); + Attributes.of(DURABLE_EXECUTION_ARN, durableExecutionArn)); // A null decision is unresolved on the agent path: defer to DurableSampler's own delegate (keyed by trace ID), // rather than fabricating a decision that would bypass an installed drop/rate-limit policy. - samplingIntent = decision != null + this.samplingIntent = decision != null ? DurableSamplingDecision.Intent.resolved(decision) : DurableSamplingDecision.Intent.deferred(canonicalTraceId); var sampled = OtelPluginSupport.isSampled(decision); var execCtx = ExecutionTraceContext.resolve( - extracted, canonicalTraceId, info.durableExecutionArn(), idGenerator, () -> sampled); - executionTrace = new ExecutionTrace(canonicalTraceId, execCtx.traceFlags()); - executionAncestor = execCtx.executionAncestor(); - executionStartTime = info.executionStartTime(); + extracted, canonicalTraceId, durableExecutionArn, idGenerator, () -> sampled); + this.executionTrace = new ExecutionTrace(canonicalTraceId, execCtx.traceFlags()); + this.executionAncestor = execCtx.executionAncestor(); // Invocation span parent — the same-trace ambient span when available, then the execution ancestor, so the // Invocation span stays on the execution trace. @@ -242,46 +307,71 @@ public void onInvocationStart(InvocationInfo info) { var spanBuilder = tracer.spanBuilder("Invocation") .setSpanKind(SpanKind.INTERNAL) .setParent(parentContext) - .setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn()) + .setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn) .setAttribute(DURABLE_FIRST_INVOCATION, info.isFirstInvocation()); if (info.requestId() != null) { spanBuilder.setAttribute(AttributeKey.stringKey("faas.invocation_id"), info.requestId()); } - invocationSpan = startDurableSpan(spanBuilder); + this.invocationSpan = startDurableSpan(spanBuilder); // Defer the recording Workflow span until terminal completion. The placeholder uses the Invocation span's // resolved sampling metadata so operation links match the span that is eventually exported. - var workflowSpanId = idGenerator.generateWorkflowSpanId(info.durableExecutionArn()); var invocationContext = invocationSpan.getSpanContext(); - workflowSpanContext = SpanContext.create( - canonicalTraceId, workflowSpanId, invocationContext.getTraceFlags(), invocationContext.getTraceState()); + this.workflowSpanContext = SpanContext.create( + canonicalTraceId, + idGenerator.generateWorkflowSpanId(durableExecutionArn), + invocationContext.getTraceFlags(), + invocationContext.getTraceState()); + } - // Inject MDC on the handler thread so handler-level logs (between steps) have trace context. - // This runs on the same thread as context.getLogger() calls in the handler. - if (enableMdc) { - MDC.put( - MdcSpanEnricher.MDC_TRACE_ID, - invocationSpan.getSpanContext().getTraceId()); + // ─── Invocation hooks ──────────────────────────────────────────────── + + @Override + public void onInvocationStart(InvocationInfo info) { + // This invocation's identity and its Invocation span were resolved in the constructor, from the very + // InvocationInfo this hook receives. What is left is the MDC injection, which belongs here because it must run + // on the handler thread — the same thread as the context.getLogger() calls in the handler — so handler-level + // logs between steps carry trace context. + if (invocationSpan == null || !enableMdc) { + return; + } + MDC.put(MdcSpanEnricher.MDC_TRACE_ID, invocationSpan.getSpanContext().getTraceId()); + } + + /** JDK-only scope bridge; does not dispatch to coincidental subclass methods. */ + public static final class HandlerScopeOpener implements Function { + @Override + public AutoCloseable apply(InvocationOtelPlugin plugin) { + return plugin.activateHandlerContext(); } - tracingEnabled = true; + } + + private AutoCloseable activateHandlerContext() { + var trace = executionTrace; + if (ended || tracer == null || trace == null) return null; + var ambient = Span.current().getSpanContext(); + // Preserve a compatible ambient Lambda span. An absent or unrelated ambient span must not leave + // handler instrumentation outside the durable execution's canonical trace. + if (ambient.isValid() && trace.traceId().equals(ambient.getTraceId())) return Scope.noop(); + return invocationSpan.makeCurrent(); } @Override public void onInvocationEnd(InvocationEndInfo info) { - if (!tracingEnabled) { + if (disabled()) { return; } - tracingEnabled = false; + // Set before the spans are ended, so a straggler hook from another thread of this invocation cannot open a span + // under one that is already closed. Never cleared: this instance serves no second invocation. + ended = true; // Clear invocation-level MDC (set in onInvocationStart on the handler thread) if (enableMdc) { MdcSpanEnricher.clear(); } - if (invocationSpan == null) return; - endOpenSpansChildFirst(); // End invocation span @@ -311,10 +401,9 @@ public void onInvocationEnd(InvocationEndInfo info) { } invocationSpan.end(); - invocationSpan = null; // Materialize the Workflow span only on terminal status. - if (isTerminal(info) && workflowSpanContext != null && executionAncestor != null) { + if (isTerminal(info)) { var workflowSpanBuilder = tracer.spanBuilder(workflowSpanName) .setSpanKind(SpanKind.INTERNAL) .setParent(withDurableDecision(Context.root().with(Span.wrap(executionAncestor)))) @@ -338,10 +427,8 @@ public void onInvocationEnd(InvocationEndInfo info) { } workflowSpan.end(); } - workflowSpanContext = null; - executionAncestor = null; - executionStartTime = null; - samplingIntent = null; + OtelPluginSupport.exportExecutionRoot( + tracer, idGenerator, executionAncestor, durableExecutionArn, executionStartTime, samplingIntent); if (sdkTracerProvider != null) { // Flush spans before Lambda freezes @@ -356,7 +443,7 @@ public void onInvocationEnd(InvocationEndInfo info) { @Override public void onOperationStart(OperationInfo info) { - if (!tracingEnabled) return; + if (disabled()) return; if (info.id() == null) return; var parentContext = resolveParentContext(info.parentId()); @@ -397,7 +484,7 @@ public void onOperationStart(OperationInfo info) { @Override public void onOperationEnd(OperationEndInfo info) { - if (!tracingEnabled) return; + if (disabled()) return; if (info.id() == null) return; var span = operationSpans.remove(info.id()); @@ -468,7 +555,7 @@ public void onOperationEnd(OperationEndInfo info) { @Override public void onUserFunctionStart(UserFunctionStartInfo info) { - if (!tracingEnabled) return; + if (disabled()) return; // Skip attempt spans for CONTEXT operations — they are a scoping construct, not a // retriable unit of work, so attempt number/outcome attributes don't apply. @@ -528,7 +615,7 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { @Override public void onUserFunctionEnd(UserFunctionEndInfo info) { - if (!tracingEnabled) return; + if (disabled()) return; var key = attemptKey(info.id(), info.attempt()); @@ -575,34 +662,22 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) { // ─── Helpers ───────────────────────────────────────────────────────── - private boolean bindTracer() { - if (tracer != null) { - return true; - } - synchronized (this) { - if (tracer != null) { - return true; - } - var setup = OtelPluginSupport.tryResolveGlobalProvider(instrumentationName, "InvocationOtelPlugin"); - if (setup == null) { - return false; - } - sdkTracerProvider = setup.sdkTracerProvider(); - tracer = setup.tracer(); - return true; - } + /** + * True when this instance emits no telemetry: either the tracer could not be bound for this invocation, or the + * invocation has already ended and its spans are closed. + */ + private boolean disabled() { + return invocationSpan == null || ended; } private void endOpenSpansChildFirst() { - // Attempt spans are children of operation spans. + // Attempt spans are children of operation spans, so release their scopes and end them first. for (var scope : attemptScopes.values()) { scope.close(); } - attemptScopes.clear(); for (var span : attemptSpans.values()) { span.end(); } - attemptSpans.clear(); // End still-open operation spans with the STARTED status set in onOperationStart. // A later invocation's onOperationEnd emits a continuation span with the real terminal status. @@ -613,8 +688,8 @@ private void endOpenSpansChildFirst() { span.end(); } } - operationSpans.clear(); - operationContexts.clear(); + // The registries are not emptied afterwards: every span they held has been ended above, and this instance is + // dropped when the invocation returns, so there is nothing to recycle them for. } /** @@ -710,17 +785,11 @@ private void addInitialOperationLink(SpanBuilder spanBuilder, String operationId } private TraceFlags effectiveTraceFlags() { - var invocation = invocationSpan; - if (invocation != null) { - return invocation.getSpanContext().getTraceFlags(); - } - var trace = executionTrace; - return trace != null ? trace.flags() : TraceFlags.getDefault(); + return invocationSpan.getSpanContext().getTraceFlags(); } private TraceState effectiveTraceState() { - var invocation = invocationSpan; - return invocation != null ? invocation.getSpanContext().getTraceState() : TraceState.getDefault(); + return invocationSpan.getSpanContext().getTraceState(); } private static boolean isTerminal(InvocationEndInfo info) { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPluginProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPluginProvider.java index 165ed88fc..caa5a0c0a 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPluginProvider.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPluginProvider.java @@ -3,31 +3,33 @@ package software.amazon.lambda.durable.otel; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider; +import software.amazon.lambda.durable.plugin.InvocationInfo; /** * Dynamically loads {@link InvocationOtelPlugin} when {@code DURABLE_EXECUTION_PLUGINS} contains * {@code otel-invocation}. + * + *

    The provider is itself the per-invocation factory: it holds the environment-lifetime state (the ADOT global + * provider binding, the ID generator) once and creates one plugin instance per invocation from it. */ public final class InvocationOtelPluginProvider implements DurableExecutionPluginProvider { + private final DurableExecutionPluginFactory factory = InvocationOtelPlugin.factory(); + @Override public String getName() { return "otel-invocation"; } @Override - public int getApiVersion() { - return API_VERSION; - } - - @Override - public Class getPluginType() { - return InvocationOtelPlugin.class; + public DurableExecutionPlugin createPlugin(InvocationInfo invocationInfo) { + return factory.createPlugin(invocationInfo); } @Override - public DurableExecutionPlugin createPlugin() { - return new InvocationOtelPlugin(); + public String getExclusiveGroup() { + return factory.getExclusiveGroup(); } } diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java index ed4ac9be5..8fc5b136a 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java @@ -10,8 +10,8 @@ * mirrors the {@code OtelPluginConfig} object in the JavaScript SDK and the {@code OtelPluginConfig} dataclass in the * Python SDK for cross-SDK parity. * - *

    Construct via {@link #builder()} and pass to a plugin's {@code (SdkTracerProviderBuilder, OtelPluginConfig)} - * constructor: + *

    Construct via {@link #builder()} and pass to a plugin's {@code factory(SdkTracerProviderBuilder, + * OtelPluginConfig)}: * * 

    {@code
  * var config = OtelPluginConfig.builder()
@@ -20,7 +20,7 @@
  *     .workflowSpanName("Workflow")
  *     .instrumentationName("my-scope")
  *     .build();
- * var plugin = new InvocationOtelPlugin(tracerProviderBuilder, config);
+ * var factory = InvocationOtelPlugin.factory(tracerProviderBuilder, config);
  * }
    * *

    Defaults: {@code contextExtractor = new XRayContextExtractor()}, {@code enableMdc = true}, {@code workflowSpanName diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginEnvironment.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginEnvironment.java new file mode 100644 index 000000000..69b87515b --- /dev/null +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginEnvironment.java @@ -0,0 +1,104 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import java.util.Objects; + +/** + * Everything the OTel plugins need that belongs to the execution environment rather than to one invocation. + * + *

    A plugin instance now serves exactly one Lambda invocation, so the objects that must exist once per environment + * live here: the resolved {@link OtelPluginConfig}, the {@link DeterministicIdGenerator}, and — for an + * application-owned tracer provider — the built provider and its tracer. {@code InvocationOtelPlugin.factory(...)} and + * {@code ExecutionOtelPlugin.factory(...)} create one of these and hand the same instance to every plugin instance they + * create, so the provider is built (and its ID generator and sampler installed) once per environment rather than once + * per invocation. + * + *

    On the ADOT Java agent path there is no provider to build here: the global provider is resolved on first use and + * then cached. An invocation that runs before the agent has finished initializing therefore disables telemetry for + * itself only, and the next invocation's instance resolves the provider again. + */ +final class OtelPluginEnvironment { + + private final OtelPluginConfig config; + private final DeterministicIdGenerator idGenerator; + + /** The application-owned provider and tracer, or null on the Java agent path. */ + private final OtelPluginSupport.ProviderSetup ownedSetup; + + /** + * The global provider and tracer, once resolved. Environment-lifetime state shared by every invocation's instance, + * hence volatile; a lost race only resolves the same global provider twice. + */ + private volatile OtelPluginSupport.ProviderSetup resolvedGlobalSetup; + + /** + * @throws NullPointerException if the config is null. The check belongs here because every factory overload on both + * plugins reaches this constructor, and because the alternative is silence: the global-provider path only + * stores the config, so a null one would first be dereferenced when an invocation's plugin instance is built, + * where {@code PluginRunner} contains the failure. The function would then run without the telemetry it asked + * for, reporting one warning per invocation. Registration is where a caller can still act on it. + */ + private OtelPluginEnvironment( + OtelPluginConfig config, DeterministicIdGenerator idGenerator, OtelPluginSupport.ProviderSetup ownedSetup) { + this.config = Objects.requireNonNull(config, "config must not be null"); + this.idGenerator = idGenerator; + this.ownedSetup = ownedSetup; + } + + /** + * Builds the application-owned provider once: wraps the builder's ID generator and sampler, builds the provider and + * gets the tracer. Every invocation's plugin instance then shares them. + */ + static OtelPluginEnvironment forProviderBuilder( + SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) { + // Checked before anything is consumed. The constructor below checks it too, but by then this method has + // installed the ID generator and the sampler on the caller's builder and built a provider -- and a provider + // that fails validation is unreachable, so its span processors and their worker threads are never shut down. + Objects.requireNonNull(config, "config must not be null"); + var idGenerator = DeterministicIdGenerator.installOn(tracerProviderBuilder); + // Wrap the configured sampler so durable spans use the execution's single precomputed decision. + DurableSampler.installOn(tracerProviderBuilder); + var sdkTracerProvider = tracerProviderBuilder.build(); + var setup = new OtelPluginSupport.ProviderSetup( + sdkTracerProvider, sdkTracerProvider.get(config.instrumentationName())); + return new OtelPluginEnvironment(config, idGenerator, setup); + } + + /** The Java agent path: the global provider is resolved lazily, when an invocation's instance first needs it. */ + static OtelPluginEnvironment forGlobalProvider(OtelPluginConfig config) { + return new OtelPluginEnvironment(config, OtelPluginSupport.createDefaultIdGenerator(), null); + } + + OtelPluginConfig config() { + return config; + } + + DeterministicIdGenerator idGenerator() { + return idGenerator; + } + + /** + * The provider and tracer one invocation's plugin instance should use, or {@code null} when telemetry must be + * disabled for that invocation because the agent's global provider is not available yet. + * + * @param pluginName the plugin name used in diagnostics + */ + OtelPluginSupport.ProviderSetup bind(String pluginName) { + if (ownedSetup != null) { + return ownedSetup; + } + var alreadyResolved = resolvedGlobalSetup; + if (alreadyResolved != null) { + return alreadyResolved; + } + var setup = OtelPluginSupport.tryResolveGlobalProvider(config.instrumentationName(), pluginName); + if (setup != null) { + // Resolution succeeded, so it holds for the rest of this environment's life: cache it so later invocations + // neither re-resolve nor re-log it. A failure is not cached — that is what makes the retry per invocation. + resolvedGlobalSetup = setup; + } + return setup; + } +} diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 63fbec908..00a3abb77 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -2,9 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.otel; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_ARN; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_SYNTHETIC_ROOT; + import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.api.trace.TracerProvider; @@ -13,13 +17,43 @@ import io.opentelemetry.sdk.trace.samplers.SamplingResult; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; import java.util.Collections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Shared utilities for OTel plugin default constructor support (ADOT Java agent SPI path). */ +/** Shared utilities for the OTel plugins' ADOT Java agent SPI path. */ final class OtelPluginSupport { + /** + * Exports the SDK-owned fallback ancestor before each invocation returns, including PENDING. It is a zero-duration + * identity anchor at the checkpointed execution start, not a completion summary. Recovery re-exports use identical + * identity and timestamps; Workflow alone carries execution duration and outcome. Remote parents are never owned. + */ + static void exportExecutionRoot( + Tracer tracer, + DeterministicIdGenerator idGenerator, + SpanContext ancestor, + String arn, + Instant start, + DurableSamplingDecision.Intent intent) { + if (ancestor == null || ancestor.isRemote()) { + return; + } + var builder = tracer.spanBuilder("DurableExecutionRoot") + .setSpanKind(SpanKind.INTERNAL) + .setParent(DurableSamplingDecision.store(Context.root(), intent)) + .setAttribute(DURABLE_EXECUTION_ARN, arn) + .setAttribute(DURABLE_EXECUTION_SYNTHETIC_ROOT, true) + .setStartTimestamp(start); + Span root; + try (var ignored = DurableSamplingDecision.openScope(intent)) { + root = idGenerator.startSpan(builder, ancestor.getTraceId(), ancestor.getSpanId()); + } + // End processors/exporters may create unrelated spans; they must not inherit this sampling override. + root.end(start); + } + private static final Logger logger = LoggerFactory.getLogger(OtelPluginSupport.class); private OtelPluginSupport() {} @@ -47,7 +81,7 @@ static DeterministicIdGenerator createDefaultIdGenerator() { * sampled span yields {@code RECORD_AND_SAMPLE}; an unsampled but recording span yields {@code RECORD_ONLY} * (its spans still reach processors); only an unsampled, non-recording span yields {@code DROP}; *

  • Application-owned provider: configured sampler, once. When the tracer provider is reachable (the - * two-argument constructor path), its sampler is read directly and evaluated a single time with + * application-owned provider path), its sampler is read directly and evaluated a single time with * {@code ROOT_CONTEXT} (so a parent-based sampler applies its root policy), the canonical trace ID, span * name, and attributes, and its full result is returned; *
  • Java-agent path: defer to the installed sampler. When the provider is not visible diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java index 62510c547..c669c86b9 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/SpanAttributes.java @@ -10,6 +10,8 @@ final class SpanAttributes { private SpanAttributes() {} static final AttributeKey DURABLE_EXECUTION_ARN = AttributeKey.stringKey("durable.execution.arn"); + static final AttributeKey DURABLE_EXECUTION_SYNTHETIC_ROOT = + AttributeKey.booleanKey("durable.execution.synthetic_root"); static final AttributeKey DURABLE_EXECUTION_STATUS = AttributeKey.stringKey("durable.execution.status"); static final AttributeKey DURABLE_OPERATION_ID = AttributeKey.stringKey("durable.operation.id"); static final AttributeKey DURABLE_OPERATION_TYPE = AttributeKey.stringKey("durable.operation.type"); diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java index a42b5ee5c..a728132ae 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java @@ -7,9 +7,11 @@ import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.lambda.durable.plugin.InvocationInfo; /** - * Extracts OTel trace context from the AWS X-Ray {@code _X_AMZN_TRACE_ID} environment variable. + * Extracts OTel trace context from the invocation-local X-Ray header, using ordinary Lambda carriers only when no + * runtime snapshot is available. * *

    This extractor parses the Lambda/X-Ray header and returns the trace ID in OTel format (32 hex chars) along with * the parent span ID (16 hex chars). Plugins use it as a fallback parent for Invocation spans; the deterministic @@ -28,8 +30,40 @@ public class XRayContextExtractor implements ContextExtractor { private static final Pattern HEX_32 = Pattern.compile("[0-9a-f]{32}"); private static final Pattern HEX_16 = Pattern.compile("[0-9a-f]{16}"); + // Scoped only around extraction so legacy no-argument overrides retain virtual dispatch. + private final ThreadLocal invocationTraceHeader = new ThreadLocal<>(); + + @Override + public ExtractedContext extract(InvocationInfo info) { + var previous = invocationTraceHeader.get(); + invocationTraceHeader.set(runtimeHeader(info)); + try { + return extract(); + } finally { + if (previous == null) { + invocationTraceHeader.remove(); + } else { + invocationTraceHeader.set(previous); + } + } + } + + private static String runtimeHeader(InvocationInfo info) { + try { + return info.xRayTraceId(); + } catch (NoSuchMethodError olderCore) { + // Preserve extractor compatibility when InvocationInfo predates the optional accessor. + return null; + } + } + @Override public ExtractedContext extract() { + var invocationHeader = invocationTraceHeader.get(); + // A captured snapshot is authoritative even when empty or malformed; only null permits legacy fallback. + if (invocationHeader != null) { + return parseHeader(invocationHeader); + } // Try system property first — the Lambda runtime interface client updates this per invocation, so it reflects // the current invocation and avoids the JVM's process-lifetime environment-variable caching. var traceHeader = System.getProperty(XRAY_SYSTEM_PROPERTY); @@ -42,6 +76,10 @@ public ExtractedContext extract() { return null; } + return parseHeader(traceHeader); + } + + private static ExtractedContext parseHeader(String traceHeader) { String root = null; String parent = null; String sampled = null; diff --git a/otel-plugin/src/test/compatibility/PluginLayerCompatibilityProbe.java b/otel-plugin/src/test/compatibility/PluginLayerCompatibilityProbe.java new file mode 100644 index 000000000..096c7e9d9 --- /dev/null +++ b/otel-plugin/src/test/compatibility/PluginLayerCompatibilityProbe.java @@ -0,0 +1,76 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.ServiceLoader; +import software.amazon.lambda.durable.plugin.*; + +public class PluginLayerCompatibilityProbe { + public static void main(String[] args) throws Exception { + var path = Path.of(args[0]); + var view = args[1]; + var header = "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=1"; + var old = System.getProperty("com.amazonaws.xray.traceHeader"); + try (var layer = new URLClassLoader(new URL[] {path.toUri().toURL()}, + PluginLayerCompatibilityProbe.class.getClassLoader())) { + System.setProperty("com.amazonaws.xray.traceHeader", header); + var provider = ServiceLoader.load(DurableExecutionPluginProvider.class, layer).stream() + .map(ServiceLoader.Provider::get).filter(p -> p.getName().equals(view)).findFirst().orElseThrow(); + if (provider.getApiVersion() != DurableExecutionPluginProvider.API_VERSION) throw new AssertionError("provider API"); + var type = provider.getPluginType(); + if (type.getClassLoader() != layer) throw new AssertionError("plugin did not load from separate layer"); + if (provider.createPlugin() == null) throw new AssertionError("dynamic provider constructor"); + var configType = layer.loadClass("software.amazon.lambda.durable.otel.OtelPluginConfig"); + var configBuilder = configType.getMethod("builder").invoke(null); + configBuilder.getClass().getMethod("enableMdc", boolean.class).invoke(configBuilder, false); + var config = configBuilder.getClass().getMethod("build").invoke(configBuilder); + var exporter = InMemorySpanExporter.create(); + var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var plugin = (DurableExecutionPlugin) type.getConstructor(SdkTracerProviderBuilder.class, configType) + .newInstance(builder, config); + var runner = new PluginRunner(List.of(plugin)); + runner.onInvocationStart(new InvocationInfo("request", "arn:exec", true, Instant.EPOCH)); + runner.onInvocationEnd(new InvocationEndInfo("request", "arn:exec", true, InvocationStatus.SUCCEEDED, null)); + var spans = exporter.getFinishedSpanItems(); + if (spans.size() != 2 || spans.stream().anyMatch(s -> !s.getTraceId().equals("6955b900123456789012345678901234"))) + throw new AssertionError("legacy tracing failed: " + spans); + type.getMethods(); // Public reflection must also work with an older core. + checkOptionalHeaderDispatch(runner, plugin, exporter, type); + System.out.println("PASS " + view + " core=" + DurableExecutionPlugin.class.getProtectionDomain() + .getCodeSource().getLocation() + " layer=" + path); + } finally { + if (old == null) System.clearProperty("com.amazonaws.xray.traceHeader"); + else System.setProperty("com.amazonaws.xray.traceHeader", old); + } + } + private static void checkOptionalHeaderDispatch(PluginRunner runner, DurableExecutionPlugin plugin, + InMemorySpanExporter exporter, Class type) throws Exception { + Method hook; + try { + hook = PluginRunner.class.getMethod("onInvocationStart", InvocationInfo.class, String.class); + } catch (NoSuchMethodException olderCore) { + return; // The older core retains the original hook and ordinary carriers. + } + boolean supportsHeader; + try { + type.getDeclaredMethod("onInvocationStart", InvocationInfo.class, String.class); + supportsHeader = true; + } catch (NoSuchMethodException olderPlugin) { + supportsHeader = false; + } + exporter.reset(); + hook.invoke(runner, new InvocationInfo("request-2", "arn:exec", false, Instant.EPOCH), + "Root=1-6955b900-aaaaaaaaaaaaaaaaaaaaaaaa;Parent=abcdefabcdefabcd;Sampled=0"); + runner.onInvocationEnd(new InvocationEndInfo("request-2", "arn:exec", false, InvocationStatus.SUCCEEDED, null)); + if (exporter.getFinishedSpanItems().size() != (supportsHeader ? 0 : 2)) + throw new AssertionError("Optional dispatch did not preserve new/legacy behavior"); + } +} diff --git a/otel-plugin/src/test/compatibility/run_matrix.py b/otel-plugin/src/test/compatibility/run_matrix.py new file mode 100644 index 000000000..d38b5d629 --- /dev/null +++ b/otel-plugin/src/test/compatibility/run_matrix.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run 2.x-only installed core/plugin compatibility with an actual separate layer loader and a legacy-compiled caller. + +No downloads or AWS calls. Supply released core/plugin jars, built current artifacts/directories, and an existing +classpath file containing their common dependencies. No SDK production dependencies are added for this check. +""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile +import zipfile +import xml.etree.ElementTree as ET + +parser = argparse.ArgumentParser(description=__doc__) +for name in ("old-core", "old-plugin", "new-core", "new-plugin", "dependencies-classpath"): + parser.add_argument("--" + name, required=True, type=Path) +parser.add_argument("--java-home", required=True, type=Path) +args = parser.parse_args() +def artifact_version(path): + if path.is_dir(): + version_file = path / "version.prop" + if version_file.exists(): + properties = version_file.read_text() + else: + pom = path.parent.parent / "pom.xml" + if not pom.exists(): + parser.error(f"Cannot verify SDK version for {path}; supply a Maven artifact") + root = ET.parse(pom).getroot() + ns = {"m": "http://maven.apache.org/POM/4.0.0"} + version = root.find("m:version", ns) + if version is None: + version = root.find("m:parent/m:version", ns) + return version.text if version is not None else None + else: + with zipfile.ZipFile(path) as jar: + names = [name for name in jar.namelist() + if name.startswith("META-INF/maven/software.amazon.lambda.durable/") + and name.endswith("/pom.properties")] + if not names: + parser.error(f"Cannot verify SDK version for {path}") + properties = jar.read(names[0]).decode() + for line in properties.splitlines(): + if line.startswith("version="): + return line.split("=", 1)[1].strip() + return None + +for artifact in (args.old_core, args.old_plugin, args.new_core, args.new_plugin): + version = artifact_version(artifact) + if not version or not version.startswith("2."): + parser.error(f"This legacy ABI harness accepts only 2.x artifacts; {artifact} has version {version!r}. " + "Use the factory/provider migration tests for 3.x.") + +known = {p.resolve() for p in (args.old_core, args.old_plugin, args.new_core, args.new_plugin)} +entries = args.dependencies_classpath.read_text().strip().split(os.pathsep) +deps = [p for p in entries if Path(p).resolve() not in known + and not Path(p).name.startswith("aws-durable-execution-sdk-java")] +source = Path(__file__).with_name("PluginLayerCompatibilityProbe.java") +target = Path.cwd() / "target" +target.mkdir(exist_ok=True) +with tempfile.TemporaryDirectory(prefix="layer-compat-", dir=target) as output: + subprocess.run([str(args.java_home / "bin/javac"), "--release", "17", "-cp", + os.pathsep.join([str(args.old_core), *deps]), "-d", output, str(source)], check=True) + for label, core, plugin in [("old/old", args.old_core, args.old_plugin), + ("old/new", args.old_core, args.new_plugin), + ("new/old", args.new_core, args.old_plugin), + ("new/new", args.new_core, args.new_plugin)]: + for view in ("otel-invocation", "otel-execution"): + result = subprocess.run([str(args.java_home / "bin/java"), "-cp", + os.pathsep.join([output, str(core), *deps]), "PluginLayerCompatibilityProbe", str(plugin), view], + capture_output=True, text=True) + if result.returncode: + raise RuntimeError(f"{label} {view} failed:\n{result.stdout}{result.stderr}") + print(label, result.stdout.strip()) diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/OtelViewRegistrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/OtelViewRegistrationTest.java new file mode 100644 index 000000000..4b3337352 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/OtelViewRegistrationTest.java @@ -0,0 +1,199 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.MDC; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.otel.ExecutionOtelPlugin; +import software.amazon.lambda.durable.otel.ExecutionOtelPluginProvider; +import software.amazon.lambda.durable.otel.InvocationOtelPlugin; +import software.amazon.lambda.durable.otel.InvocationOtelPluginProvider; +import software.amazon.lambda.durable.otel.OtelPluginConfig; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class OtelViewRegistrationTest { + @Test + void factoryMetadataRejectsConflictsBeforeCreatingInstances() { + var created = new AtomicInteger(); + var factory = new DurableExecutionPluginFactory() { + @Override + public String getExclusiveGroup() { + return "custom-tracing"; + } + + @Override + public DurableExecutionPlugin createPlugin(InvocationInfo info) { + created.incrementAndGet(); + return new DurableExecutionPlugin() {}; + } + }; + assertThrows( + IllegalArgumentException.class, + () -> DurableConfig.builder().withPlugins(factory, factory).build()); + assertDoesNotThrow(() -> DurableConfig.builder().withPlugins(factory).build()); + assertEquals(0, created.get()); + } + + @ParameterizedTest + @CsvSource({"explicit,false", "explicit,true", "dynamic,false", "dynamic,true", "mixed,false", "mixed,true"}) + void rejectsBothViewsBeforeEmissionAndLeavesContextUntouched(String path, boolean reversed) { + var exporter = InMemorySpanExporter.create(); + var first = plugin(reversed, exporter); + var second = plugin(!reversed, exporter); + var providers = List.of(new ExecutionOtelPluginProvider(), new InvocationOtelPluginProvider()); + var names = reversed ? "otel-execution,otel-invocation" : "otel-invocation,otel-execution"; + var plugins = + switch (path) { + case "dynamic" -> DynamicPluginLoader.loadConfiguredPluginFactories(names, providers, List.of()); + case "mixed" -> + DynamicPluginLoader.loadConfiguredPluginFactories( + reversed ? "otel-execution" : "otel-invocation", providers, List.of(second)); + default -> List.of(first, second); + }; + var ambient = Span.wrap(SpanContext.create( + "12345678901234567890123456789012", + "1234567890123456", + TraceFlags.getSampled(), + TraceState.getDefault())); + try (var ignored = ambient.makeCurrent()) { + var context = Context.current(); + MDC.put("trace_id", "existing"); + try { + var error = assertThrows( + IllegalArgumentException.class, + () -> DurableConfig.builder() + .withPlugins(plugins.toArray(DurableExecutionPluginFactory[]::new)) + .build()); + assertTrue(error.getMessage().contains("InvocationOtelPlugin")); + assertTrue(error.getMessage().contains("ExecutionOtelPlugin")); + assertTrue(error.getMessage().contains("only one")); + assertTrue(exporter.getFinishedSpanItems().isEmpty()); + assertSame(context, Context.current()); + assertEquals("existing", MDC.get("trace_id")); + } finally { + MDC.remove("trace_id"); + } + } + } + + @ParameterizedTest + @CsvSource({"true,true", "true,false", "false,true", "false,false"}) + void singleViewWithUnrelatedPluginPreservesResumeAndOutcome(boolean executionView, boolean success) { + var exporter = InMemorySpanExporter.create(); + var view = plugin(executionView, exporter); + var config = DurableConfig.builder() + .withPlugins(view, info -> new DurableExecutionPlugin() {}) + .build(); + var effects = new AtomicInteger(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> { + ctx.step("once", Integer.class, stepCtx -> effects.incrementAndGet()); + ctx.wait("pause", Duration.ofSeconds(10)); + if (!success) { + throw new IllegalArgumentException("expected failure"); + } + return input; + }, + config); + var original = Context.current(); + assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); + runner.advanceTime(); + assertEquals( + success ? ExecutionStatus.SUCCEEDED : ExecutionStatus.FAILED, + runner.runUntilComplete("input").getStatus()); + assertSame(original, Context.current()); + assertEquals(1, effects.get()); + var spans = exporter.getFinishedSpanItems(); + assertEquals( + 1, spans.stream().filter(s -> s.getName().equals("Workflow")).count()); + assertEquals( + 2, spans.stream().filter(s -> s.getName().equals("Invocation")).count()); + } + + @ParameterizedTest + @ValueSource(strings = {"otel-invocation", "otel-execution"}) + void environmentSelectedSingleViewCanBeCopiedIntoLocalRunner(String provider, @TempDir Path directory) + throws Exception { + var java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + var classpath = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")); + var output = directory.resolve("child.log"); + var builder = new ProcessBuilder(java, "-cp", classpath, EnvironmentRunnerCheck.class.getName()) + .redirectErrorStream(true) + .redirectOutput(output.toFile()); + builder.environment().put("DURABLE_EXECUTION_PLUGINS", provider); + var child = builder.start(); + try { + assertTrue(child.waitFor(30, TimeUnit.SECONDS), "Child JVM did not finish"); + assertEquals(0, child.exitValue(), () -> read(output)); + } finally { + child.destroyForcibly(); + } + } + + private static String read(Path path) { + try { + return Files.readString(path); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + public static class EnvironmentRunnerCheck { + public static void main(String[] args) { + var config = DurableConfig.builder() + .withDeserializeAfterSerialization(false) + .build(); + assertEquals(1, config.getPluginFactories().size()); + var copy = config.toBuilder().build(); + assertEquals(config.getPluginFactories(), copy.getPluginFactories()); + assertFalse(copy.shouldDeserializeAfterSerialization()); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> { + ctx.wait("pause", Duration.ofSeconds(1)); + return input; + }, + config); + assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); + runner.advanceTime(); + assertEquals( + ExecutionStatus.SUCCEEDED, runner.runUntilComplete("input").getStatus()); + } + } + + private static DurableExecutionPluginFactory plugin(boolean executionView, InMemorySpanExporter exporter) { + var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var config = OtelPluginConfig.builder() + .contextExtractor(() -> null) + .enableMdc(false) + .build(); + return executionView + ? ExecutionOtelPlugin.factory(builder, config) + : InvocationOtelPlugin.factory(builder, config); + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DelayedFlush.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DelayedFlush.java new file mode 100644 index 000000000..a762d23e4 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DelayedFlush.java @@ -0,0 +1,57 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SpanProcessor; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +class DelayedFlush implements SpanProcessor { + public void onStart(Context parent, ReadWriteSpan span) {} + + final HandlerScopeFinalizationTest.Deadline deadline; + final AtomicInteger calls = new AtomicInteger(); + final AtomicInteger remainingAtFlush = new AtomicInteger(); + final AtomicLong flushMillis = new AtomicLong(); + final ScheduledExecutorService clock; + + DelayedFlush(HandlerScopeFinalizationTest.Deadline deadline, ScheduledExecutorService clock) { + this.deadline = deadline; + this.clock = clock; + } + + public boolean isStartRequired() { + return false; + } + + public void onEnd(ReadableSpan span) {} + + public boolean isEndRequired() { + return false; + } + + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } + + public CompletableResultCode forceFlush() { + calls.incrementAndGet(); + remainingAtFlush.set(deadline.remaining()); + var result = new CompletableResultCode(); + long start = System.nanoTime(); + clock.schedule( + () -> { + flushMillis.set(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)); + result.succeed(); + }, + 250, + TimeUnit.MILLISECONDS); + return result; + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java index 6fea5784e..f84338d2e 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplerTest.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import static software.amazon.lambda.durable.otel.Invocations.started; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.Attributes; @@ -62,6 +63,32 @@ void tearDown() { OtelPluginAutoConfigurationState.resetInstalledForTest(); } + @Test + void consumingContextIntentAlsoClearsCrossLoaderFallback() { + var delegate = new CountingSampler(Sampler.alwaysOff()); + var sampler = DurableSampler.wrap(delegate); + var parent = DurableSamplingDecision.store( + Context.root(), DurableSamplingDecision.Intent.resolved(SamplingResult.recordOnly())); + try (var ignored = DurableSamplingDecision.openScope( + DurableSamplingDecision.Intent.resolved(SamplingResult.recordAndSample()))) { + assertEquals( + SamplingDecision.RECORD_ONLY, + sampler.shouldSample(parent, TRACE_ID, "durable", SpanKind.INTERNAL, Attributes.empty(), List.of()) + .getDecision()); + assertEquals( + SamplingDecision.DROP, + sampler.shouldSample( + Context.root(), + TRACE_ID, + "callback", + SpanKind.INTERNAL, + Attributes.empty(), + List.of()) + .getDecision()); + assertEquals(1, delegate.count(), "The unrelated callback must use its own sampler"); + } + } + // ─── Unit tests for the wrapper ────────────────────────────────────── @Test @@ -124,9 +151,10 @@ void deferredIntent_evaluatesDelegateOncePerExecution() { // A stateful/quota delegate must be consulted once per execution (trace ID), not per durable span. var delegate = new CountingSampler(Sampler.alwaysOn()); var sampler = DurableSampler.wrap(delegate); - var parent = DurableSamplingDecision.store(Context.root(), DurableSamplingDecision.Intent.deferred(TRACE_ID)); - for (var i = 0; i < 4; i++) { + // Each SDK-owned span receives its own one-shot carrier, while the resolved decision stays execution-wide. + var parent = + DurableSamplingDecision.store(Context.root(), DurableSamplingDecision.Intent.deferred(TRACE_ID)); sampler.shouldSample(parent, TRACE_ID, "op" + i, SpanKind.INTERNAL, Attributes.empty(), List.of()); } @@ -172,7 +200,7 @@ private static Sampler captureInstalledSampler(Sampler effectiveSampler) { void configuredSampler_isEvaluatedAtMostOncePerInvocation() { var delegate = new CountingSampler(Sampler.alwaysOn()); var exporter = InMemorySpanExporter.create(); - var plugin = new InvocationOtelPlugin( + var pluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().setSampler(delegate).addSpanProcessor(SimpleSpanProcessor.create(exporter)), OtelPluginConfig.builder() .contextExtractor(() -> null) @@ -180,7 +208,7 @@ void configuredSampler_isEvaluatedAtMostOncePerInvocation() { .build()); // A full invocation with a Workflow span, Invocation span, operation span, and attempt span. - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(pluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onUserFunctionStart( @@ -226,7 +254,7 @@ void explicitNotSampled_winsOverConfiguredAlwaysOn() { private InMemorySpanExporter exportedWith(Sampler configuredSampler, ExtractedContext.Sampling sampling) { var exporter = InMemorySpanExporter.create(); var extracted = new ExtractedContext(TRACE_ID, SPAN_ID, sampling); - var plugin = new InvocationOtelPlugin( + var pluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder() .setSampler(configuredSampler) .addSpanProcessor(SimpleSpanProcessor.create(exporter)), @@ -235,7 +263,7 @@ private InMemorySpanExporter exportedWith(Sampler configuredSampler, ExtractedCo .enableMdc(false) .build()); - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(pluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); return exporter; } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java index 121f4c8fb..bcb9e6c32 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java @@ -53,13 +53,16 @@ void decisionCrossesClassLoaderBoundary_viaScopedProperty() throws Exception { openScope.setAccessible(true); var get = agentDecision.getDeclaredMethod("get", Context.class); get.setAccessible(true); + var consume = agentDecision.getDeclaredMethod("consume", Context.class); + consume.setAccessible(true); // The application-side loader publishes the intent on this thread; the agent-side loader reads it back from // a ROOT context (its context key would be a different instance and would miss), reconstructing its own // Intent from the bridged value. var scope = (AutoCloseable) openScope.invoke(null, appIntent); try { - var crossLoaderIntent = get.invoke(null, Context.root()); + var crossLoaderIntent = consume.invoke(null, Context.root()); + assertNull(get.invoke(null, Context.root()), "The bridge is consumed before onStart processors run"); assertNotNull( crossLoaderIntent, "The agent-side loader must read the intent published by the app side"); // Its Intent type is the agent loader's copy; read the resolved SamplingResult reflectively. diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java index 129a1b045..471d3505e 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginIntegrationTest.java @@ -42,14 +42,15 @@ void setUp() { OtelPluginAutoConfigurationState.resetInstalledForTest(); spanExporter = InMemorySpanExporter.create(); - var plugin = new ExecutionOtelPlugin( + // One factory for the environment; the SDK creates one plugin instance per invocation from it. + var factory = ExecutionOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), OtelPluginConfig.builder() .contextExtractor(() -> null) .enableMdc(false) .build()); - otelConfig = DurableConfig.builder().withPlugins(plugin).build(); + otelConfig = DurableConfig.builder().withPlugins(factory).build(); } @AfterEach @@ -170,8 +171,9 @@ public ContextPropagators getPropagators() { } }); - var defaultConfig = - DurableConfig.builder().withPlugins(new ExecutionOtelPlugin()).build(); + var defaultConfig = DurableConfig.builder() + .withPlugins(ExecutionOtelPlugin.factory()) + .build(); var runner = LocalDurableTestRunner.create( String.class, (input, ctx) -> ctx.step("wrapped-step", String.class, stepCtx -> "Hello " + input), diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java index 8b84efc10..c5d607297 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionOtelPluginTest.java @@ -3,6 +3,7 @@ package software.amazon.lambda.durable.otel; import static org.junit.jupiter.api.Assertions.*; +import static software.amazon.lambda.durable.otel.Invocations.started; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.AttributeKey; @@ -37,7 +38,9 @@ class ExecutionOtelPluginTest { private static final String CONFIGURED_SERVICE_NAME = "durable-execution-conformance"; private InMemorySpanExporter spanExporter; - private ExecutionOtelPlugin plugin; + + /** The environment's plugin factory; each test creates one instance per invocation from it. */ + private DurableExecutionPluginFactory factory; @BeforeEach void setUp() { @@ -46,7 +49,7 @@ void setUp() { OtelPluginAutoConfigurationState.resetInstalledForTest(); spanExporter = InMemorySpanExporter.create(); var resource = Resource.create(Attributes.of(SERVICE_NAME, CONFIGURED_SERVICE_NAME)); - plugin = new ExecutionOtelPlugin( + factory = ExecutionOtelPlugin.factory( SdkTracerProvider.builder() .setResource(resource) .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), @@ -65,12 +68,12 @@ void tearDown() { OtelPluginAutoConfigurationState.resetInstalledForTest(); } - // ─── Default constructor ───────────────────────────────────────────── + // ─── Java agent path (global provider) ─────────────────────────────── @Test void customInstrumentationName_isUsedForTracerScope() { var exporter = InMemorySpanExporter.create(); - var customPlugin = new ExecutionOtelPlugin( + var customPluginFactory = ExecutionOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), OtelPluginConfig.builder() .contextExtractor(() -> null) @@ -78,7 +81,7 @@ void customInstrumentationName_isUsedForTracerScope() { .workflowSpanName("Workflow") .instrumentationName("my-custom-scope") .build()); - customPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var customPlugin = started(customPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now())); customPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); var spans = exporter.getFinishedSpanItems(); @@ -89,12 +92,13 @@ void customInstrumentationName_isUsedForTracerScope() { } @Test - void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() { + void agentPathFactory_bindsGlobalProviderOnALaterInvocationsInstance() { GlobalOpenTelemetry.resetForTest(); OtelPluginAutoConfigurationState.markInstalled(); - var defaultPlugin = new ExecutionOtelPlugin(); - defaultPlugin.onInvocationStart(new InvocationInfo("req-disabled", "arn:disabled", true, Instant.now())); + var defaultPluginFactory = ExecutionOtelPlugin.factory(); + var defaultPlugin = + started(defaultPluginFactory, new InvocationInfo("req-disabled", "arn:disabled", true, Instant.now())); defaultPlugin.onOperationStart(new OperationInfo( "op-disabled", "disabled-step", "STEP", "Step", null, Instant.now(), null, null, false)); defaultPlugin.onOperationEnd(new OperationEndInfo( @@ -121,7 +125,8 @@ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() { .build(); OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); - defaultPlugin.onInvocationStart(new InvocationInfo("req-enabled", "arn:enabled", true, Instant.now())); + defaultPlugin = + started(defaultPluginFactory, new InvocationInfo("req-enabled", "arn:enabled", true, Instant.now())); defaultPlugin.onOperationStart(new OperationInfo( "op-enabled", "enabled-step", "STEP", "Step", null, Instant.now(), null, null, false)); defaultPlugin.onOperationEnd(new OperationEndInfo( @@ -141,14 +146,14 @@ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() { new InvocationEndInfo("req-enabled", "arn:enabled", true, InvocationStatus.SUCCEEDED, null)); var spans = globalExporter.getFinishedSpanItems(); - assertEquals(3, spans.size()); + assertEquals(4, spans.size()); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("enabled-step"))); assertFalse(spans.stream().anyMatch(span -> span.getName().equals("disabled-step"))); } @Test - void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { - var defaultPlugin = new ExecutionOtelPlugin(); + void agentPathFactory_usesGlobalSdkTracerProviderDirectly() { + var defaultPluginFactory = ExecutionOtelPlugin.factory(); assertFalse(GlobalOpenTelemetry.isSet()); OtelPluginAutoConfigurationState.markInstalled(); @@ -158,7 +163,8 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { .build(); OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); - defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var defaultPlugin = + started(defaultPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); defaultPlugin.onOperationStart( new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, null, false)); defaultPlugin.onOperationEnd(new OperationEndInfo( @@ -178,8 +184,8 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = globalExporter.getFinishedSpanItems(); - // Workflow + Invocation + operation = 3 - assertEquals(3, spans.size()); + // Workflow + Invocation + operation + DurableExecutionRoot = 4 + assertEquals(4, spans.size()); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("Workflow"))); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("Invocation"))); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); @@ -194,19 +200,26 @@ void executionOtelPluginProvider_isRegisteredAsServiceProvider() { .get(); assertEquals("otel-execution", provider.getName()); - assertEquals(DurableExecutionPluginProvider.API_VERSION, provider.getApiVersion()); - assertEquals(ExecutionOtelPlugin.class, provider.getPluginType()); + + // The provider is the per-invocation factory: it creates an ExecutionOtelPlugin for the invocation it is + // handed, + // and a distinct instance for the next one. + var first = provider.createPlugin(new InvocationInfo("req-1", ARN, true, Instant.now())); + var second = provider.createPlugin(new InvocationInfo("req-2", ARN, false, Instant.now())); + assertInstanceOf(ExecutionOtelPlugin.class, first); + assertInstanceOf(ExecutionOtelPlugin.class, second); + assertNotSame(first, second, "Each invocation gets its own plugin instance"); } // ─── Workflow root span lifecycle ──────────────────────────────────── @Test void terminalInvocation_exportsWorkflowAndInvocationSpans() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size(), "Terminal invocation should export the Workflow span and the invocation span"); + assertEquals(3, spans.size(), "Terminal invocation exports Workflow, Invocation, and DurableExecutionRoot"); var workflowSpan = spanByName(spans, "Workflow"); var invocationSpan = spanByName(spans, "Invocation"); @@ -217,7 +230,7 @@ void terminalInvocation_exportsWorkflowAndInvocationSpans() { @Test void spans_preserveConfiguredServiceName() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); for (var span : spanExporter.getFinishedSpanItems()) { @@ -231,7 +244,7 @@ void spans_preserveConfiguredServiceName() { @Test void workflowSpan_startsAtExecutionStartTime() { var start = Instant.parse("2026-01-15T08:00:00Z"); - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, start)); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, start)); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); var workflowSpan = spanByName(spanExporter.getFinishedSpanItems(), "Workflow"); @@ -243,7 +256,7 @@ void workflowSpan_startsAtExecutionStartTime() { @Test void workflowSpan_hasInternalKind() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); assertEquals( @@ -254,7 +267,7 @@ void workflowSpan_hasInternalKind() { @Test void workflowAndInvocationSpans_shareExecutionTrace_withoutAmbientContext() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); @@ -280,7 +293,8 @@ void workflowAndInvocationSpans_shareExecutionTrace_withoutAmbientContext() { void invocationStart_joinsAmbientTrace_whenAmbientIsOnExecutionTrace() { // Drive an invocation to learn the canonical execution trace ID, then start a fresh invocation with an ambient // span on that same trace: the Invocation span joins the ambient span directly. - plugin.onInvocationStart(new InvocationInfo("req-0", ARN, true, Instant.now())); + var executionStart = Instant.parse("2026-01-01T00:00:00Z"); + var plugin = started(factory, new InvocationInfo("req-0", ARN, true, executionStart)); plugin.onInvocationEnd(new InvocationEndInfo("req-0", ARN, true, InvocationStatus.SUCCEEDED, null)); var canonicalTraceId = spanByName(spanExporter.getFinishedSpanItems(), "Workflow").getTraceId(); @@ -290,7 +304,7 @@ void invocationStart_joinsAmbientTrace_whenAmbientIsOnExecutionTrace() { var ambient = SpanContext.create(canonicalTraceId, ambientSpanId, TraceFlags.getSampled(), TraceState.getDefault()); try (var ignored = Span.wrap(ambient).makeCurrent()) { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, false, Instant.now())); + plugin = started(factory, new InvocationInfo("req-1", ARN, false, executionStart)); } plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, false, InvocationStatus.SUCCEEDED, null)); @@ -309,8 +323,11 @@ void invocationStart_staysOnExecutionTrace_withoutLinkingAmbientSpan() { var ambientSpanId = "1111111111111111"; var ambient = SpanContext.create(ambientTraceId, ambientSpanId, TraceFlags.getSampled(), TraceState.getDefault()); + // The instance is created inside the ambient scope because the invocation's parent resolution happens when the + // factory creates it, not later. + DurableExecutionPlugin plugin; try (var ignored = Span.wrap(ambient).makeCurrent()) { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); } plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); @@ -330,7 +347,7 @@ void contextExtractor_isInvokedEveryInvocation_evenWithAmbientSpan_andBackendCon var backendParentId = "2222222222222222"; var extractCalls = new AtomicInteger(); var exporter = InMemorySpanExporter.create(); - var extractorPlugin = new ExecutionOtelPlugin( + var extractorPluginFactory = ExecutionOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), OtelPluginConfig.builder() .contextExtractor(() -> { @@ -347,8 +364,9 @@ void contextExtractor_isInvokedEveryInvocation_evenWithAmbientSpan_andBackendCon "1111111111111111", TraceFlags.getSampled(), TraceState.getDefault()); + DurableExecutionPlugin extractorPlugin; try (var ignored = Span.wrap(ambient).makeCurrent()) { - extractorPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + extractorPlugin = started(extractorPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now())); } extractorPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); @@ -375,8 +393,10 @@ void executionTrace_isStableAcrossReinvocations_withDifferentAmbientTraces() { TraceState.getDefault()); var startTime = Instant.now(); + // Two invocations of the same execution, so two instances from the same factory. + DurableExecutionPlugin plugin; try (var ignored = Span.wrap(ambientA).makeCurrent()) { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, startTime)); + plugin = started(factory, new InvocationInfo("req-1", ARN, true, startTime)); } plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); var firstInvocationTrace = @@ -384,7 +404,7 @@ void executionTrace_isStableAcrossReinvocations_withDifferentAmbientTraces() { spanExporter.reset(); try (var ignored = Span.wrap(ambientB).makeCurrent()) { - plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, startTime)); + plugin = started(factory, new InvocationInfo("req-2", ARN, false, startTime)); } plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null)); var secondInvocationTrace = @@ -398,12 +418,12 @@ void executionTrace_isStableAcrossReinvocations_withDifferentAmbientTraces() { @Test void nonTerminalInvocation_doesNotExportWorkflowSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); var spans = spanExporter.getFinishedSpanItems(); - // Only the invocation span is exported; the Workflow span is not ended on non-terminal status. - assertEquals(1, spans.size()); + // Invocation and DurableExecutionRoot are exported; Workflow remains deferred until terminal status. + assertEquals(2, spans.size()); assertEquals("Invocation", spans.get(0).getName()); assertEquals(StatusCode.OK, spans.get(0).getStatus().getStatusCode(), "PENDING invocation span maps to OK"); } @@ -411,7 +431,7 @@ void nonTerminalInvocation_doesNotExportWorkflowSpan() { @Test void workflowSpan_exportedOnceAcrossInvocations_sameSpanId() { // Invocation 1: non-terminal → no Workflow span exported - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); assertTrue( spanExporter.getFinishedSpanItems().stream() @@ -420,7 +440,7 @@ void workflowSpan_exportedOnceAcrossInvocations_sameSpanId() { spanExporter.reset(); // Invocation 2: terminal → Workflow span exported with the deterministic ID - plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, Instant.now())); + plugin = started(factory, new InvocationInfo("req-2", ARN, false, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null)); var workflowSpan = spanByName(spanExporter.getFinishedSpanItems(), "Workflow"); @@ -432,7 +452,7 @@ void workflowSpan_exportedOnceAcrossInvocations_sameSpanId() { @Test void failedInvocation_setsErrorOnBothWorkflowAndInvocationSpans() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onInvocationEnd( new InvocationEndInfo("req-1", ARN, true, InvocationStatus.FAILED, new RuntimeException("boom"))); @@ -444,12 +464,12 @@ void failedInvocation_setsErrorOnBothWorkflowAndInvocationSpans() { @Test void retryingInvocation_invocationSpanUnset_workflowNotExported() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo( "req-1", ARN, true, InvocationStatus.RETRYING, new RuntimeException("transient"))); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(1, spans.size(), "RETRYING is non-terminal — Workflow span not exported"); + assertEquals(2, spans.size(), "RETRYING is non-terminal — Workflow span not exported"); var invocationSpan = spans.get(0); assertEquals("Invocation", invocationSpan.getName()); assertEquals( @@ -462,7 +482,7 @@ void retryingInvocation_invocationSpanUnset_workflowNotExported() { @Test void operationSpan_carriesAttemptNumberAtEnd() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "flaky", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onOperationEnd(new OperationEndInfo( @@ -491,7 +511,7 @@ void operationSpan_carriesAttemptNumberAtEnd() { @Test void continuationOperationSpan_carriesAttemptNumber() { - plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-2", ARN, false, Instant.now())); // No matching onOperationStart in this invocation — continuation branch. plugin.onOperationEnd(new OperationEndInfo( "op-1", @@ -521,7 +541,7 @@ void continuationOperationSpan_carriesAttemptNumber() { void operationSpan_startsAtOperationStartTimestamp() { var opStart = Instant.parse("2026-02-01T10:00:00Z"); var opEnd = Instant.parse("2026-02-01T10:00:03Z"); - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onOperationStart(new OperationInfo("op-1", "step-a", "STEP", "Step", null, opStart, null, null, false)); plugin.onOperationEnd(new OperationEndInfo( "op-1", "step-a", "STEP", "Step", null, opStart, opEnd, "SUCCEEDED", null, false, null, null)); @@ -536,7 +556,7 @@ void operationSpan_startsAtOperationStartTimestamp() { @Test void operationSpan_parentedToWorkflow_linkedToInvocation() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onOperationEnd(new OperationEndInfo( @@ -571,7 +591,7 @@ void operationSpan_parentedToWorkflow_linkedToInvocation() { @Test void attemptSpan_childOfOperation_linkedToInvocation() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "compute", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onUserFunctionStart( @@ -624,7 +644,7 @@ void attemptSpan_childOfOperation_linkedToInvocation() { @Test void attemptSpan_carriesOperationSubtype() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "process-order", "STEP", "Step", null, Instant.now(), false, 1)); plugin.onUserFunctionEnd(new UserFunctionEndInfo( @@ -653,7 +673,7 @@ void attemptSpan_carriesOperationSubtype() { @Test void childOperation_parentedToParentOperationSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onOperationStart(new OperationInfo( "op-parent", "my-context", "CONTEXT", "RunInChildContext", null, Instant.now(), null, null, false)); plugin.onOperationStart(new OperationInfo( @@ -702,7 +722,7 @@ void contextOperation_currentContextCarriesResolvedFlags_withAlwaysOffSampler() // DurableSampler), so descendants inherit the resolved decision. With always_off the resolved decision is // unsampled, so the current context inside the context body must be unsampled — not a provisional sampled bit. var exporter = InMemorySpanExporter.create(); - var offPlugin = new ExecutionOtelPlugin( + var offPluginFactory = ExecutionOtelPlugin.factory( SdkTracerProvider.builder() .setSampler(Sampler.alwaysOff()) .addSpanProcessor(SimpleSpanProcessor.create(exporter)), @@ -712,7 +732,7 @@ void contextOperation_currentContextCarriesResolvedFlags_withAlwaysOffSampler() .workflowSpanName("Workflow") .build()); - offPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var offPlugin = started(offPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now())); offPlugin.onOperationStart( new OperationInfo("ctx-1", "my-ctx", "CONTEXT", "Context", null, Instant.now(), null, null, false)); offPlugin.onUserFunctionStart( @@ -757,7 +777,7 @@ void contextOperation_currentContextCarriesResolvedFlags_withAlwaysOffSampler() @Test void userFunctionFailure_setsErrorOnAttemptSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "failing", "STEP", "Step", null, Instant.now(), false, 1)); plugin.onUserFunctionEnd(new UserFunctionEndInfo( @@ -783,7 +803,7 @@ void userFunctionFailure_setsErrorOnAttemptSpan() { @Test void userFunctionSuccess_setsOkOnAttemptSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "compute", "STEP", "Step", null, Instant.now(), false, 1)); plugin.onUserFunctionEnd(new UserFunctionEndInfo( @@ -809,7 +829,7 @@ void userFunctionSuccess_setsOkOnAttemptSpan() { @Test void userFunctionIncomplete_leavesAttemptSpanUnset() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "waiting", "STEP", "Step", null, Instant.now(), false, 1)); plugin.onUserFunctionEnd(new UserFunctionEndInfo( @@ -834,7 +854,7 @@ void userFunctionIncomplete_leavesAttemptSpanUnset() { @Test void operationSuccess_setsOkOnOperationSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "step-ok", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onOperationEnd(new OperationEndInfo( @@ -861,7 +881,7 @@ void operationEnd_withNonSuccessStatusAndNoError_leavesOperationSpanUnset() { // onOperationEnd fires for every terminal status. A CANCELLED operation (or an error-less // FAILED/TIMED_OUT/STOPPED) carries a non-null, non-SUCCEEDED status with a null error. It must NOT be // stamped OK — the span status stays UNSET. - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-cancel", "step-cancel", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onOperationEnd(new OperationEndInfo( @@ -887,7 +907,7 @@ void operationEnd_withNonSuccessStatusAndNoError_leavesOperationSpanUnset() { void operationEnd_withoutStart_nonSuccessStatusAndNoError_leavesContinuationSpanUnset() { // Same guard on the continuation-span branch (operation completed between invocations): an error-less // TIMED_OUT terminal status must NOT be stamped OK. - plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-2", ARN, false, Instant.now())); plugin.onOperationEnd(new OperationEndInfo( "op-cb-timeout", "my-callback", @@ -911,7 +931,7 @@ void operationEnd_withoutStart_nonSuccessStatusAndNoError_leavesContinuationSpan void operationEnd_withNullStatusAndNoError_setsOkOnOperationSpan() { // A successful statusless virtual (FLAT CONTEXT) operation fires onOperationEnd with a null operation -> // null status and null error. This is genuine success and must be stamped OK. - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-ctx", "my-ctx", "CONTEXT", null, null, Instant.now(), null, null, false)); plugin.onOperationEnd(new OperationEndInfo( @@ -935,16 +955,17 @@ void operationEnd_withNullStatusAndNoError_setsOkOnOperationSpan() { @Test void operationNotCompleted_notEndedAtInvocationEnd() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "my-wait", "WAIT", "Wait", null, Instant.now(), null, null, false)); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); var spans = spanExporter.getFinishedSpanItems(); - // Only the invocation span is exported. A still-open operation has no recording span (creation is deferred to + // Invocation and DurableExecutionRoot are exported. A still-open operation has no recording span (creation is + // deferred to // onOperationEnd), so there is nothing to abandon, and the Workflow span is not exported on a non-terminal // invocation. - assertEquals(1, spans.size()); + assertEquals(2, spans.size()); assertEquals("Invocation", spans.get(0).getName()); assertTrue( spans.stream().noneMatch(s -> s.getName().equals("my-wait")), @@ -955,7 +976,7 @@ void operationNotCompleted_notEndedAtInvocationEnd() { void openAttemptSpan_isEndedAtInvocationEnd_notAbandoned() { // A user function that starts but never ends (e.g. the execution suspends mid-attempt) must not leave a // recording span abandoned: onInvocationEnd force-ends it so it is exported. - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "stuck", "STEP", "Step", null, Instant.now(), false, 1)); // No onUserFunctionEnd — the invocation suspends. @@ -971,7 +992,7 @@ void openAttemptSpan_isEndedAtInvocationEnd_notAbandoned() { @Test void everyRecordingSpanIsEnded_onNonTerminalInvocation() { // No recording span may be left un-ended when the execution returns a non-terminal status. - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onUserFunctionStart( @@ -1006,7 +1027,7 @@ void noRecordingSpanIsLeftOpen_onRetrying_trackedByLifecycleProcessor() { */ private void assertNoOpenSpansOnNonTerminal(InvocationStatus status) { var lifecycle = new LifecycleTrackingSpanProcessor(); - var trackingPlugin = new ExecutionOtelPlugin( + var trackingPluginFactory = ExecutionOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(lifecycle), OtelPluginConfig.builder() .contextExtractor(() -> null) @@ -1014,7 +1035,7 @@ private void assertNoOpenSpansOnNonTerminal(InvocationStatus status) { .workflowSpanName("Workflow") .build()); - trackingPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var trackingPlugin = started(trackingPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now())); trackingPlugin.onOperationStart( new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false)); trackingPlugin.onUserFunctionStart( @@ -1037,7 +1058,7 @@ private void assertNoOpenSpansOnNonTerminal(InvocationStatus status) { @Test void operationOpenedThenCompletedNextInvocation_exportedOnceOnOperationEnd() { // Invocation 1: operation opens but does not complete. - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "my-wait", "WAIT", "Wait", null, Instant.now(), null, null, false)); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.PENDING, null)); @@ -1048,7 +1069,7 @@ void operationOpenedThenCompletedNextInvocation_exportedOnceOnOperationEnd() { spanExporter.reset(); // Invocation 2: the operation completes → materialized once via onOperationEnd, linked to this invocation. - plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, Instant.now())); + plugin = started(factory, new InvocationInfo("req-2", ARN, false, Instant.now())); plugin.onOperationEnd(new OperationEndInfo( "op-1", "my-wait", @@ -1080,7 +1101,7 @@ void operationOpenedThenCompletedNextInvocation_exportedOnceOnOperationEnd() { @Test void executionTraceIsStableAcrossInvocations_andSharedByInvocationSpans() { var executionStartTime = Instant.parse("2026-08-15T00:00:00Z"); - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, executionStartTime)); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, executionStartTime)); plugin.onOperationStart( new OperationInfo("op-1", "step-1", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onOperationEnd(new OperationEndInfo( @@ -1102,7 +1123,7 @@ void executionTraceIsStableAcrossInvocations_andSharedByInvocationSpans() { var firstInvocationTraceId = spanByName(firstSpans, "Invocation").getTraceId(); spanExporter.reset(); - plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, executionStartTime)); + plugin = started(factory, new InvocationInfo("req-2", ARN, false, executionStartTime)); plugin.onInvocationEnd(new InvocationEndInfo("req-2", ARN, false, InvocationStatus.SUCCEEDED, null)); var secondSpans = spanExporter.getFinishedSpanItems(); var workflowSpan = spanByName(secondSpans, "Workflow"); @@ -1116,7 +1137,7 @@ void executionTraceIsStableAcrossInvocations_andSharedByInvocationSpans() { @Test void operationEnd_withoutStart_createsContinuationSpanWithLink() { - plugin.onInvocationStart(new InvocationInfo("req-2", ARN, false, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-2", ARN, false, Instant.now())); // Operation completed between invocations — no matching onOperationStart in this invocation. plugin.onOperationEnd(new OperationEndInfo( "op-wait-1", @@ -1145,7 +1166,7 @@ void operationEnd_withoutStart_createsContinuationSpanWithLink() { @Test void deterministicWorkflowSpanId_stableAcrossInvocations() { - plugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", ARN, true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); var firstWorkflowSpanId = spanByName(spanExporter.getFinishedSpanItems(), "Workflow").getSpanId(); @@ -1153,14 +1174,14 @@ void deterministicWorkflowSpanId_stableAcrossInvocations() { // A second (independent) plugin for the same execution ARN must derive the same Workflow span ID. var exporter2 = InMemorySpanExporter.create(); - var plugin2 = new ExecutionOtelPlugin( + var plugin2Factory = ExecutionOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter2)), OtelPluginConfig.builder() .contextExtractor(() -> null) .enableMdc(false) .workflowSpanName("Workflow") .build()); - plugin2.onInvocationStart(new InvocationInfo("req-9", ARN, true, Instant.now())); + var plugin2 = started(plugin2Factory, new InvocationInfo("req-9", ARN, true, Instant.now())); plugin2.onInvocationEnd(new InvocationEndInfo("req-9", ARN, true, InvocationStatus.SUCCEEDED, null)); var secondWorkflowSpanId = spanByName(exporter2.getFinishedSpanItems(), "Workflow").getSpanId(); @@ -1176,7 +1197,7 @@ void deterministicWorkflowSpanId_stableAcrossInvocations() { @Test void sampling_disabled_producesNoSpans() { var exporter = InMemorySpanExporter.create(); - var sampledPlugin = new ExecutionOtelPlugin( + var sampledPluginFactory = ExecutionOtelPlugin.factory( SdkTracerProvider.builder() .setSampler(io.opentelemetry.sdk.trace.samplers.Sampler.alwaysOff()) .addSpanProcessor(SimpleSpanProcessor.create(exporter)), @@ -1185,7 +1206,7 @@ void sampling_disabled_producesNoSpans() { .enableMdc(false) .workflowSpanName("Workflow") .build()); - sampledPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var sampledPlugin = started(sampledPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now())); sampledPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); assertTrue(exporter.getFinishedSpanItems().isEmpty(), "No spans should be exported with 0% sampling"); } @@ -1200,7 +1221,7 @@ void xrayExtraction_undecidedSampling_remoteParentIsAncestor_flagUnset() { // Two-arg context → UNDECIDED sampling: the valid remote parent is still the authoritative ancestor. A // non-parent-based alwaysOn sampler exports the spans so the topology is observable (a plain parent-based // sampler would drop them, since the remote parent's sampled flag is left unset). - var xrayPlugin = new ExecutionOtelPlugin( + var xrayPluginFactory = ExecutionOtelPlugin.factory( SdkTracerProvider.builder() .setSampler(Sampler.alwaysOn()) .addSpanProcessor(SimpleSpanProcessor.create(exporter)), @@ -1209,7 +1230,7 @@ void xrayExtraction_undecidedSampling_remoteParentIsAncestor_flagUnset() { .enableMdc(false) .workflowSpanName("Workflow") .build()); - xrayPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now())); xrayPlugin.onOperationStart( new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false)); xrayPlugin.onOperationEnd(new OperationEndInfo( @@ -1249,7 +1270,7 @@ void xrayExtraction_undecidedSampling_parentBasedSampler_defersToSamplerAndExpor var xrayTraceId = "aabbccddee112233445566778899aabb"; var parentSpanId = "53995c3f42cd8ad8"; var exporter = InMemorySpanExporter.create(); - var xrayPlugin = new ExecutionOtelPlugin( + var xrayPluginFactory = ExecutionOtelPlugin.factory( SdkTracerProvider.builder() .setSampler(Sampler.parentBased(Sampler.alwaysOn())) .addSpanProcessor(SimpleSpanProcessor.create(exporter)), @@ -1258,7 +1279,7 @@ void xrayExtraction_undecidedSampling_parentBasedSampler_defersToSamplerAndExpor .enableMdc(false) .workflowSpanName("Workflow") .build()); - xrayPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now())); xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); var spans = exporter.getFinishedSpanItems(); @@ -1279,7 +1300,7 @@ void xrayExtraction_undecidedSampling_parentBasedNeverSampler_dropsExecutionTrac var xrayTraceId = "aabbccddee112233445566778899aabb"; var parentSpanId = "53995c3f42cd8ad8"; var exporter = InMemorySpanExporter.create(); - var xrayPlugin = new ExecutionOtelPlugin( + var xrayPluginFactory = ExecutionOtelPlugin.factory( SdkTracerProvider.builder() .setSampler(Sampler.parentBased(Sampler.alwaysOff())) .addSpanProcessor(SimpleSpanProcessor.create(exporter)), @@ -1288,7 +1309,7 @@ void xrayExtraction_undecidedSampling_parentBasedNeverSampler_dropsExecutionTrac .enableMdc(false) .workflowSpanName("Workflow") .build()); - xrayPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now())); xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); assertTrue( @@ -1302,7 +1323,7 @@ void xrayExtraction_explicitSampled_remoteParentIsExecutionAncestor() { var parentSpanId = "53995c3f42cd8ad8"; var exporter = InMemorySpanExporter.create(); // Explicit Sampled=1 with a complete parent → the remote context is the execution ancestor directly. - var xrayPlugin = new ExecutionOtelPlugin( + var xrayPluginFactory = ExecutionOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), OtelPluginConfig.builder() .contextExtractor(() -> @@ -1311,7 +1332,7 @@ void xrayExtraction_explicitSampled_remoteParentIsExecutionAncestor() { .workflowSpanName("Workflow") .build()); - xrayPlugin.onInvocationStart(new InvocationInfo("req-1", ARN, true, Instant.now())); + var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", ARN, true, Instant.now())); xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", ARN, true, InvocationStatus.SUCCEEDED, null)); var spans = exporter.getFinishedSpanItems(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java new file mode 100644 index 000000000..7f6ebd1f2 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ExecutionRootTest.java @@ -0,0 +1,362 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_ARN; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_STATUS; +import static software.amazon.lambda.durable.otel.SpanAttributes.DURABLE_EXECUTION_SYNTHETIC_ROOT; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class ExecutionRootTest { + private static final Instant START = Instant.parse("2026-10-02T00:00:00Z"); + private static final String TRACE = "6955b900123456789012345678901234"; + private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id"; + + @ParameterizedTest + @CsvSource({"true,true", "true,false", "false,true", "false,false"}) + void anchorIsExportedOnFirstPendingInvocationThenConnectsTerminalWorkflow(boolean executionView, boolean success) { + var exporter = InMemorySpanExporter.create(); + var plugin = plugin(executionView, exporter, null, Sampler.alwaysOn()); + var effects = new AtomicInteger(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> { + ctx.step("once", Integer.class, stepCtx -> effects.incrementAndGet()); + ctx.wait("pause", Duration.ofSeconds(10)); + if (!success) throw new IllegalArgumentException("expected"); + return input; + }, + DurableConfig.builder().withPlugins(plugin).build()); + assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); + var first = exporter.getFinishedSpanItems(); + var anchor = named(first, "DurableExecutionRoot"); + assertEquals(true, anchor.getAttributes().get(DURABLE_EXECUTION_SYNTHETIC_ROOT)); + assertFalse(anchor.getParentSpanContext().isValid()); + assertEquals(anchor.getStartEpochNanos(), anchor.getEndEpochNanos()); + assertEquals(StatusCode.UNSET, anchor.getStatus().getStatusCode()); + assertNull(anchor.getAttributes().get(DURABLE_EXECUTION_STATUS)); + assertTrue(first.stream().noneMatch(span -> span.getName().equals("Workflow"))); + assertEquals(anchor.getSpanId(), named(first, "Invocation").getParentSpanId()); + runner.advanceTime(); + assertEquals( + success ? ExecutionStatus.SUCCEEDED : ExecutionStatus.FAILED, + runner.runUntilComplete("input").getStatus()); + assertEquals(1, effects.get()); + var all = exporter.getFinishedSpanItems(); + assertEquals(anchor.getSpanId(), named(all, "Workflow").getParentSpanId()); + var anchors = all.stream() + .filter(span -> span.getName().equals("DurableExecutionRoot")) + .toList(); + assertEquals(2, anchors.size()); + assertStable(anchor, anchors.get(1)); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void fallbackRecoveryKeepsIdentityAndRemoteParentsRemainExternal(boolean executionView) { + var cases = new ExtractedContext[] { + null, + new ExtractedContext("invalid", "bad", ExtractedContext.Sampling.UNDECIDED), + new ExtractedContext(TRACE, null, ExtractedContext.Sampling.SAMPLED), + new ExtractedContext(TRACE, "0000000000000000", ExtractedContext.Sampling.SAMPLED) + }; + for (var extracted : cases) { + var exporter = InMemorySpanExporter.create(); + var plugin = plugin(executionView, exporter, extracted, Sampler.alwaysOn()); + invoke(plugin, ARN, true, InvocationStatus.RETRYING); + var root = named(exporter.getFinishedSpanItems(), "DurableExecutionRoot"); + invoke(plugin, ARN, true, InvocationStatus.PENDING); // redelivery after a failed invocation + invoke(plugin, ARN, false, InvocationStatus.SUCCEEDED); + var roots = exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("DurableExecutionRoot")) + .toList(); + assertEquals(3, roots.size()); + roots.forEach(span -> assertStable(root, span)); + assertEquals(new DeterministicIdGenerator().generateExecutionRootSpanId(ARN), root.getSpanId()); + assertEquals(START.getEpochSecond() * 1_000_000_000L, root.getStartEpochNanos()); + if (extracted != null && extracted.hasValidTraceId()) assertEquals(TRACE, root.getTraceId()); + } + var exporter = InMemorySpanExporter.create(); + var plugin = plugin( + executionView, + exporter, + new ExtractedContext(TRACE, "1234567890123456", ExtractedContext.Sampling.SAMPLED), + Sampler.alwaysOn()); + invoke(plugin, ARN, true, InvocationStatus.SUCCEEDED); + assertEquals(2, exporter.getFinishedSpanItems().size()); + assertTrue(exporter.getFinishedSpanItems().stream() + .allMatch(span -> span.getParentSpanId().equals("1234567890123456"))); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void anchorsHonorSamplingAndAreOwnedByExecutionOnSharedTrace(boolean executionView) { + for (var extracted : + new ExtractedContext[] {null, new ExtractedContext(TRACE, null, ExtractedContext.Sampling.NOT_SAMPLED) + }) { + var exporter = InMemorySpanExporter.create(); + var plugin = plugin( + executionView, exporter, extracted, extracted == null ? Sampler.alwaysOff() : Sampler.alwaysOn()); + invoke(plugin, ARN, true, InvocationStatus.PENDING); + invoke(plugin, ARN, false, InvocationStatus.SUCCEEDED); + assertTrue(exporter.getFinishedSpanItems().isEmpty()); + } + var exporter = InMemorySpanExporter.create(); + var extracted = new ExtractedContext(TRACE, null, ExtractedContext.Sampling.SAMPLED); + for (var service : List.of("caller", "callee")) { + var builder = SdkTracerProvider.builder() + .setResource(Resource.create(Attributes.of(AttributeKey.stringKey("service.name"), service))) + .setSampler(Sampler.alwaysOff()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var plugin = plugin(executionView, builder, extracted); + var start = service.equals("caller") ? START : START.plusSeconds(10); + invoke(plugin, ARN + "-" + service, true, InvocationStatus.SUCCEEDED, start); + } + var roots = exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("DurableExecutionRoot")) + .toList(); + assertEquals(2, roots.size()); + assertTrue(roots.stream().allMatch(span -> span.getTraceId().equals(TRACE))); + assertNotEquals(roots.get(0).getSpanId(), roots.get(1).getSpanId()); + assertNotEquals( + roots.get(0).getAttributes().get(DURABLE_EXECUTION_ARN), + roots.get(1).getAttributes().get(DURABLE_EXECUTION_ARN)); + assertNotEquals(roots.get(0).getStartEpochNanos(), roots.get(1).getStartEpochNanos()); + assertNotEquals(roots.get(0).getResource(), roots.get(1).getResource()); + for (var root : roots) { + var workflow = exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("Workflow")) + .filter(span -> span.getAttributes() + .get(DURABLE_EXECUTION_ARN) + .equals(root.getAttributes().get(DURABLE_EXECUTION_ARN))) + .findFirst() + .orElseThrow(); + assertEquals(root.getSpanContext(), workflow.getParentSpanContext()); + } + } + + @ParameterizedTest + @CsvSource({"true,true", "true,false", "false,true", "false,false"}) + void failedFirstFlushIsRecoveredInAnotherEnvironment(boolean executionView, boolean success) { + var first = failedSuspendedAnchor(executionView); + var exporter = InMemorySpanExporter.create(); + var builder = SdkTracerProvider.builder() + .setResource(environmentResource("resumed")) + .setSampler(Sampler.alwaysOn()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)); + invoke( + plugin(executionView, builder, null), + ARN, + false, + success ? InvocationStatus.SUCCEEDED : InvocationStatus.FAILED); + var recovered = named(exporter.getFinishedSpanItems(), "DurableExecutionRoot"); + assertStableSpanFields(first, recovered); + assertNotEquals(first.getResource(), recovered.getResource()); + assertEquals( + recovered.getSpanContext(), + named(exporter.getFinishedSpanItems(), "Workflow").getParentSpanContext()); + } + + private static SpanData failedSuspendedAnchor(boolean executionView) { + var attempted = new ArrayList(); + try (var processor = BatchSpanProcessor.builder(failingExporter(attempted)) + .setScheduleDelay(Duration.ofDays(1)) + .build()) { + var builder = SdkTracerProvider.builder() + .setResource(environmentResource("first")) + .setSampler(Sampler.alwaysOn()) + .addSpanProcessor(processor); + invoke(plugin(executionView, builder, null), ARN, true, InvocationStatus.PENDING); + // The end hook's forceFlush attempts the anchor export before returning, even without a terminal hook. + assertTrue(attempted.stream().noneMatch(span -> span.getName().equals("Workflow"))); + assertEquals( + START.getEpochSecond() * 1_000_000_000L, + named(attempted, "DurableExecutionRoot").getStartEpochNanos()); + } + return named(attempted, "DurableExecutionRoot"); + } + + private static SpanExporter failingExporter(List attempted) { + return new SpanExporter() { + @Override + public CompletableResultCode export(Collection spans) { + attempted.addAll(spans); + return CompletableResultCode.ofFailure(); + } + + @Override + public CompletableResultCode flush() { + return CompletableResultCode.ofFailure(); + } + + @Override + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } + }; + } + + private static Resource environmentResource(String instance) { + return Resource.create(Attributes.of(AttributeKey.stringKey("faas.instance"), instance)); + } + + @ParameterizedTest + @CsvSource({"true,true", "true,false", "false,true", "false,false"}) + void processorSpansDoNotInheritDurableSamplingOverride(boolean executionView, boolean observeStart) { + var exporter = InMemorySpanExporter.create(); + var observed = new AtomicReference(); + try (var unrelated = SdkTracerProvider.builder() + .setSampler(DurableSampler.wrap(Sampler.alwaysOff())) + .build()) { + var builder = SdkTracerProvider.builder() + .setSampler(Sampler.alwaysOff()) + .addSpanProcessor(new SamplingObserver(unrelated, observed, observeStart)) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var config = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(() -> new ExtractedContext(TRACE, null, ExtractedContext.Sampling.SAMPLED)) + .build(); + DurableExecutionPluginFactory plugin = executionView + ? ExecutionOtelPlugin.factory(builder, config) + : InvocationOtelPlugin.factory(builder, config); + invoke(plugin, ARN, true, InvocationStatus.PENDING); + assertTrue(named(exporter.getFinishedSpanItems(), "DurableExecutionRoot") + .getSpanContext() + .isSampled()); + assertNotNull(observed.get(), "Root processor callback must have run"); + assertFalse(observed.get().isSampled(), "Processor's unrelated span must retain its always-off policy"); + } + } + + private record SamplingObserver( + SdkTracerProvider provider, AtomicReference observed, boolean observeStart) + implements SpanProcessor { + @Override + public void onStart(Context parent, ReadWriteSpan span) { + if (observeStart) { + observe(span); + } + } + + @Override + public boolean isStartRequired() { + return observeStart; + } + + @Override + public void onEnd(ReadableSpan span) { + if (!observeStart) { + observe(span); + } + } + + private void observe(ReadableSpan span) { + if (span.getName().equals("DurableExecutionRoot")) { + var callback = + provider.get("processor").spanBuilder("unrelated").startSpan(); + observed.set(callback.getSpanContext()); + callback.end(); + } + } + + @Override + public boolean isEndRequired() { + return !observeStart; + } + } + + private static void assertStable(SpanData first, SpanData replay) { + assertStableSpanFields(first, replay); + assertEquals(first.getResource(), replay.getResource()); + } + + private static void assertStableSpanFields(SpanData first, SpanData replay) { + assertEquals(first.getSpanContext(), replay.getSpanContext()); + assertEquals(first.getParentSpanContext(), replay.getParentSpanContext()); + assertEquals(first.getName(), replay.getName()); + assertEquals(first.getKind(), replay.getKind()); + assertEquals(first.getStartEpochNanos(), replay.getStartEpochNanos()); + assertEquals(first.getEndEpochNanos(), replay.getEndEpochNanos()); + assertEquals(first.getAttributes(), replay.getAttributes()); + assertEquals(first.getStatus(), replay.getStatus()); + assertEquals(first.getEvents(), replay.getEvents()); + assertEquals(first.getLinks(), replay.getLinks()); + assertEquals(first.hasEnded(), replay.hasEnded()); + assertEquals(first.getTotalAttributeCount(), replay.getTotalAttributeCount()); + assertEquals(first.getTotalRecordedEvents(), replay.getTotalRecordedEvents()); + assertEquals(first.getTotalRecordedLinks(), replay.getTotalRecordedLinks()); + assertEquals(first.getInstrumentationScopeInfo(), replay.getInstrumentationScopeInfo()); + } + + private static SpanData named(List spans, String name) { + return spans.stream() + .filter(span -> span.getName().equals(name)) + .findFirst() + .orElseThrow(); + } + + private static void invoke( + DurableExecutionPluginFactory plugin, String arn, boolean first, InvocationStatus status) { + invoke(plugin, arn, first, status, START); + } + + private static void invoke( + DurableExecutionPluginFactory plugin, String arn, boolean first, InvocationStatus status, Instant start) { + var instance = plugin.createPlugin(new InvocationInfo("request", arn, first, start)); + instance.onInvocationEnd(new InvocationEndInfo("request", arn, first, status, null)); + } + + private static DurableExecutionPluginFactory plugin( + boolean executionView, InMemorySpanExporter exporter, ExtractedContext extracted, Sampler sampler) { + var builder = + SdkTracerProvider.builder().setSampler(sampler).addSpanProcessor(SimpleSpanProcessor.create(exporter)); + return plugin(executionView, builder, extracted); + } + + private static DurableExecutionPluginFactory plugin( + boolean executionView, SdkTracerProviderBuilder builder, ExtractedContext extracted) { + var config = OtelPluginConfig.builder() + .contextExtractor(() -> extracted) + .enableMdc(false) + .build(); + return executionView + ? ExecutionOtelPlugin.factory(builder, config) + : InvocationOtelPlugin.factory(builder, config); + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerContextIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerContextIntegrationTest.java new file mode 100644 index 000000000..e69f1a6ce --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerContextIntegrationTest.java @@ -0,0 +1,268 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.slf4j.MDC; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.PluginRunner; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class HandlerContextIntegrationTest { + private static final String TRACE_ID = "12345678901234567890123456789012"; + + @ParameterizedTest + @CsvSource({"true,success", "false,success", "true,failure", "false,failure", "true,suspension", "false,suspension" + }) + void reusedHandlerWorkerHasNoLeakedMdc(boolean executionView, String outcome) throws Exception { + var pluginConfig = OtelPluginConfig.builder() + .enableMdc(true) + .contextExtractor( + () -> new ExtractedContext(TRACE_ID, "1234567890123456", ExtractedContext.Sampling.SAMPLED)) + .build(); + DurableExecutionPluginFactory plugin = executionView + ? ExecutionOtelPlugin.factory(SdkTracerProvider.builder(), pluginConfig) + : InvocationOtelPlugin.factory(SdkTracerProvider.builder(), pluginConfig); + var executor = Executors.newSingleThreadExecutor(); + var config = DurableConfig.builder() + .withExecutorService(executor) + .withPlugins(plugin) + .build(); + try { + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> { + assertEquals(TRACE_ID, MDC.get(MdcSpanEnricher.MDC_TRACE_ID)); + if (outcome.equals("failure")) throw new IllegalStateException("body"); + if (outcome.equals("suspension")) ctx.wait("pause", Duration.ofSeconds(1)); + return "done"; + }, + config); + var result = runner.run("input"); + assertEquals( + switch (outcome) { + case "failure" -> ExecutionStatus.FAILED; + case "suspension" -> ExecutionStatus.PENDING; + default -> ExecutionStatus.SUCCEEDED; + }, + result.getStatus()); + executor.submit(() -> { + assertNull( + MDC.get(MdcSpanEnricher.MDC_TRACE_ID), + "a reused worker must not inherit the prior trace"); + assertNull(MDC.get(MdcSpanEnricher.MDC_SPAN_ID)); + assertNull(MDC.get(MdcSpanEnricher.MDC_TRACE_SAMPLED)); + }) + .get(2, TimeUnit.SECONDS); + } finally { + executor.shutdownNow(); + } + } + + @ParameterizedTest + @CsvSource({"true,success", "false,success", "true,failure", "false,failure", "true,suspension", "false,suspension" + }) + void reusedHandlerWorkerHasNoLeakedScope(boolean executionView, String outcome) throws Exception { + var exporter = InMemorySpanExporter.create(); + var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var pluginConfig = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor( + () -> new ExtractedContext(TRACE_ID, "1234567890123456", ExtractedContext.Sampling.SAMPLED)) + .build(); + DurableExecutionPluginFactory factory = executionView + ? ExecutionOtelPlugin.factory(builder, pluginConfig) + : InvocationOtelPlugin.factory(builder, pluginConfig); + var executor = Executors.newSingleThreadExecutor(); + var config = DurableConfig.builder() + .withExecutorService(executor) + .withPlugins(factory) + .build(); + try { + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> { + assertEquals(TRACE_ID, Span.current().getSpanContext().getTraceId()); + if (outcome.equals("failure")) throw new IllegalStateException("user failure"); + if (outcome.equals("suspension")) ctx.wait("resume", Duration.ofSeconds(1)); + return "done"; + }, + config); + var result = runner.run("input"); + assertEquals( + switch (outcome) { + case "failure" -> ExecutionStatus.FAILED; + case "suspension" -> ExecutionStatus.PENDING; + default -> ExecutionStatus.SUCCEEDED; + }, + result.getStatus()); + assertFalse( + executor.submit(() -> Span.current().getSpanContext().isValid()) + .get(5, TimeUnit.SECONDS), + "cleanup must run on the worker, even if invocation finalization ran on another thread"); + } finally { + executor.shutdownNow(); + } + } + + @ParameterizedTest + @CsvSource({"true,true", "false,true", "true,false", "false,false"}) + void preservesCompatibleAmbientAndRestoresUnrelatedAmbientAfterFailure(boolean executionView, boolean sameTrace) { + var exporter = InMemorySpanExporter.create(); + var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var pluginConfig = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor( + () -> new ExtractedContext(TRACE_ID, "1234567890123456", ExtractedContext.Sampling.SAMPLED)) + .build(); + DurableExecutionPluginFactory factory = executionView + ? ExecutionOtelPlugin.factory(builder, pluginConfig) + : InvocationOtelPlugin.factory(builder, pluginConfig); + var ambient = SpanContext.create( + sameTrace ? TRACE_ID : "abcdefabcdefabcdefabcdefabcdefab", + "abcdefabcdefabcd", + TraceFlags.getSampled(), + TraceState.getDefault()); + var previous = Context.current(); + try (var ignored = Span.wrap(ambient).makeCurrent()) { + var runner = new PluginRunner(List.of(factory)); + runner.onInvocationStart(new InvocationInfo( + "req", + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/name/id", + true, + Instant.now())); + var error = new IllegalStateException("handler failure"); + assertSame( + error, + assertThrows( + IllegalStateException.class, + () -> runner.runHandler(() -> { + var active = Span.current().getSpanContext(); + assertEquals(TRACE_ID, active.getTraceId()); + if (sameTrace) assertEquals(ambient, active); + else assertNotEquals(ambient.getSpanId(), active.getSpanId()); + throw error; + }))); + assertEquals(ambient, Span.current().getSpanContext()); + } + assertSame(previous, Context.current()); + } + + @ParameterizedTest + @CsvSource({"true,true", "false,true", "true,false", "false,false"}) + void rootContextIsValidAndRestoredAcrossNestedWorkAndResume(boolean executionView, boolean sampled) { + var exporter = InMemorySpanExporter.create(); + var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var pluginConfig = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(() -> new ExtractedContext( + TRACE_ID, + "1234567890123456", + sampled ? ExtractedContext.Sampling.SAMPLED : ExtractedContext.Sampling.NOT_SAMPLED)) + .build(); + DurableExecutionPluginFactory factory = executionView + ? ExecutionOtelPlugin.factory(builder, pluginConfig) + : InvocationOtelPlugin.factory(builder, pluginConfig); + var executor = Executors.newCachedThreadPool(); + var config = DurableConfig.builder() + .withExecutorService(executor) + .withPlugins(factory) + .build(); + var userProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build(); + var userTracer = userProvider.get("user"); + var roots = new ArrayList(); + var bodyCalls = new AtomicInteger(); + try { + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> { + var root = Span.current().getSpanContext(); + assertTrue(root.isValid(), "root handler must have a valid context without an agent"); + assertEquals(TRACE_ID, root.getTraceId()); + assertEquals(sampled, root.isSampled()); + roots.add(root); + userTracer.spanBuilder("user-handler").startSpan().end(); + ctx.step("before", String.class, step -> { + bodyCalls.incrementAndGet(); + assertNotEquals( + root.getSpanId(), + Span.current().getSpanContext().getSpanId()); + return "before"; + }); + assertEquals(root, Span.current().getSpanContext()); + ctx.runInChildContext("child", String.class, child -> { + assertNotEquals( + root.getSpanId(), + Span.current().getSpanContext().getSpanId()); + return "child"; + }); + assertEquals(root, Span.current().getSpanContext()); + userTracer + .spanBuilder("user-handler-restored") + .startSpan() + .end(); + ctx.wait("resume", Duration.ofSeconds(1)); + assertEquals(root, Span.current().getSpanContext()); + userTracer + .spanBuilder("user-handler-after-resume") + .startSpan() + .end(); + return "done"; + }, + config); + var first = runner.run("input"); + assertEquals(ExecutionStatus.PENDING, first.getStatus()); + runner.advanceTime(); + var last = runner.run("input"); + assertEquals(ExecutionStatus.SUCCEEDED, last.getStatus()); + assertEquals(2, roots.size()); + assertEquals(1, bodyCalls.get()); + assertFalse(Span.current().getSpanContext().isValid()); + if (sampled) { + var rootName = executionView ? "Workflow" : "Invocation"; + var spans = exporter.getFinishedSpanItems(); + var userSpans = spans.stream() + .filter(span -> span.getName().startsWith("user-handler")) + .toList(); + assertEquals(5, userSpans.size()); + for (var span : userSpans) { + assertEquals(TRACE_ID, span.getTraceId()); + assertTrue(roots.stream().anyMatch(root -> root.getSpanId().equals(span.getParentSpanId()))); + } + for (var root : roots) { + assertTrue(spans.stream() + .anyMatch(s -> rootName.equals(s.getName()) + && root.getSpanId().equals(s.getSpanId()))); + } + } else { + assertTrue(exporter.getFinishedSpanItems().isEmpty()); + } + } finally { + executor.shutdownNow(); + userProvider.close(); + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerMdcIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerMdcIntegrationTest.java new file mode 100644 index 000000000..237e859af --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerMdcIntegrationTest.java @@ -0,0 +1,185 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.MDC; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class HandlerMdcIntegrationTest { + private static final String TRACE_ID = "12345678901234567890123456789012"; + + @ParameterizedTest + @ValueSource(strings = {"success", "failure", "suspension", "inputFailure"}) + void noPluginPathRetainsExistingMdcBehavior(String outcome) throws Exception { + var executor = Executors.newSingleThreadExecutor(); + var callerBefore = MDC.getCopyOfContextMap(); + var callerMdc = Map.of(MdcSpanEnricher.MDC_TRACE_ID, "caller-trace", "caller", "retained"); + var workerMdc = ambientMdc(); + try { + executor.submit(() -> MDC.setContextMap(workerMdc)).get(5, TimeUnit.SECONDS); + MDC.setContextMap(callerMdc); + runWithoutPlugins(executor, outcome); + // The existing factory worker always clears MDC, including startup/input failures. + var expectedWorker = Map.of(); + assertRestored(executor, expectedWorker, callerMdc); + } finally { + executor.shutdownNow(); + if (callerBefore == null) MDC.clear(); + else MDC.setContextMap(callerBefore); + } + } + + private static void runWithoutPlugins(ExecutorService executor, String outcome) { + var config = DurableConfig.builder() + .withExecutorService(executor) + .withSerDes(outcome.equals("inputFailure") ? failingInputSerDes() : new JacksonSerDes()) + .build(); + assertTrue(config.getPluginFactories().isEmpty()); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> { + if (outcome.equals("failure")) throw new IllegalStateException("handler failure"); + if (outcome.equals("suspension")) ctx.wait("pause", Duration.ofSeconds(1)); + return "done"; + }, + config); + var expected = + switch (outcome) { + case "failure", "inputFailure" -> ExecutionStatus.FAILED; + case "suspension" -> ExecutionStatus.PENDING; + default -> ExecutionStatus.SUCCEEDED; + }; + assertEquals(expected, runner.run("input").getStatus()); + } + + @ParameterizedTest + @CsvSource({ + "true,success,false", "false,success,false", "true,failure,false", "false,failure,false", + "true,suspension,false", "false,suspension,false", "true,inputFailure,false", "false,inputFailure,false", + "true,success,true", "false,success,true", "true,failure,true", "false,failure,true", + "true,suspension,true", "false,suspension,true", "true,inputFailure,true", "false,inputFailure,true" + }) + void preservesCallerAndReusedWorkerMdc(boolean executionView, String outcome, boolean ambient) throws Exception { + var executor = Executors.newSingleThreadExecutor(); + var callerBefore = MDC.getCopyOfContextMap(); + var callerMdc = Map.of(MdcSpanEnricher.MDC_TRACE_ID, "caller-trace", "caller", "retained"); + var workerMdc = ambient ? ambientMdc() : Map.of(); + try { + executor.submit(() -> MDC.setContextMap(workerMdc)).get(5, TimeUnit.SECONDS); + MDC.setContextMap(callerMdc); + runInvocations(executor, executionView, outcome, workerMdc, callerMdc); + } finally { + executor.shutdownNow(); + if (callerBefore == null) MDC.clear(); + else MDC.setContextMap(callerBefore); + } + } + + private static Map ambientMdc() { + return Map.of( + MdcSpanEnricher.MDC_TRACE_ID, + "worker-trace", + MdcSpanEnricher.MDC_SPAN_ID, + "worker-span", + MdcSpanEnricher.MDC_TRACE_SAMPLED, + "worker-sampled", + "application", + "retained"); + } + + private static void runInvocations( + ExecutorService executor, + boolean executionView, + String outcome, + Map workerMdc, + Map callerMdc) + throws Exception { + var calls = new AtomicInteger(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, ctx) -> { + calls.incrementAndGet(); + assertEquals(TRACE_ID, MDC.get(MdcSpanEnricher.MDC_TRACE_ID)); + if (outcome.equals("failure")) throw new IllegalStateException("handler failure"); + if (outcome.equals("suspension")) ctx.wait("resume", Duration.ofSeconds(1)); + return "done"; + }, + config(executor, executionView, outcome)); + var first = runner.run("input"); + assertEquals( + switch (outcome) { + case "failure", "inputFailure" -> ExecutionStatus.FAILED; + case "suspension" -> ExecutionStatus.PENDING; + default -> ExecutionStatus.SUCCEEDED; + }, + first.getStatus()); + assertEquals(outcome.equals("inputFailure") ? 0 : 1, calls.get()); + assertRestored(executor, workerMdc, callerMdc); + if (outcome.equals("suspension")) { + runner.advanceTime(); + assertEquals(ExecutionStatus.SUCCEEDED, runner.run("input").getStatus()); + assertEquals(2, calls.get()); + assertRestored(executor, workerMdc, callerMdc); + } + } + + private static DurableConfig config(ExecutorService executor, boolean executionView, String outcome) { + var settings = OtelPluginConfig.builder() + .enableMdc(true) + .contextExtractor( + () -> new ExtractedContext(TRACE_ID, "1234567890123456", ExtractedContext.Sampling.SAMPLED)) + .build(); + DurableExecutionPluginFactory plugin = executionView + ? ExecutionOtelPlugin.factory(SdkTracerProvider.builder(), settings) + : InvocationOtelPlugin.factory(SdkTracerProvider.builder(), settings); + return DurableConfig.builder() + .withExecutorService(executor) + .withPlugins(plugin) + .withSerDes(outcome.equals("inputFailure") ? failingInputSerDes() : new JacksonSerDes()) + .build(); + } + + private static SerDes failingInputSerDes() { + return new SerDes() { + private final JacksonSerDes delegate = new JacksonSerDes(); + + @Override + public String serialize(Object value) { + return delegate.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken type) { + if ("\"input\"".equals(data)) throw new IllegalStateException("input failure"); + return delegate.deserialize(data, type); + } + }; + } + + private static void assertRestored( + ExecutorService executor, Map workerMdc, Map callerMdc) throws Exception { + var callerAfter = MDC.getCopyOfContextMap(); + var after = executor.submit(MDC::getCopyOfContextMap).get(5, TimeUnit.SECONDS); + assertAll( + () -> assertEquals(callerMdc, callerAfter, "finalization must preserve caller MDC"), + () -> assertEquals(workerMdc, after == null ? Map.of() : after, "worker must regain its original MDC")); + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerScopeFinalizationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerScopeFinalizationTest.java new file mode 100644 index 000000000..6fb8edda0 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerScopeFinalizationTest.java @@ -0,0 +1,456 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import com.amazonaws.services.lambda.runtime.Context; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import java.lang.reflect.Proxy; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.function.Function; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.DurableExecutionOutput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.HandlerScoped; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +class HandlerScopeFinalizationTest { + @ParameterizedTest + @CsvSource({ + "false,false,false", + "false,false,true", + "false,true,false", + "false,true,true", + "true,false,false", + "true,false,true", + "true,true,false", + "true,true,true" + }) + void preservesTimeForRealOtelFinalization(boolean executionView, boolean retry, boolean hasScope) throws Exception { + var deadline = new Deadline(); + var clock = Executors.newSingleThreadScheduledExecutor(r -> daemon(r, "flush-clock")); + var flush = new DelayedFlush(deadline, clock); + var builder = SdkTracerProvider.builder().addSpanProcessor(flush); + var settings = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(() -> new ExtractedContext( + "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED)) + .build(); + var factory = executionView + ? ExecutionOtelPlugin.factory(builder, settings) + : InvocationOtelPlugin.factory(builder, settings); + DurableExecutionPluginFactory plugin = info -> { + var delegate = factory.createPlugin(info); + return new ScopedPlugin() { + @Override + public AutoCloseable openHandlerScope() { + if (!hasScope) return null; + if (delegate instanceof InvocationOtelPlugin invocation) + return new InvocationOtelPlugin.HandlerScopeOpener().apply(invocation); + return new ExecutionOtelPlugin.HandlerScopeOpener().apply((ExecutionOtelPlugin) delegate); + } + + @Override + public void onInvocationStart(InvocationInfo invocation) { + delegate.onInvocationStart(invocation); + } + + @Override + public void onInvocationEnd(InvocationEndInfo invocation) { + delegate.onInvocationEnd(invocation); + } + }; + }; + var workers = Executors.newCachedThreadPool(r -> daemon(r, "handler-owner")); + var callers = Executors.newSingleThreadExecutor(r -> daemon(r, "invocation-caller")); + var enteredFinally = new CountDownLatch(1); + var releaseFinally = new CountDownLatch(1); + var original = retryError(); + var cfg = config(workers, plugin); + try { + var response = callers.submit(() -> DurableExecutor.execute( + input(), + deadline.context(), + TypeToken.get(String.class), + (value, ctx) -> { + deadline.arm(400); + try { + if (retry) + ctx.step("retry", String.class, step -> { + throw original; + }); + else ctx.wait("pause", Duration.ofSeconds(1)); + return "done"; + } finally { + enteredFinally.countDown(); + await(releaseFinally); + } + }, + cfg)); + check(enteredFinally.await(2, TimeUnit.SECONDS), "handler finally not reached"); + outcome(response, retry, original); + long total = deadline.elapsedMillis(); + check(flush.calls.get() == 1, "expected one actual bundled OTel forceFlush"); + check(flush.flushMillis.get() >= 240, "actual bundled OTel join did not await forceFlush result"); + check(total < 400, "unexpected response/deadline relation: " + total); + check(releaseFinally.getCount() == 1, "test cleanup was not blocked"); + + } finally { + releaseFinally.countDown(); + workers.shutdown(); + workers.awaitTermination(2, TimeUnit.SECONDS); + callers.shutdownNow(); + clock.shutdownNow(); + } + } + + @SuppressWarnings("removal") + @ParameterizedTest + @CsvSource({ + "false,false,false", + "false,false,true", + "false,true,false", + "false,true,true", + "true,false,false", + "true,false,true", + "true,true,false", + "true,true,true" + }) + void scopeFatal(boolean retry, boolean wrapped, boolean threadDeath) throws Exception { + scopeFatal(retry, wrapped, threadDeath ? new ThreadDeath() : new InternalError("scope fatal"), false); + } + + @Test + void losingBodyFinallyFatalPreservesTheOriginalWinner() throws Exception { + scopeFatal(false, false, new InternalError("body finally fatal"), true); + } + + @Test + void fatalAfterTheHandoffTimeoutCannotChangeAnAlreadyReturnedOutcome() throws Exception { + scopeFatal(false, false, new InternalError("late scope fatal"), false, true); + } + + @SuppressWarnings("removal") + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void lateScopeFatalAfterFinalizationPreservesCallerOutcomeAndEscapesOwner(boolean threadDeath) throws Exception { + var releaseScope = new CountDownLatch(1); + var ownerFinished = new CountDownLatch(1); + var shutdownEntered = new AtomicBoolean(); + var ended = new AtomicBoolean(); + var endCalls = new AtomicInteger(); + var endInfo = new AtomicReference(); + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("scope failure during manager shutdown"); + var ownerFatal = new AtomicReference(); + var fatalEscaped = new CountDownLatch(1); + var plugin = new ScopedPlugin() { + public AutoCloseable openHandlerScope() { + return () -> { + await(releaseScope); + throw fatal; + }; + } + + public void onInvocationEnd(InvocationEndInfo info) { + ended.set(true); + endCalls.incrementAndGet(); + endInfo.set(info); + } + }; + var workers = + new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, new SynchronousQueue<>(), task -> { + var owner = daemon(task, "shutdown-owner"); + owner.setUncaughtExceptionHandler((thread, failure) -> { + ownerFatal.set(failure); + fatalEscaped.countDown(); + }); + return owner; + }) { + @Override + public void execute(Runnable task) { + super.execute(() -> { + try { + task.run(); + } finally { + ownerFinished.countDown(); + } + }); + } + + @Override + public int getActiveCount() { + // ExecutionManager.close reaches this after invocation-end callbacks. Hold resource closure + // until the owner has reported the fatal, without altering the handler/control futures. + assertTrue(ended.get()); + shutdownEntered.set(true); + releaseScope.countDown(); + await(ownerFinished); + return super.getActiveCount(); + } + }; + var deadline = new Deadline(); + try { + var output = assertDoesNotThrow(() -> DurableExecutor.execute( + input(), + deadline.context(), + TypeToken.get(String.class), + (value, ctx) -> { + deadline.arm(0); + ctx.wait("pause", Duration.ofSeconds(1)); + return "done"; + }, + config(workers, info -> plugin))); + assertEquals(ExecutionStatus.PENDING, output.status(), "the already finalized caller outcome stays frozen"); + assertTrue(fatalEscaped.await(2, TimeUnit.SECONDS), "fatal must still escape the actual owner thread"); + assertSame(fatal, ownerFatal.get()); + assertTrue(shutdownEntered.get()); + assertEquals(1, endCalls.get(), "a late fatal must not replay already delivered end hooks"); + assertEquals( + InvocationStatus.PENDING, + endInfo.get().invocationStatus(), + "snapshot reflects outcome known at dispatch"); + assertNull(endInfo.get().executionError(), "the fatal is reported only later, during shutdown"); + } finally { + releaseScope.countDown(); + workers.shutdown(); + assertTrue(workers.awaitTermination(2, TimeUnit.SECONDS)); + } + } + + static void scopeFatal(boolean retry, boolean wrapped, Error fatal, boolean fromBodyFinally) throws Exception { + scopeFatal(retry, wrapped, fatal, fromBodyFinally, false); + } + + static void scopeFatal(boolean retry, boolean wrapped, Error fatal, boolean fromBodyFinally, boolean afterTimeout) + throws Exception { + var deadline = new Deadline(); + var closeEntered = new CountDownLatch(1); + var releaseFatal = new CountDownLatch(1); + var fatalRaised = new CountDownLatch(1); + var fatalBeforeEnd = new AtomicBoolean(); + var endCalls = new AtomicInteger(); + var endStatus = new AtomicReference(); + var endError = new AtomicReference(); + var plugin = new ScopedPlugin() { + public AutoCloseable openHandlerScope() { + var owner = Thread.currentThread(); + return () -> { + check(owner == Thread.currentThread(), "wrong scope close thread"); + if (!fromBodyFinally) raiseFatal(closeEntered, releaseFatal, fatalRaised, wrapped, fatal); + }; + } + + public void onInvocationEnd(InvocationEndInfo info) { + fatalBeforeEnd.set(fatalRaised.getCount() == 0); + endCalls.incrementAndGet(); + endStatus.set(info.invocationStatus()); + endError.set(info.executionError()); + } + }; + var workers = Executors.newCachedThreadPool(r -> daemon(r, "fatal-owner")); + var callers = Executors.newSingleThreadExecutor(r -> daemon(r, "fatal-caller")); + var original = retryError(); + var cfg = config(workers, info -> plugin); + try { + var response = callers.submit(() -> DurableExecutor.execute( + input(), + deadline.context(), + TypeToken.get(String.class), + (value, ctx) -> { + deadline.arm(30_000); + try { + if (retry) + ctx.step("retry", String.class, step -> { + throw original; + }); + else ctx.wait("pause", Duration.ofSeconds(1)); + return "done"; + } finally { + if (fromBodyFinally) raiseFatal(closeEntered, releaseFatal, fatalRaised, wrapped, fatal); + } + }, + cfg)); + check(closeEntered.await(2, TimeUnit.SECONDS), "fatal site not reached"); + check(deadline.budgetRead.await(2, TimeUnit.SECONDS), "winner not observed before fatal"); + if (afterTimeout) { + outcome(response, retry, original); + assertFalse(fatalBeforeEnd.get()); + releaseFatal.countDown(); + assertTrue(fatalRaised.await(2, TimeUnit.SECONDS)); + outcome(response, retry, original); + return; + } + releaseFatal.countDown(); + if (fromBodyFinally) { + outcome(response, retry, original); + assertTrue(fatalBeforeEnd.get()); + } else { + var thrown = assertThrows(ExecutionException.class, () -> response.get(3, TimeUnit.SECONDS)); + assertSame(fatal, thrown.getCause(), "scope-owned fatal must escape the invocation caller"); + assertEquals(1, endCalls.get(), "major invocation lifetime finalizes created plugins once"); + assertEquals(InvocationStatus.RETRYING, endStatus.get()); + assertSame(fatal, endError.get()); + } + + } finally { + releaseFatal.countDown(); + workers.shutdown(); + workers.awaitTermination(2, TimeUnit.SECONDS); + callers.shutdownNow(); + } + } + + @SuppressWarnings("removal") + @ParameterizedTest + @CsvSource({"false", "true"}) + void majorBodyFatalRetainsCallerPropagation(boolean threadDeath) { + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("body fatal"); + var workers = Executors.newCachedThreadPool(r -> daemon(r, "body-fatal")); + try { + assertSame( + fatal, + assertThrows( + Error.class, + () -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, ctx) -> { + throw fatal; + }, + config(workers)))); + } finally { + workers.shutdownNow(); + } + } + + static void raiseFatal( + CountDownLatch entered, CountDownLatch release, CountDownLatch raised, boolean wrapped, Error fatal) { + entered.countDown(); + await(release); + raised.countDown(); + if (wrapped) throw new CompletionException(new ExecutionException(fatal)); + throw fatal; + } + + static void outcome(Future response, boolean retry, Throwable original) throws Exception { + if (retry) { + try { + response.get(3, TimeUnit.SECONDS); + throw new AssertionError("retry returned output"); + } catch (ExecutionException e) { + check(e.getCause() == original, "retry failure changed to " + e.getCause()); + } + } else check(response.get(3, TimeUnit.SECONDS).status() == ExecutionStatus.PENDING, "suspension changed"); + } + + static DurableConfig config(ExecutorService workers, DurableExecutionPluginFactory... plugins) { + return DurableConfig.builder() + .withDurableExecutionClient(new LocalMemoryExecutionClient()) + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(plugins) + .build(); + } + + static UnrecoverableDurableExecutionException retryError() { + return new UnrecoverableDurableExecutionException( + ErrorObject.builder().errorMessage("original retry").build(), true); + } + + static DurableExecutionInput input() { + var operation = Operation.builder() + .id("id") + .name("test") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.now()) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/name/id", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } + + static Thread daemon(Runnable runnable, String name) { + var thread = new Thread(runnable, name); + thread.setDaemon(true); + return thread; + } + + static void await(CountDownLatch latch) { + try { + check(latch.await(3, TimeUnit.SECONDS), "fixture latch timed out"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("owner interrupted", e); + } + } + + static void check(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } + + static class Deadline { + volatile long start; + volatile long end; + final CountDownLatch budgetRead = new CountDownLatch(1); + + void arm(int millis) { + start = System.nanoTime(); + end = start + TimeUnit.MILLISECONDS.toNanos(millis); + } + + long elapsedMillis() { + return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + } + + int remaining() { + return end == 0 ? 30_000 : (int) Math.max(0L, TimeUnit.NANOSECONDS.toMillis(end - System.nanoTime())); + } + + Context context() { + return (Context) Proxy.newProxyInstance( + Context.class.getClassLoader(), new Class[] {Context.class}, (proxy, method, args) -> { + if (method.getName().equals("getRemainingTimeInMillis")) { + budgetRead.countDown(); + return remaining(); + } + if (method.getName().equals("getAwsRequestId")) return "test-request"; + if (method.getName().equals("getMemoryLimitInMB")) return 512; + return null; + }); + } + } + + @HandlerScoped(ScopedPlugin.Opener.class) + private abstract static class ScopedPlugin implements DurableExecutionPlugin { + public abstract AutoCloseable openHandlerScope(); + + public static class Opener implements Function { + public AutoCloseable apply(ScopedPlugin plugin) { + return plugin.openHandlerScope(); + } + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndCleanupTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndCleanupTest.java new file mode 100644 index 000000000..ab3fc4f06 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationEndCleanupTest.java @@ -0,0 +1,76 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class InvocationEndCleanupTest { + @SuppressWarnings("removal") + static Stream fatalCases() { + return Stream.of(false, true) + .flatMap(executionView -> Stream.of(false, true) + .flatMap(wrapped -> Stream.of(new InternalError("first end hook"), new ThreadDeath()) + .map(fatal -> Arguments.of(executionView, wrapped, fatal)))); + } + + @ParameterizedTest + @MethodSource("fatalCases") + void laterOtelAndObserverFinalizeBeforeEndFatalEscapes(boolean executionView, boolean wrapped, Error fatal) { + var exporter = InMemorySpanExporter.create(); + var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var config = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(() -> new ExtractedContext( + "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED)) + .build(); + var otel = executionView + ? ExecutionOtelPlugin.factory(builder, config) + : InvocationOtelPlugin.factory(builder, config); + var calls = new ArrayList(); + var snapshots = new ArrayList(); + DurableExecutionPluginFactory first = info -> new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo end) { + calls.add("first"); + snapshots.add(end); + if (wrapped) throw new CompletionException(new ExecutionException(fatal)); + throw fatal; + } + }; + DurableExecutionPluginFactory last = info -> new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo end) { + calls.add("last"); + snapshots.add(end); + } + }; + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> "done", + DurableConfig.builder().withPlugins(first, otel, last).build()); + assertSame(fatal, assertThrows(Error.class, () -> runner.run("input"))); + assertEquals(List.of("first", "last"), calls); + assertSame(snapshots.get(0), snapshots.get(1), "each plugin receives the same single dispatch snapshot"); + assertEquals(InvocationStatus.SUCCEEDED, snapshots.get(1).invocationStatus()); + assertTrue( + exporter.getFinishedSpanItems().stream() + .anyMatch(span -> span.getName().equals("Invocation")), + "the later OTel plugin must finish and export its span before the fatal reaches the caller"); + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationHeaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationHeaderTest.java new file mode 100644 index 000000000..1f4b1577e --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationHeaderTest.java @@ -0,0 +1,171 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.time.Instant; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; + +class InvocationHeaderTest { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void overlappingInvocationsAndResumeKeepTheirOwnHeader(boolean executionView) throws Exception { + var barrier = new CyclicBarrier(2); + var sampled = CompletableFuture.runAsync(() -> runInvocations(executionView, true, barrier)); + var unsampled = CompletableFuture.runAsync(() -> runInvocations(executionView, false, barrier)); + CompletableFuture.allOf(sampled, unsampled).get(20, TimeUnit.SECONDS); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void factoryHeaderPrecedesSpanCreationAndPreservesExtractorOverride(boolean executionView) { + var exporter = InMemorySpanExporter.create(); + var builder = SdkTracerProvider.builder() + .setSampler(Sampler.alwaysOn()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var calls = new AtomicInteger(); + var config = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(new XRayContextExtractor() { + @Override + public ExtractedContext extract() { + calls.incrementAndGet(); + return super.extract(); + } + }) + .build(); + var factory = executionView + ? ExecutionOtelPlugin.factory(builder, config) + : InvocationOtelPlugin.factory(builder, config); + var info = + invocation("request", "arn", true, Instant.EPOCH, "Root=1-6955b900-123456789012345678901234;Sampled=0"); + var plugin = factory.createPlugin(info); + plugin.onInvocationEnd(new InvocationEndInfo("request", "arn", true, InvocationStatus.SUCCEEDED, null)); + assertEquals(1, calls.get()); + assertTrue(exporter.getFinishedSpanItems().isEmpty()); + var next = factory.createPlugin(new InvocationInfo("request", "arn", true, Instant.EPOCH)); + next.onInvocationEnd(new InvocationEndInfo("request", "arn", true, InvocationStatus.SUCCEEDED, null)); + assertEquals(2, calls.get()); + assertTrue(exporter.getFinishedSpanItems().stream() + .anyMatch(span -> span.getName().equals("Workflow"))); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void capturedMissingHeaderDoesNotBorrowTheOverlappingGlobalCarrier(boolean executionView) throws Exception { + var property = "com.amazonaws.xray.traceHeader"; + var previous = System.getProperty(property); + var conflictingTrace = "6955b900aaaaaaaaaaaaaaaaaaaaaaaa"; + System.setProperty(property, "Root=1-6955b900-aaaaaaaaaaaaaaaaaaaaaaaa;Sampled=0"); + var barrier = new CyclicBarrier(2); + var sharedExtractor = new XRayContextExtractor(); + try { + var missing = CompletableFuture.runAsync(() -> { + var exporter = InMemorySpanExporter.create(); + var builder = SdkTracerProvider.builder() + .setSampler(Sampler.alwaysOn()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var config = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(sharedExtractor) + .build(); + var factory = executionView + ? ExecutionOtelPlugin.factory(builder, config) + : InvocationOtelPlugin.factory(builder, config); + var info = invocation("missing", "arn:missing", true, Instant.EPOCH, ""); + var plugin = factory.createPlugin(info); + plugin.onInvocationStart(info); // API available, but this invocation has no header. + await(barrier); + plugin.onInvocationEnd( + new InvocationEndInfo("missing", "arn:missing", true, InvocationStatus.SUCCEEDED, null)); + var spans = exporter.getFinishedSpanItems(); + assertEquals( + 3, + spans.size(), + "the synthetic anchor and both SDK spans must retain their own sampling decision"); + assertTrue(spans.stream().allMatch(span -> !conflictingTrace.equals(span.getTraceId()))); + }); + var present = CompletableFuture.runAsync(() -> { + var extracted = sharedExtractor.extract(invocation( + "present", + "arn:present", + true, + Instant.EPOCH, + "Root=1-6955b900-123456789012345678901234;Sampled=1")); + assertEquals("6955b900123456789012345678901234", extracted.traceId()); + assertEquals(ExtractedContext.Sampling.SAMPLED, extracted.sampling()); + await(barrier); + }); + CompletableFuture.allOf(missing, present).get(20, TimeUnit.SECONDS); + assertNull(sharedExtractor.extract(invocation("missing", "arn:missing", true, Instant.EPOCH, ""))); + assertEquals( + conflictingTrace, + sharedExtractor.extract().traceId(), + "legacy direct extraction keeps its fallback"); + } finally { + if (previous == null) System.clearProperty(property); + else System.setProperty(property, previous); + } + } + + private static void runInvocations(boolean executionView, boolean sampled, CyclicBarrier barrier) { + var exporter = InMemorySpanExporter.create(); + var builder = SdkTracerProvider.builder() + .setSampler(Sampler.alwaysOn()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var config = OtelPluginConfig.builder().enableMdc(false).build(); + DurableExecutionPluginFactory factory = executionView + ? ExecutionOtelPlugin.factory(builder, config) + : InvocationOtelPlugin.factory(builder, config); + var traceId = sampled ? "6955b900123456789012345678901234" : "6955b900aaaaaaaaaaaaaaaaaaaaaaaa"; + var header = "Root=1-" + traceId.substring(0, 8) + "-" + traceId.substring(8) + + ";Parent=1234567890123456;Sampled=" + (sampled ? "1" : "0"); + var arn = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/" + sampled; + for (int invocation = 0; invocation < 2; invocation++) { + var plugin = factory.createPlugin(invocation( + "request-" + invocation, arn, invocation == 0, Instant.parse("2026-10-02T00:00:00Z"), header)); + await(barrier); + plugin.onInvocationEnd(new InvocationEndInfo( + "request-" + invocation, + arn, + invocation == 0, + invocation == 0 ? InvocationStatus.PENDING : InvocationStatus.SUCCEEDED, + null)); + } + var spans = exporter.getFinishedSpanItems(); + if (!sampled) { + assertTrue(spans.isEmpty(), "Upstream Sampled=0 overrides always-on fallback"); + return; + } + assertEquals(3, spans.size()); + assertTrue(spans.stream().allMatch(span -> traceId.equals(span.getTraceId()))); + assertTrue(spans.stream().allMatch(span -> "1234567890123456".equals(span.getParentSpanId()))); + } + + private static InvocationInfo invocation(String request, String arn, boolean first, Instant start, String header) { + return new InvocationInfo(request, arn, first, start, null, Map.of(), Map.of(), header); + } + + private static void await(CyclicBarrier barrier) { + try { + barrier.await(10, TimeUnit.SECONDS); + } catch (Exception e) { + throw new AssertionError(e); + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationInfoHeaderIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationInfoHeaderIntegrationTest.java new file mode 100644 index 000000000..a598d393f --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationInfoHeaderIntegrationTest.java @@ -0,0 +1,153 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import com.amazonaws.services.lambda.runtime.Context; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.time.Instant; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.ExecutionDetails; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.client.DurableExecutionClient; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; + +class InvocationInfoHeaderIntegrationTest { + private static final String TRACE = "6955b900123456789012345678901234"; + private static final String PARENT = "1234567890123456"; + private static final String HEADER = "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=1"; + private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/name/id"; + + @ParameterizedTest + @CsvSource({ + "factory,present", "hook,present", "factory,empty", "hook,empty", + "factory,null", "hook,null", "factory,unavailable", "hook,unavailable" + }) + void ordinaryFactoryAndHookCanReadTheCapturedHeader(String reader, String carrier) { + var runtimeThread = Thread.currentThread(); + var capturedThread = new AtomicReference(); + var factoryInfo = new AtomicReference(); + var hookInfo = new AtomicReference(); + var observed = new AtomicReference<>("callback did not run"); + var runtime = "unavailable".equals(carrier) ? null : mock(RuntimeContext.class); + if (runtime != null) { + when(runtime.getRemainingTimeInMillis()).thenReturn(30000); + when(runtime.getXrayTraceId()).thenAnswer(ignored -> { + capturedThread.set(Thread.currentThread()); + return switch (carrier) { + case "present" -> HEADER; + case "empty" -> ""; + default -> null; + }; + }); + } + var config = DurableConfig.builder() + .withDurableExecutionClient(mock(DurableExecutionClient.class)) + .withPlugins(info -> { + factoryInfo.set(info); + if (reader.equals("factory")) observed.set(info.xRayTraceId()); + return new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo info) { + hookInfo.set(info); + if (reader.equals("hook")) observed.set(info.xRayTraceId()); + } + }; + }) + .build(); + var output = + DurableExecutor.execute(input(), runtime, TypeToken.get(String.class), (value, ctx) -> value, config); + assertEquals(ExecutionStatus.SUCCEEDED, output.status()); + assertEquals("\"input\"", output.result()); + assertNotNull(factoryInfo.get()); + assertSame(factoryInfo.get(), hookInfo.get(), "Factory and hook must receive the same immutable snapshot"); + var expected = + switch (carrier) { + case "present" -> HEADER; + case "unavailable" -> null; + default -> ""; + }; + assertEquals(expected, observed.get(), reader + " must read the captured value from InvocationInfo"); + if (runtime != null) { + verify(runtime, times(1)).getXrayTraceId(); + assertSame(runtimeThread, capturedThread.get()); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void directFactoryDispatchesTheInvocationAwareCustomExtractor(boolean executionView) { + var seen = new AtomicReference(); + var exporter = InMemorySpanExporter.create(); + var config = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(new ContextExtractor() { + @Override + public ExtractedContext extract() { + return null; + } + + @Override + public ExtractedContext extract(InvocationInfo info) { + seen.set(info); + assertEquals(HEADER, info.xRayTraceId()); + return new ExtractedContext(TRACE, PARENT, ExtractedContext.Sampling.SAMPLED); + } + }) + .build(); + var provider = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var factory = executionView + ? ExecutionOtelPlugin.factory(provider, config) + : InvocationOtelPlugin.factory(provider, config); + var info = new InvocationInfo("request", ARN, true, Instant.EPOCH, null, Map.of(), Map.of(), HEADER); + var plugin = factory.createPlugin(info); + plugin.onInvocationStart(info); + plugin.onInvocationEnd(new InvocationEndInfo("request", ARN, true, InvocationStatus.SUCCEEDED, null)); + assertSame(info, seen.get(), "The extract(info) extension must run before factory span creation"); + var spans = exporter.getFinishedSpanItems(); + assertEquals(2, spans.size(), "A complete remote parent is external, not a synthetic root"); + assertTrue(spans.stream().allMatch(span -> TRACE.equals(span.getTraceId()))); + assertTrue(spans.stream().allMatch(span -> PARENT.equals(span.getParentSpanId()))); + } + + private static DurableExecutionInput input() { + var operation = Operation.builder() + .id("id") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + ARN, + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } + + private abstract static class RuntimeContext implements Context { + @Override + public String getXrayTraceId() { + return null; + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationMdcCleanupTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationMdcCleanupTest.java new file mode 100644 index 000000000..3b2ca0cb8 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationMdcCleanupTest.java @@ -0,0 +1,122 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.slf4j.MDC; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class InvocationMdcCleanupTest { + private static final String TRACE_ID = "12345678901234567890123456789012"; + + @ParameterizedTest + @CsvSource({"true,input", "false,input", "true,factory", "false,factory", "true,start", "false,start"}) + void startupFailureClearsMdcOnTheOwningWorker(boolean executionView, String stage) throws Exception { + var fatal = new InternalError("startup hook"); + var uncaught = new AtomicReference(); + var escapedMdc = new AtomicReference(); + var escaped = new CountDownLatch(1); + var executor = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "startup-mdc-owner"); + thread.setDaemon(true); + thread.setUncaughtExceptionHandler((owner, failure) -> { + escapedMdc.set(MDC.get(MdcSpanEnricher.MDC_TRACE_ID)); + uncaught.set(failure); + escaped.countDown(); + }); + return thread; + }); + var config = OtelPluginConfig.builder() + .enableMdc(true) + .contextExtractor( + () -> new ExtractedContext(TRACE_ID, "1234567890123456", ExtractedContext.Sampling.SAMPLED)) + .build(); + var otel = executionView + ? ExecutionOtelPlugin.factory(SdkTracerProvider.builder(), config) + : InvocationOtelPlugin.factory(SdkTracerProvider.builder(), config); + var startMdc = new AtomicReference(); + var ends = new AtomicInteger(); + var end = new AtomicReference(); + DurableExecutionPluginFactory observer = info -> new DurableExecutionPlugin() { + public void onInvocationStart(InvocationInfo value) { + startMdc.set(MDC.get(MdcSpanEnricher.MDC_TRACE_ID)); + } + + public void onInvocationEnd(InvocationEndInfo value) { + ends.incrementAndGet(); + end.set(value); + } + }; + DurableExecutionPluginFactory faulty = info -> { + if (stage.equals("factory")) throw fatal; + return new DurableExecutionPlugin() { + public void onInvocationStart(InvocationInfo value) { + if (stage.equals("start")) throw fatal; + } + }; + }; + var builder = DurableConfig.builder().withExecutorService(executor).withPlugins(otel, observer, faulty); + if (stage.equals("input")) + builder.withSerDes(new SerDes() { + private final JacksonSerDes delegate = new JacksonSerDes(); + + public String serialize(Object value) { + return delegate.serialize(value); + } + + public T deserialize(String data, TypeToken type) { + throw new IllegalArgumentException("bad input"); + } + }); + var bodies = new AtomicInteger(); + try { + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + bodies.incrementAndGet(); + return "unreachable"; + }, + builder.build()); + if (stage.equals("input")) { + assertEquals(ExecutionStatus.FAILED, runner.run("input").getStatus()); + assertNull( + executor.submit(() -> MDC.get(MdcSpanEnricher.MDC_TRACE_ID)) + .get(2, TimeUnit.SECONDS), + "startup failure must not leave a trace on the reused handler worker"); + } else { + assertSame(fatal, assertThrows(InternalError.class, () -> runner.run("input"))); + assertTrue(escaped.await(2, TimeUnit.SECONDS)); + assertSame(fatal, uncaught.get()); + assertNull(escapedMdc.get(), "cleanup must run before the fatal escapes its owner thread"); + } + assertEquals(TRACE_ID, startMdc.get(), "later startup plugins must still see the invocation trace"); + assertEquals(0, bodies.get()); + assertEquals(1, ends.get()); + assertEquals( + stage.equals("input") ? InvocationStatus.FAILED : InvocationStatus.RETRYING, + end.get().invocationStatus()); + if (!stage.equals("input")) assertSame(fatal, end.get().executionError()); + } finally { + executor.shutdownNow(); + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java index f9ff81d20..fe54f3499 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginIntegrationTest.java @@ -45,14 +45,15 @@ void setUp() { OtelPluginAutoConfigurationState.resetInstalledForTest(); spanExporter = InMemorySpanExporter.create(); - var plugin = new InvocationOtelPlugin( + // One factory for the environment; the SDK creates one plugin instance per invocation from it. + var factory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), OtelPluginConfig.builder() .contextExtractor(() -> null) .enableMdc(false) .build()); - otelConfig = DurableConfig.builder().withPlugins(plugin).build(); + otelConfig = DurableConfig.builder().withPlugins(factory).build(); } @AfterEach @@ -337,7 +338,7 @@ void failedStep_producesErrorSpan() { void sampling_off_producesNoSpans() { var sampledExporter = InMemorySpanExporter.create(); - var noSamplePlugin = new InvocationOtelPlugin( + var noSampleFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder() .setSampler(Sampler.alwaysOff()) .addSpanProcessor(SimpleSpanProcessor.create(sampledExporter)), @@ -346,7 +347,8 @@ void sampling_off_producesNoSpans() { .enableMdc(false) .build()); - var noSampleConfig = DurableConfig.builder().withPlugins(noSamplePlugin).build(); + var noSampleConfig = + DurableConfig.builder().withPlugins(noSampleFactory).build(); var runner = LocalDurableTestRunner.create( String.class, (input, ctx) -> ctx.step("step", String.class, stepCtx -> "result"), noSampleConfig); @@ -545,8 +547,8 @@ void waitForCondition_producesSpansWithAttempts() { } @Test - void defaultConstructor_lateBindsGlobalSdkTracerProviderAtInvocationStart() { - var defaultPlugin = new InvocationOtelPlugin(); + void agentPathFactory_bindsGlobalSdkTracerProviderWhenTheInvocationsInstanceIsCreated() { + var defaultFactory = InvocationOtelPlugin.factory(); assertFalse(GlobalOpenTelemetry.isSet()); OtelPluginAutoConfigurationState.markInstalled(); @@ -556,7 +558,7 @@ void defaultConstructor_lateBindsGlobalSdkTracerProviderAtInvocationStart() { .build(); OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); - var defaultConfig = DurableConfig.builder().withPlugins(defaultPlugin).build(); + var defaultConfig = DurableConfig.builder().withPlugins(defaultFactory).build(); var runner = LocalDurableTestRunner.create( String.class, (input, ctx) -> ctx.step("global-step", String.class, stepCtx -> "Hello " + input), @@ -573,7 +575,7 @@ void defaultConstructor_lateBindsGlobalSdkTracerProviderAtInvocationStart() { } @Test - void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() { + void agentPathFactory_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() { OtelPluginAutoConfigurationState.markInstalled(); GlobalOpenTelemetry.resetForTest(); var globalExporter = InMemorySpanExporter.create(); @@ -595,8 +597,9 @@ public ContextPropagators getPropagators() { } }); - var defaultConfig = - DurableConfig.builder().withPlugins(new InvocationOtelPlugin()).build(); + var defaultConfig = DurableConfig.builder() + .withPlugins(InvocationOtelPlugin.factory()) + .build(); var runner = LocalDurableTestRunner.create( String.class, (input, ctx) -> ctx.step("javaagent-step", String.class, stepCtx -> "Hello " + input), diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java index d8d7e7f43..585ed3ad9 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationOtelPluginTest.java @@ -4,9 +4,12 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static software.amazon.lambda.durable.otel.Invocations.started; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; @@ -24,9 +27,11 @@ import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.testing.time.TestClock; import io.opentelemetry.sdk.trace.IdGenerator; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.opentelemetry.sdk.trace.SpanProcessor; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.time.Instant; import java.util.List; @@ -43,7 +48,9 @@ class InvocationOtelPluginTest { private InMemorySpanExporter spanExporter; - private InvocationOtelPlugin plugin; + + /** The environment's plugin factory; each test creates one instance per invocation from it. */ + private DurableExecutionPluginFactory factory; @BeforeEach void setUp() { @@ -52,7 +59,7 @@ void setUp() { OtelPluginAutoConfigurationState.resetInstalledForTest(); spanExporter = InMemorySpanExporter.create(); - plugin = new InvocationOtelPlugin( + factory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), OtelPluginConfig.builder() .contextExtractor(() -> null) @@ -69,12 +76,13 @@ void tearDown() { } @Test - void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() { + void agentPathFactory_bindsGlobalProviderOnALaterInvocationsInstance() { GlobalOpenTelemetry.resetForTest(); OtelPluginAutoConfigurationState.markInstalled(); - var defaultPlugin = new InvocationOtelPlugin(); - defaultPlugin.onInvocationStart(new InvocationInfo("req-disabled", "arn:disabled", true, Instant.now())); + var defaultPluginFactory = InvocationOtelPlugin.factory(); + var defaultPlugin = + started(defaultPluginFactory, new InvocationInfo("req-disabled", "arn:disabled", true, Instant.now())); defaultPlugin.onOperationStart(new OperationInfo( "op-disabled", "disabled-step", "STEP", "Step", null, Instant.now(), null, null, false)); defaultPlugin.onOperationEnd(new OperationEndInfo( @@ -101,7 +109,8 @@ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() { .build(); OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); - defaultPlugin.onInvocationStart(new InvocationInfo("req-enabled", "arn:enabled", true, Instant.now())); + defaultPlugin = + started(defaultPluginFactory, new InvocationInfo("req-enabled", "arn:enabled", true, Instant.now())); defaultPlugin.onOperationStart(new OperationInfo( "op-enabled", "enabled-step", "STEP", "Step", null, Instant.now(), null, null, false)); defaultPlugin.onOperationEnd(new OperationEndInfo( @@ -121,14 +130,14 @@ void defaultConstructor_retriesGlobalProviderBindingOnNextInvocation() { new InvocationEndInfo("req-enabled", "arn:enabled", true, InvocationStatus.SUCCEEDED, null)); var spans = globalExporter.getFinishedSpanItems(); - assertEquals(3, spans.size()); + assertEquals(4, spans.size()); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("enabled-step"))); assertFalse(spans.stream().anyMatch(span -> span.getName().equals("disabled-step"))); } @Test - void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { - var defaultPlugin = new InvocationOtelPlugin(); + void agentPathFactory_usesGlobalSdkTracerProviderDirectly() { + var defaultPluginFactory = InvocationOtelPlugin.factory(); assertFalse(GlobalOpenTelemetry.isSet()); OtelPluginAutoConfigurationState.markInstalled(); @@ -138,7 +147,8 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { .build(); OpenTelemetrySdk.builder().setTracerProvider(globalTracerProvider).buildAndRegisterGlobal(); - defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var defaultPlugin = + started(defaultPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); defaultPlugin.onOperationStart( new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, null, false)); defaultPlugin.onOperationEnd(new OperationEndInfo( @@ -159,12 +169,12 @@ void defaultConstructor_usesGlobalSdkTracerProviderDirectly() { var spans = globalExporter.getFinishedSpanItems(); // Plugin creates Workflow + Invocation + operation spans - assertEquals(3, spans.size()); + assertEquals(4, spans.size()); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); } @Test - void defaultConstructor_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() { + void agentPathFactory_usesJavaAgentGlobalTracerProviderDirectly_withSeparateAutoConfiguredIdGenerator() { OtelPluginAutoConfigurationState.markInstalled(); GlobalOpenTelemetry.resetForTest(); var globalExporter = InMemorySpanExporter.create(); @@ -186,8 +196,9 @@ public ContextPropagators getPropagators() { } }); - var defaultPlugin = new InvocationOtelPlugin(); - defaultPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var defaultPluginFactory = InvocationOtelPlugin.factory(); + var defaultPlugin = + started(defaultPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); defaultPlugin.onOperationStart( new OperationInfo("op-1", "step", "STEP", "Step", null, Instant.now(), null, null, false)); defaultPlugin.onOperationEnd(new OperationEndInfo( @@ -208,7 +219,7 @@ public ContextPropagators getPropagators() { var spans = globalExporter.getFinishedSpanItems(); // Plugin creates Workflow + Invocation + operation spans - assertEquals(3, spans.size()); + assertEquals(4, spans.size()); assertTrue(spans.stream().anyMatch(span -> span.getName().equals("step"))); var expectedIds = new DeterministicIdGenerator(); expectedIds.setDurableExecutionArn("arn:exec1"); @@ -291,8 +302,14 @@ void invocationOtelPluginProvider_isRegisteredAsServiceProvider() { .get(); assertEquals("otel-invocation", provider.getName()); - assertEquals(DurableExecutionPluginProvider.API_VERSION, provider.getApiVersion()); - assertEquals(InvocationOtelPlugin.class, provider.getPluginType()); + + // The provider is the per-invocation factory: it creates an InvocationOtelPlugin for the invocation it is + // handed, and a distinct instance for the next one. + var first = provider.createPlugin(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var second = provider.createPlugin(new InvocationInfo("req-2", "arn:exec1", false, Instant.now())); + assertInstanceOf(InvocationOtelPlugin.class, first); + assertInstanceOf(InvocationOtelPlugin.class, second); + assertNotSame(first, second, "Each invocation gets its own plugin instance"); } @Test @@ -306,8 +323,11 @@ void invocationStart_staysOnExecutionTrace_withoutLinkingAmbientSpan() { var ambientSpanContext = SpanContext.create(ambientTraceId, ambientSpanId, TraceFlags.getSampled(), TraceState.getDefault()); + // The instance is created inside the ambient scope because the invocation's parent resolution happens when the + // factory creates it, not later. + DurableExecutionPlugin plugin; try (var ignored = Span.wrap(ambientSpanContext).makeCurrent()) { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); } plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); @@ -320,8 +340,13 @@ void invocationStart_staysOnExecutionTrace_withoutLinkingAmbientSpan() { @Test void invocationStart_and_end_createsSpan() { - plugin.onInvocationStart(new InvocationInfo( - "req-123", "arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1", true, Instant.now())); + var plugin = started( + factory, + new InvocationInfo( + "req-123", + "arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1", + true, + Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo( "req-123", "arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1", @@ -330,7 +355,7 @@ void invocationStart_and_end_createsSpan() { null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size()); // invocation + Workflow + assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot var span = spans.get(0); assertEquals("Invocation", span.getName()); @@ -340,7 +365,7 @@ void invocationStart_and_end_createsSpan() { @Test void customInstrumentationName_isUsedForTracerScope() { var exporter = InMemorySpanExporter.create(); - var customPlugin = new InvocationOtelPlugin( + var customPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), OtelPluginConfig.builder() .contextExtractor(() -> null) @@ -348,7 +373,7 @@ void customInstrumentationName_isUsedForTracerScope() { .workflowSpanName("Workflow") .instrumentationName("my-custom-scope") .build()); - customPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var customPlugin = started(customPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); customPlugin.onInvocationEnd( new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); @@ -361,11 +386,14 @@ void customInstrumentationName_isUsedForTracerScope() { @Test void explicitProvider_unrelatedRootSpansKeepFreshTraceIds() { + // This invocation's instance and the unrelated library share the one provider the factory built. + var info = new InvocationInfo("req-1", "arn:exec1", true, Instant.now()); + var plugin = (InvocationOtelPlugin) factory.createPlugin(info); var provider = sdkTracerProvider(plugin); var unrelatedTracer = provider.get("unrelated-library"); var before = unrelatedTracer.spanBuilder("before").setNoParent().startSpan(); - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + plugin.onInvocationStart(info); var during = unrelatedTracer.spanBuilder("during").setNoParent().startSpan(); plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var after = unrelatedTracer.spanBuilder("after").setNoParent().startSpan(); @@ -394,8 +422,8 @@ void globalProvider_unrelatedRootSpansKeepFreshTraceIds() { var unrelatedTracer = provider.get("unrelated-library"); var before = unrelatedTracer.spanBuilder("before").setNoParent().startSpan(); - var globalPlugin = new InvocationOtelPlugin(); - globalPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var globalPluginFactory = InvocationOtelPlugin.factory(); + var globalPlugin = started(globalPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); var during = unrelatedTracer.spanBuilder("during").setNoParent().startSpan(); globalPlugin.onInvocationEnd( new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); @@ -415,7 +443,7 @@ void globalProvider_unrelatedRootSpansKeepFreshTraceIds() { @Test void invocationSpan_hasInternalKind() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var span = spanExporter.getFinishedSpanItems().get(0); @@ -424,7 +452,7 @@ void invocationSpan_hasInternalKind() { @Test void operationSpanName_usesOperationName_withoutPrefix() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "create-greeting", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onOperationEnd(new OperationEndInfo( @@ -454,7 +482,7 @@ void operationSpanName_usesOperationName_withoutPrefix() { @Test void attemptSpanName_usesOperationNameWithAttemptNumber() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "process-order", "STEP", "Step", null, Instant.now(), false, 1)); plugin.onUserFunctionEnd(new UserFunctionEndInfo( @@ -483,7 +511,7 @@ void attemptSpanName_usesOperationNameWithAttemptNumber() { @Test void operationEnd_withAttempt_stampsAttemptNumberOnOperationSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "flaky", "STEP", "Step", null, Instant.now(), null, null, false)); @@ -514,7 +542,7 @@ void operationEnd_withAttempt_stampsAttemptNumberOnOperationSpan() { @Test void attemptSpan_carriesOperationSubtype() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "process-order", "STEP", "Step", null, Instant.now(), false, 1)); plugin.onUserFunctionEnd(new UserFunctionEndInfo( @@ -543,7 +571,7 @@ void attemptSpan_carriesOperationSubtype() { @Test void operationEnd_withoutMatchingStart_stampsAttemptNumberOnContinuationSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); // No onOperationStart in this invocation → onOperationEnd takes the continuation-span branch. plugin.onOperationEnd(new OperationEndInfo( @@ -577,23 +605,23 @@ void operationEnd_withoutMatchingStart_stampsAttemptNumberOnContinuationSpan() { @Test void invocationEnd_withFailure_setsErrorStatus() { - plugin.onInvocationStart(new InvocationInfo("req-123", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-123", "arn:exec1", true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo( "req-123", "arn:exec1", true, InvocationStatus.FAILED, new RuntimeException("boom"))); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size()); // invocation + Workflow + assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot assertEquals(StatusCode.ERROR, spans.get(0).getStatus().getStatusCode()); } @Test void invocationEnd_withRetrying_leavesStatusUnset() { - plugin.onInvocationStart(new InvocationInfo("req-123", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-123", "arn:exec1", true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo( "req-123", "arn:exec1", true, InvocationStatus.RETRYING, new RuntimeException("transient"))); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(1, spans.size()); + assertEquals(2, spans.size()); assertEquals( StatusCode.UNSET, spans.get(0).getStatus().getStatusCode(), @@ -602,7 +630,7 @@ void invocationEnd_withRetrying_leavesStatusUnset() { @Test void operationStart_createsSpan_operationEnd_endsIt() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); var start = Instant.parse("2026-06-01T10:00:00Z"); var end = Instant.parse("2026-06-01T10:00:05Z"); @@ -618,7 +646,7 @@ void operationStart_createsSpan_operationEnd_endsIt() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(3, spans.size()); // operation + invocation + Workflow + assertEquals(4, spans.size()); // operation + invocation + Workflow + DurableExecutionRoot var operationSpan = spans.stream() .filter(s -> s.getName().contains("step")) @@ -629,7 +657,7 @@ void operationStart_createsSpan_operationEnd_endsIt() { @Test void userFunctionStart_and_end_createsAttemptSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "compute", "STEP", "Step", null, Instant.now(), false, 1)); @@ -650,7 +678,7 @@ void userFunctionStart_and_end_createsAttemptSpan() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(3, spans.size()); // attempt + invocation + Workflow + assertEquals(4, spans.size()); // attempt + invocation + Workflow + DurableExecutionRoot var attemptSpan = spans.stream() .filter(s -> s.getName().contains("attempt")) @@ -662,7 +690,7 @@ void userFunctionStart_and_end_createsAttemptSpan() { @Test void userFunctionEnd_withFailure_setsErrorOnAttemptSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "failing", "STEP", "Step", null, Instant.now(), false, 1)); @@ -691,7 +719,7 @@ void userFunctionEnd_withFailure_setsErrorOnAttemptSpan() { @Test void userFunctionEnd_withSuccess_setsOkOnAttemptSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "compute", "STEP", "Step", null, Instant.now(), false, 1)); @@ -719,7 +747,7 @@ void userFunctionEnd_withSuccess_setsOkOnAttemptSpan() { @Test void userFunctionEnd_withIncomplete_leavesAttemptSpanUnset() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "waiting", "STEP", "Step", null, Instant.now(), false, 1)); @@ -749,7 +777,7 @@ void userFunctionEnd_withIncomplete_leavesAttemptSpanUnset() { @Test void operationEnd_withSuccess_setsOkOnOperationSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "step-ok", "STEP", "Step", null, Instant.now(), null, null, false)); @@ -783,7 +811,7 @@ void operationEnd_withNonSuccessStatusAndNoError_leavesOperationSpanUnset() { // onOperationEnd fires for every terminal status. A CANCELLED operation (or an error-less // FAILED/TIMED_OUT/STOPPED) carries a non-null, non-SUCCEEDED status with a null error. It must NOT be // stamped OK — the span status stays UNSET. - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-cancel", "step-cancel", "STEP", "Step", null, Instant.now(), null, null, false)); @@ -814,7 +842,7 @@ void operationEnd_withNonSuccessStatusAndNoError_leavesOperationSpanUnset() { void operationEnd_withoutMatchingStart_nonSuccessStatusAndNoError_leavesContinuationSpanUnset() { // Same guard on the continuation-span branch (operation completed between invocations): an error-less // TIMED_OUT terminal status must NOT be stamped OK. - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onOperationEnd(new OperationEndInfo( "op-cb-timeout", @@ -843,7 +871,7 @@ void operationEnd_withoutMatchingStart_nonSuccessStatusAndNoError_leavesContinua void operationEnd_withNullStatusAndNoError_setsOkOnOperationSpan() { // A successful statusless virtual (FLAT CONTEXT) operation fires onOperationEnd with a null operation -> // null status and null error. This is genuine success and must be stamped OK. - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-ctx", "my-ctx", "CONTEXT", null, null, Instant.now(), null, null, false)); @@ -873,7 +901,7 @@ void operationEnd_withNullStatusAndNoError_setsOkOnOperationSpan() { @Test void fullLifecycle_producesCorrectSpanHierarchy() { var arn = "arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1"; - plugin.onInvocationStart(new InvocationInfo("req-1", arn, true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", arn, true, Instant.now())); // Step 1: operation starts, user function runs, operation completes plugin.onOperationStart( @@ -941,7 +969,7 @@ void fullLifecycle_producesCorrectSpanHierarchy() { var spans = spanExporter.getFinishedSpanItems(); // 2 attempt spans + 2 operation spans + 1 invocation span + 1 Workflow span = 6 - assertEquals(6, spans.size()); + assertEquals(7, spans.size()); var workflowTraceId = spanByName("Workflow").getTraceId(); var invocationTraceId = spanByName("Invocation").getTraceId(); @@ -956,14 +984,14 @@ void invocationRoots_sameExecutionShareExecutionTrace() { // Same execution start time across invocations so the ARN-derived canonical trace ID is reproducible. var startTime = Instant.now(); - plugin.onInvocationStart(new InvocationInfo("req-1", arn, true, startTime)); + var plugin = started(factory, new InvocationInfo("req-1", arn, true, startTime)); plugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.PENDING, null)); var firstTraceId = spanByName("Invocation").getTraceId(); spanExporter.reset(); // Second invocation of same execution - plugin.onInvocationStart(new InvocationInfo("req-2", arn, false, startTime)); + plugin = started(factory, new InvocationInfo("req-2", arn, false, startTime)); plugin.onInvocationEnd(new InvocationEndInfo("req-2", arn, false, InvocationStatus.SUCCEEDED, null)); var secondTraceId = spanByName("Invocation").getTraceId(); @@ -977,7 +1005,7 @@ void invocationRoots_sameExecutionShareExecutionTrace() { @Test void operationNotCompleted_spanEndedAtInvocationEnd() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); // Operation starts but never completes (e.g., wait operation, invocation suspends) plugin.onOperationStart( @@ -988,7 +1016,7 @@ void operationNotCompleted_spanEndedAtInvocationEnd() { var spans = spanExporter.getFinishedSpanItems(); // Should have: operation span (ended at invocation end) + invocation span - assertEquals(2, spans.size()); + assertEquals(3, spans.size()); var operationSpan = spans.stream() .filter(s -> s.getName().contains("wait")) @@ -1000,7 +1028,7 @@ void operationNotCompleted_spanEndedAtInvocationEnd() { @Test void operationStart_withStatus_preservesStatus() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", false, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", false, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "my-step", "STEP", "Step", null, Instant.now(), null, "PENDING", true)); @@ -1015,16 +1043,38 @@ void operationStart_withStatus_preservesStatus() { @Test void invocationEnd_closesNestedSpansChildFirst() { + var start = Instant.parse("2026-01-01T00:00:00Z"); + var clock = TestClock.create(start); + // Compare timestamps on a controlled clock: independently anchored real span clocks can differ slightly. + // Advancing at each onEnd preserves a strict time-order assertion as well as the exporter-order assertions. + var clockAdvancer = mock(SpanProcessor.class, CALLS_REAL_METHODS); + when(clockAdvancer.isEndRequired()).thenReturn(true); + doAnswer(invocation -> { + clock.advance(1, TimeUnit.NANOSECONDS); + return null; + }) + .when(clockAdvancer) + .onEnd(any()); + var timedFactory = InvocationOtelPlugin.factory( + SdkTracerProvider.builder() + .setClock(clock) + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .addSpanProcessor(clockAdvancer), + OtelPluginConfig.builder() + .contextExtractor(() -> null) + .enableMdc(false) + .build()); var parentId = "op-parent"; var childId = "op-child"; - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(timedFactory, new InvocationInfo("req-1", "arn:exec1", true, start)); plugin.onOperationStart(new OperationInfo( - parentId, "parent-context", "CONTEXT", "RunInChildContext", null, Instant.now(), null, null, false)); + parentId, "parent-context", "CONTEXT", "RunInChildContext", null, start, null, null, false)); plugin.onOperationStart( - new OperationInfo(childId, "child-step", "STEP", "Step", parentId, Instant.now(), null, null, false)); + new OperationInfo(childId, "child-step", "STEP", "Step", parentId, start, null, null, false)); plugin.onUserFunctionStart( - new UserFunctionStartInfo(childId, "child-step", "STEP", "Step", parentId, Instant.now(), false, 1)); + new UserFunctionStartInfo(childId, "child-step", "STEP", "Step", parentId, start, false, 1)); + clock.advance(1, TimeUnit.SECONDS); plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.PENDING, null)); var parentSpan = spanByName("parent-context"); @@ -1038,17 +1088,17 @@ void invocationEnd_closesNestedSpansChildFirst() { spans.indexOf(childSpan) < spans.indexOf(parentSpan), "Child operation span must be exported before its parent operation span"); assertTrue( - attemptSpan.getEndEpochNanos() <= childSpan.getEndEpochNanos(), + attemptSpan.getEndEpochNanos() < childSpan.getEndEpochNanos(), "Attempt span must end before its operation span"); assertTrue( - childSpan.getEndEpochNanos() <= parentSpan.getEndEpochNanos(), + childSpan.getEndEpochNanos() < parentSpan.getEndEpochNanos(), "Child operation span must end before its parent operation span"); } @Test void sampling_disabled_producesNoSpans() { spanExporter = InMemorySpanExporter.create(); - var sampledPlugin = new InvocationOtelPlugin( + var sampledPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder() .setSampler(io.opentelemetry.sdk.trace.samplers.Sampler.alwaysOff()) .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), @@ -1057,7 +1107,8 @@ void sampling_disabled_producesNoSpans() { .enableMdc(false) .build()); - sampledPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var sampledPlugin = + started(sampledPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); sampledPlugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "step", "STEP", "Step", null, Instant.now(), false, 1)); sampledPlugin.onUserFunctionEnd(new UserFunctionEndInfo( @@ -1099,18 +1150,18 @@ void xrayExtraction_withoutParentDoesNotForceTraceId() { var extractedContext = new ExtractedContext(xrayTraceId, null); spanExporter = InMemorySpanExporter.create(); - var xrayPlugin = new InvocationOtelPlugin( + var xrayPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), OtelPluginConfig.builder() .contextExtractor(() -> extractedContext) .enableMdc(false) .build()); - xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size()); // invocation + Workflow + assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot var invocationSpan = spanByName("Invocation"); var workflowSpan = spanByName("Workflow"); // Remote trace but no parent → a synthetic execution root on the remote trace ID anchors the execution. Both @@ -1131,14 +1182,14 @@ void xrayExtraction_invocationTreeUsesExtractedTraceId_workflowJoinsExecutionTra var extractedContext = new ExtractedContext(xrayTraceId, parentSpanId); spanExporter = InMemorySpanExporter.create(); - var xrayPlugin = new InvocationOtelPlugin( + var xrayPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), OtelPluginConfig.builder() .contextExtractor(() -> extractedContext) .enableMdc(false) .build()); - xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); xrayPlugin.onOperationStart( new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false)); xrayPlugin.onUserFunctionStart( @@ -1185,14 +1236,14 @@ void xrayExtraction_withParentSpanId_invocationSpanHasCorrectParent() { var extractedContext = new ExtractedContext(xrayTraceId, parentSpanId, ExtractedContext.Sampling.SAMPLED); spanExporter = InMemorySpanExporter.create(); - var xrayPlugin = new InvocationOtelPlugin( + var xrayPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), OtelPluginConfig.builder() .contextExtractor(() -> extractedContext) .enableMdc(false) .build()); - xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); @@ -1219,18 +1270,18 @@ void xrayExtraction_withoutParentSpanId_invocationSpanParentsOntoSyntheticRoot() var extractedContext = new ExtractedContext(xrayTraceId, null); spanExporter = InMemorySpanExporter.create(); - var xrayPlugin = new InvocationOtelPlugin( + var xrayPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), OtelPluginConfig.builder() .contextExtractor(() -> extractedContext) .enableMdc(false) .build()); - xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size()); // invocation + Workflow + assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot // Remote trace, no parent → a synthetic execution root on the remote trace ID anchors the execution. The // Invocation span has a valid parent (that synthetic root) and joins the remote trace. @@ -1248,7 +1299,7 @@ void xrayExtraction_multipleInvocations_sameTraceId_unifiedTrace() { var extractedContext = new ExtractedContext(xrayTraceId, "53995c3f42cd8ad8", ExtractedContext.Sampling.SAMPLED); spanExporter = InMemorySpanExporter.create(); - var xrayPlugin = new InvocationOtelPlugin( + var xrayPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), OtelPluginConfig.builder() .contextExtractor(() -> extractedContext) @@ -1256,7 +1307,7 @@ void xrayExtraction_multipleInvocations_sameTraceId_unifiedTrace() { .build()); // First invocation - xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); xrayPlugin.onOperationStart( new OperationInfo("op-1", "step-1", "STEP", "Step", null, Instant.now(), null, null, false)); xrayPlugin.onOperationEnd(new OperationEndInfo( @@ -1275,7 +1326,7 @@ void xrayExtraction_multipleInvocations_sameTraceId_unifiedTrace() { xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.PENDING, null)); // Second invocation (same execution, same X-Ray Root from backend) - xrayPlugin.onInvocationStart(new InvocationInfo("req-2", "arn:exec1", false, Instant.now())); + xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-2", "arn:exec1", false, Instant.now())); xrayPlugin.onOperationStart( new OperationInfo("op-2", "step-2", "STEP", "Step", null, Instant.now(), null, null, false)); xrayPlugin.onOperationEnd(new OperationEndInfo( @@ -1306,7 +1357,7 @@ void xrayExtraction_multipleInvocations_sameTraceId_unifiedTrace() { @Test void xrayExtraction_nullExtractor_sharesArnDerivedExecutionTrace() { spanExporter = InMemorySpanExporter.create(); - var noXrayPlugin = new InvocationOtelPlugin( + var noXrayPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), OtelPluginConfig.builder() .contextExtractor(() -> null) @@ -1314,11 +1365,11 @@ void xrayExtraction_nullExtractor_sharesArnDerivedExecutionTrace() { .build()); var arn = "arn:aws:lambda:us-east-1:123:function:test:$LATEST/durable/exec1"; - noXrayPlugin.onInvocationStart(new InvocationInfo("req-1", arn, true, Instant.now())); + var noXrayPlugin = started(noXrayPluginFactory, new InvocationInfo("req-1", arn, true, Instant.now())); noXrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(2, spans.size()); // invocation + Workflow + assertEquals(3, spans.size()); // invocation + Workflow + DurableExecutionRoot var invocationTraceId = spanByName("Invocation").getTraceId(); var workflowTraceId = spanByName("Workflow").getTraceId(); @@ -1346,14 +1397,14 @@ void xrayExtraction_extractedTraceIdMatchesXrayConversion() { // the spans export. var extractedContext = new ExtractedContext(convertedId, "53995c3f42cd8ad8", ExtractedContext.Sampling.SAMPLED); spanExporter = InMemorySpanExporter.create(); - var xrayPlugin = new InvocationOtelPlugin( + var xrayPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), OtelPluginConfig.builder() .contextExtractor(() -> extractedContext) .enableMdc(false) .build()); - xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); xrayPlugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); assertEquals(expectedOtelTraceId, spanByName("Invocation").getTraceId()); @@ -1365,7 +1416,7 @@ void xrayExtraction_extractedTraceIdMatchesXrayConversion() { @Test void operationEnd_withoutMatchingStart_createsContinuationSpanWithLink() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); // onOperationEnd without a prior onOperationStart — operation completed between invocations plugin.onOperationEnd(new OperationEndInfo( @@ -1385,7 +1436,7 @@ void operationEnd_withoutMatchingStart_createsContinuationSpanWithLink() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); var spans = spanExporter.getFinishedSpanItems(); - assertEquals(3, spans.size()); // continuation + invocation + Workflow + assertEquals(4, spans.size()); // continuation + invocation + Workflow + DurableExecutionRoot var continuationSpan = spans.stream() .filter(s -> s.getName().contains("wait")) @@ -1397,7 +1448,7 @@ void operationEnd_withoutMatchingStart_createsContinuationSpanWithLink() { @Test void operationEnd_withoutMatchingStart_startsWithinCurrentInvocation() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); var operationStart = Instant.EPOCH; var operationEnd = operationStart.plusSeconds(60); @@ -1436,7 +1487,7 @@ void operationEnd_withoutMatchingStart_startsWithinCurrentInvocation() { @Test void operationEnd_withoutMatchingStart_withError_setsErrorStatus() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onOperationEnd(new OperationEndInfo( "op-cb-1", @@ -1466,7 +1517,7 @@ void operationEnd_withoutMatchingStart_withError_setsErrorStatus() { @Test void contextOperation_doesNotCreateAttemptSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); // Create operation span first so the CONTEXT user function has a parent plugin.onOperationStart(new OperationInfo( @@ -1501,7 +1552,7 @@ void contextOperation_doesNotCreateAttemptSpan() { @Test void attemptSpan_endedAtInvocationEnd_whenUserFunctionEndNotCalled() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); // Start attempt but never call onUserFunctionEnd (simulates crash before end hook) plugin.onUserFunctionStart( @@ -1522,7 +1573,7 @@ void attemptSpan_endedAtInvocationEnd_whenUserFunctionEndNotCalled() { @Test void childOperation_parentedToParentOperationSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); // Parent context operation plugin.onOperationStart(new OperationInfo( @@ -1588,7 +1639,7 @@ void multiInvocation_stepWaitStep_producesCorrectSpans() { var startTime = Instant.now(); // Invocation 1: step completes, wait starts - plugin.onInvocationStart(new InvocationInfo("req-1", arn, true, startTime)); + var plugin = started(factory, new InvocationInfo("req-1", arn, true, startTime)); plugin.onOperationStart( new OperationInfo("op-1", "step-A", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onUserFunctionStart( @@ -1622,9 +1673,9 @@ void multiInvocation_stepWaitStep_producesCorrectSpans() { new OperationInfo("op-2", "pause", "WAIT", "Wait", null, Instant.now(), null, null, false)); plugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.PENDING, null)); - // Invocation 1 should have: step op + step attempt + wait (PENDING) + invocation = 4 + // Step op + step attempt + wait (PENDING) + Invocation + DurableExecutionRoot = 5 var inv1Spans = spanExporter.getFinishedSpanItems(); - assertEquals(4, inv1Spans.size()); + assertEquals(5, inv1Spans.size()); var inv1TraceId = inv1Spans.stream() .filter(span -> span.getName().equals("Invocation")) .findFirst() @@ -1639,7 +1690,7 @@ void multiInvocation_stepWaitStep_producesCorrectSpans() { spanExporter.reset(); // Invocation 2: wait completed between invocations, new step runs - plugin.onInvocationStart(new InvocationInfo("req-2", arn, false, startTime)); + plugin = started(factory, new InvocationInfo("req-2", arn, false, startTime)); plugin.onOperationEnd(new OperationEndInfo( "op-2", "pause", @@ -1685,8 +1736,8 @@ void multiInvocation_stepWaitStep_producesCorrectSpans() { plugin.onInvocationEnd(new InvocationEndInfo("req-2", arn, false, InvocationStatus.SUCCEEDED, null)); var inv2Spans = spanExporter.getFinishedSpanItems(); - // wait continuation + step-B op + step-B attempt + invocation + Workflow = 5 - assertEquals(5, inv2Spans.size()); + // Wait continuation + step-B op + step-B attempt + Invocation + Workflow + DurableExecutionRoot = 6 + assertEquals(6, inv2Spans.size()); // Includes DurableExecutionRoot. var inv2TraceId = inv2Spans.stream() .filter(span -> span.getName().equals("Invocation")) @@ -1741,7 +1792,7 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() { var startTime = Instant.now(); // Invocation 1: step starts, attempt 1 fails, invocation suspended during retry poll - plugin.onInvocationStart(new InvocationInfo("req-1", arn, true, startTime)); + var plugin = started(factory, new InvocationInfo("req-1", arn, true, startTime)); plugin.onOperationStart( new OperationInfo("op-1", "process-payment", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onUserFunctionStart( @@ -1761,8 +1812,8 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() { plugin.onInvocationEnd(new InvocationEndInfo("req-1", arn, true, InvocationStatus.PENDING, null)); var inv1Spans = spanExporter.getFinishedSpanItems(); - // operation span (PENDING) + attempt 1 span + invocation span = 3 - assertEquals(3, inv1Spans.size()); + // Operation (PENDING) + attempt 1 + Invocation + DurableExecutionRoot = 4 + assertEquals(4, inv1Spans.size()); var inv1OperationSpan = inv1Spans.stream() .filter(s -> @@ -1783,7 +1834,7 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() { spanExporter.reset(); // Invocation 2: step is replayed (continuation), attempt 2 executes and succeeds - plugin.onInvocationStart(new InvocationInfo("req-2", arn, false, startTime)); + plugin = started(factory, new InvocationInfo("req-2", arn, false, startTime)); // isReplay=true: this operation already exists in the execution state plugin.onOperationStart( new OperationInfo("op-1", "process-payment", "STEP", "Step", null, Instant.now(), null, null, true)); @@ -1817,8 +1868,8 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() { plugin.onInvocationEnd(new InvocationEndInfo("req-2", arn, false, InvocationStatus.SUCCEEDED, null)); var inv2Spans = spanExporter.getFinishedSpanItems(); - // operation span + attempt 2 span + invocation span + Workflow span = 4 - assertEquals(4, inv2Spans.size()); + // Operation + attempt 2 + Invocation + Workflow + DurableExecutionRoot = 5 + assertEquals(5, inv2Spans.size()); // Includes DurableExecutionRoot. var inv2OperationSpan = inv2Spans.stream() .filter(s -> @@ -1881,7 +1932,7 @@ void crossInvocation_stepRetry_attemptsParentedToRespectiveInvocations() { @Test void workflowSpan_exportedOnTerminal_internal_deterministicId() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec-wf", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec-wf", true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec-wf", true, InvocationStatus.SUCCEEDED, null)); var workflow = spanByName("Workflow"); @@ -1892,7 +1943,7 @@ void workflowSpan_exportedOnTerminal_internal_deterministicId() { @Test void workflowSpan_notExportedOnNonTerminal() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.PENDING, null)); assertTrue( @@ -1904,7 +1955,7 @@ void workflowSpan_notExportedOnNonTerminal() { @Test void workflowSpan_notExportedOnRetrying() { // RETRYING is non-terminal, so the deferred Workflow span is neither materialized nor abandoned. - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo( "req-1", "arn:exec1", true, InvocationStatus.RETRYING, new RuntimeException("transient"))); @@ -1918,7 +1969,7 @@ void workflowSpan_notExportedOnRetrying() { void deferredWorkflowSpan_whenExported_isEnded_andMatchesLinkedSpanId() { // The Workflow span is created only at the terminal invocation, but operations that ran earlier linked to its // deterministic context. When it is finally exported it must be ended and carry that same span ID. - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec-wf", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec-wf", true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onOperationEnd(new OperationEndInfo( @@ -1947,7 +1998,7 @@ void deferredWorkflowSpan_whenExported_isEnded_andMatchesLinkedSpanId() { @Test void operationAndAttemptSpans_linkToWorkflowSpan() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec-wf", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec-wf", true, Instant.now())); plugin.onOperationStart( new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false)); plugin.onUserFunctionStart( @@ -1994,7 +2045,7 @@ void operationAndAttemptSpans_linkToWorkflowSpan() { void operationLinksToWorkflow_withXRayContext() { // "Other case": invocation span is parented to the X-Ray segment, but operation spans still link to Workflow. var exporter = InMemorySpanExporter.create(); - var xrayPlugin = new InvocationOtelPlugin( + var xrayPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), OtelPluginConfig.builder() .contextExtractor(() -> new ExtractedContext( @@ -2003,7 +2054,7 @@ void operationLinksToWorkflow_withXRayContext() { ExtractedContext.Sampling.SAMPLED)) .enableMdc(false) .build()); - xrayPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var xrayPlugin = started(xrayPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); xrayPlugin.onOperationStart( new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false)); xrayPlugin.onOperationEnd(new OperationEndInfo( @@ -2039,14 +2090,14 @@ void operationLinksToWorkflow_withXRayContext() { @Test void workflowSpanName_isConfigurable() { var exporter = InMemorySpanExporter.create(); - var customPlugin = new InvocationOtelPlugin( + var customPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)), OtelPluginConfig.builder() .contextExtractor(() -> null) .enableMdc(false) .workflowSpanName("MyWorkflow") .build()); - customPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var customPlugin = started(customPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); customPlugin.onInvocationEnd( new InvocationEndInfo("req-1", "arn:exec1", true, InvocationStatus.SUCCEEDED, null)); @@ -2062,7 +2113,7 @@ void workflowSpanName_isConfigurable() { @Test void failedInvocation_setsErrorOnBothWorkflowAndInvocationSpans() { - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var plugin = started(factory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); plugin.onInvocationEnd(new InvocationEndInfo( "req-1", "arn:exec1", true, InvocationStatus.FAILED, new RuntimeException("boom"))); @@ -2163,14 +2214,15 @@ void noRecordingSpanIsLeftOpen_onRetrying_trackedByLifecycleProcessor() { */ private void assertNoOpenSpansOnNonTerminal(InvocationStatus status) { var lifecycle = new LifecycleTrackingSpanProcessor(); - var trackingPlugin = new InvocationOtelPlugin( + var trackingPluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(lifecycle), OtelPluginConfig.builder() .contextExtractor(() -> null) .enableMdc(false) .build()); - trackingPlugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); + var trackingPlugin = + started(trackingPluginFactory, new InvocationInfo("req-1", "arn:exec1", true, Instant.now())); trackingPlugin.onOperationStart( new OperationInfo("op-1", "step-a", "STEP", "Step", null, Instant.now(), null, null, false)); trackingPlugin.onUserFunctionStart( diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/Invocations.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/Invocations.java new file mode 100644 index 000000000..ed9a21e33 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/Invocations.java @@ -0,0 +1,27 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationInfo; + +/** + * Test helper: builds one invocation's plugin instance the way the SDK does. + * + *

    A plugin instance serves exactly one invocation, so a test that drives several invocations of an execution creates + * one instance per invocation from the same factory — the factory being what the environment owns. The factory is + * called with the very {@link InvocationInfo} that {@code onInvocationStart} then receives, exactly as + * {@code PluginRunner} does. + */ +final class Invocations { + + private Invocations() {} + + /** One invocation's plugin instance, created from the factory and started with the same info. */ + static DurableExecutionPlugin started(DurableExecutionPluginFactory factory, InvocationInfo info) { + var plugin = factory.createPlugin(info); + plugin.onInvocationStart(info); + return plugin; + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacyEndThreadProbeTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacyEndThreadProbeTest.java new file mode 100644 index 000000000..dd74fccbb --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacyEndThreadProbeTest.java @@ -0,0 +1,188 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +/** Identical public-API timing probe run against main and the candidate. */ +class LegacyEndThreadProbeTest { + @ParameterizedTest + @CsvSource({ + "false,false,false", "true,false,false", "false,true,false", "true,true,false", + "false,false,true", "true,false,true", "false,true,true", "true,true,true" + }) + void legacyEndThread(boolean mixed, boolean precompleted, boolean pending) throws Exception { + var local = new ThreadLocal(); + var observations = new CopyOnWriteArrayList(); + var endValues = new ArrayList(); + var callerThread = new AtomicReference(); + var caller = Executors.newSingleThreadExecutor(task -> { + var thread = daemon(task, "legacy-probe-caller"); + callerThread.set(thread); + return thread; + }); + var workers = + new ThreadPoolExecutor( + 1, + 1, + 0, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + task -> daemon(task, "legacy-probe-worker")) { + @Override + public void execute(Runnable task) { + var finished = new CountDownLatch(1); + super.execute(() -> { + try { + task.run(); + } finally { + finished.countDown(); + } + }); + if (precompleted) await(finished); + } + }; + try { + for (int round = 1; round <= 2; round++) { + var marker = "inv" + round; + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + var endValue = new AtomicReference(); + var startThread = new AtomicReference(); + var endThread = new AtomicReference(); + var previous = new AtomicReference(); + var legacy = new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo info) { + startThread.set(Thread.currentThread()); + previous.set(local.get()); + local.set(marker); + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + endThread.set(Thread.currentThread()); + endValue.set(local.get()); + local.remove(); + } + }; + var plugins = new ArrayList(); + plugins.add(info -> legacy); + if (mixed) + plugins.add(InvocationOtelPlugin.factory( + SdkTracerProvider.builder(), + OtelPluginConfig.builder().enableMdc(false).build())); + var config = DurableConfig.builder() + .withExecutorService(workers) + .withDurableExecutionClient(new LocalMemoryExecutionClient()) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(plugins.toArray(DurableExecutionPluginFactory[]::new)) + .build(); + var result = caller.submit(() -> DurableExecutor.execute( + input(marker), + null, + TypeToken.get(String.class), + (value, context) -> { + entered.countDown(); + if (!precompleted) await(release); + if (pending) context.wait("pause", Duration.ofSeconds(1)); + return "done"; + }, + config)); + try { + assertTrue(entered.await(3, TimeUnit.SECONDS)); + if (!precompleted) { + awaitCallerJoin(callerThread.get()); + release.countDown(); + } + assertEquals( + pending ? ExecutionStatus.PENDING : ExecutionStatus.SUCCEEDED, + result.get(5, TimeUnit.SECONDS).status()); + var after = workers.submit(local::get).get(3, TimeUnit.SECONDS); + observations.add( + "round=" + round + " start=" + startThread.get().getName() + + " end=" + endThread.get().getName() + " endValue=" + endValue.get() + + " workerAfter=" + after + " priorWorker=" + previous.get()); + endValues.add(endValue.get()); + } finally { + release.countDown(); + } + } + System.out.println("LEGACY_END_TRACE mixed=" + mixed + " precompleted=" + precompleted + " pending=" + + pending + " " + observations); + // This controlled success path attaches the legacy continuation before the handler completes. + // Precompleted and suspension paths are measured separately, without claiming owner-thread guarantees. + if (!precompleted && !pending) assertEquals(List.of("inv1", "inv2"), endValues); + } finally { + caller.shutdownNow(); + workers.shutdownNow(); + assertTrue(caller.awaitTermination(3, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } + + private static void awaitCallerJoin(Thread caller) { + var end = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (System.nanoTime() < end) { + if (caller.getState() == Thread.State.WAITING + && Arrays.stream(caller.getStackTrace()) + .anyMatch(frame -> frame.getClassName().equals(CompletableFuture.class.getName()) + && frame.getMethodName().equals("join"))) return; + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + } + fail("invocation caller did not reach its future join before releasing the handler"); + } + + private static DurableExecutionInput input(String marker) { + var operation = Operation.builder() + .id("id") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/" + marker + "/id", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } + + private static Thread daemon(Runnable task, String name) { + var thread = new Thread(task, name); + thread.setDaemon(true); + return thread; + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new AssertionError(error); + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacySubclassMigrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacySubclassMigrationTest.java new file mode 100644 index 000000000..e78a66f6e --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LegacySubclassMigrationTest.java @@ -0,0 +1,81 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import javax.tools.ToolProvider; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class LegacySubclassMigrationTest { + @TempDir + Path directory; + + @ParameterizedTest + @CsvSource({"Invocation,false", "Invocation,true", "Execution,false", "Execution,true"}) + void requiresFactoryMigrationForOldCompiledSubclasses(String view, boolean ownMethod) throws Exception { + var baseline = Files.createDirectories(directory.resolve("baseline")); + var classes = Files.createDirectories(directory.resolve("classes")); + var baseName = view + "OtelPlugin"; + var oldBase = directory.resolve(baseName + ".java"); + Files.writeString(oldBase, "package software.amazon.lambda.durable.otel; public class " + baseName + " {}"); + var compiler = ToolProvider.getSystemJavaCompiler(); + assertEquals( + 0, compiler.run(null, null, null, "--release", "17", "-d", baseline.toString(), oldBase.toString())); + var source = directory.resolve("LegacySubclass.java"); + var method = "public AutoCloseable openHandlerScope() { opened++; return () -> closed++; }"; + Files.writeString(source, """ + import software.amazon.lambda.durable.otel.%s; + interface ApplicationScope { + default AutoCloseable openHandlerScope() { + LegacySubclass.opened++; + return () -> LegacySubclass.closed++; + } + } + public class LegacySubclass extends %s implements ApplicationScope { + public static int opened, closed; + %s + public void originalCall() throws Exception { try (var scope = openHandlerScope()) {} } + } + """.formatted(baseName, baseName, ownMethod ? method : "")); + assertEquals( + 0, + compiler.run( + null, + null, + null, + "--release", + "17", + "-cp", + baseline.toString(), + "-d", + classes.toString(), + source.toString())); + try (var loader = new URLClassLoader( + new URL[] {classes.toUri().toURL()}, getClass().getClassLoader())) { + assertThrows( + IncompatibleClassChangeError.class, + () -> loader.loadClass("LegacySubclass"), + "3.x view implementations are final; old subclasses must migrate to factories"); + } + assertNotEquals( + 0, + compiler.run( + null, + null, + null, + "--release", + "17", + "-cp", + System.getProperty("java.class.path"), + "-d", + classes.toString(), + source.toString())); + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java index 9c4399817..88c9a50c4 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcSpanEnricherTest.java @@ -3,11 +3,14 @@ package software.amazon.lambda.durable.otel; import static org.junit.jupiter.api.Assertions.*; +import static software.amazon.lambda.durable.otel.Invocations.started; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.time.Instant; +import java.util.List; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.slf4j.MDC; @@ -56,14 +59,14 @@ void inject_withNoActiveSpan_doesNotSetMdcFields() { void plugin_withMdcEnabled_setsFieldsInMdc() { var spanExporter = InMemorySpanExporter.create(); - var plugin = new InvocationOtelPlugin( + var pluginFactory = InvocationOtelPlugin.factory( SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), OtelPluginConfig.builder() .contextExtractor(() -> null) .enableMdc(true) .build()); - plugin.onInvocationStart(new InvocationInfo("req-1", "arn:exec-mdc-test", true, Instant.now())); + var plugin = started(pluginFactory, new InvocationInfo("req-1", "arn:exec-mdc-test", true, Instant.now())); plugin.onUserFunctionStart( new UserFunctionStartInfo("op-1", "step", "STEP", "Step", null, Instant.now(), false, 1)); @@ -99,4 +102,41 @@ void plugin_withMdcEnabled_setsFieldsInMdc() { assertNull(MDC.get(MdcSpanEnricher.MDC_SPAN_ID)); assertNull(MDC.get(MdcSpanEnricher.MDC_TRACE_SAMPLED)); } + + @Test + void logCorrelationFollowsEachInvocationsOwnInstance() { + // Regression guard taken from the Python port of this refactor: there a log filter installed by the first + // invocation's plugin outlived that plugin and kept querying the discarded instance, so log correlation + // silently stopped after the first invocation. Java correlates through the SLF4J MDC, written by the hooks of + // whichever instance is serving the invocation, so every invocation publishes its own execution trace. This + // test pins that down across two invocations served by two instances of one factory. + var spanExporter = InMemorySpanExporter.create(); + var pluginFactory = InvocationOtelPlugin.factory( + SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(spanExporter)), + OtelPluginConfig.builder() + .contextExtractor(() -> null) + .enableMdc(true) + .build()); + + var first = started(pluginFactory, new InvocationInfo("req-1", "arn:exec-a", true, Instant.now())); + var firstTraceId = MDC.get(MdcSpanEnricher.MDC_TRACE_ID); + first.onInvocationEnd(new InvocationEndInfo("req-1", "arn:exec-a", true, InvocationStatus.SUCCEEDED, null)); + assertNull(MDC.get(MdcSpanEnricher.MDC_TRACE_ID), "an invocation clears the correlation it set"); + + var second = started(pluginFactory, new InvocationInfo("req-2", "arn:exec-b", true, Instant.now())); + var secondTraceId = MDC.get(MdcSpanEnricher.MDC_TRACE_ID); + second.onInvocationEnd(new InvocationEndInfo("req-2", "arn:exec-b", true, InvocationStatus.SUCCEEDED, null)); + + assertNotNull(firstTraceId); + assertNotNull(secondTraceId, "log correlation must not stop after the first invocation"); + assertNotEquals(firstTraceId, secondTraceId, "each instance publishes its own execution trace"); + var invocationTraceIds = spanExporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().equals("Invocation")) + .map(SpanData::getTraceId) + .toList(); + assertEquals( + List.of(firstTraceId, secondTraceId), + invocationTraceIds, + "the correlated trace ID is the one on that invocation's own Invocation span"); + } } diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginFactoryConfigTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginFactoryConfigTest.java new file mode 100644 index 000000000..23675c8d0 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/OtelPluginFactoryConfigTest.java @@ -0,0 +1,63 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import org.junit.jupiter.api.Test; + +/** + * Covers registration-time rejection of a null config on both plugins' factory overloads. + * + *

    The global-provider overload only stores the config, so a null one used to be dereferenced when an invocation's + * plugin instance was built. {@code PluginRunner} contains a factory failure, so the function ran without the telemetry + * it had asked for and reported one warning per invocation. Registration is where a caller can still act on it. + */ +class OtelPluginFactoryConfigTest { + + @Test + void invocationPluginRejectsANullConfigOnTheGlobalProviderOverload() { + var error = + assertThrows(NullPointerException.class, () -> InvocationOtelPlugin.factory((OtelPluginConfig) null)); + + assertTrue(error.getMessage().contains("config"), error.getMessage()); + } + + @Test + void executionPluginRejectsANullConfigOnTheGlobalProviderOverload() { + var error = + assertThrows(NullPointerException.class, () -> ExecutionOtelPlugin.factory((OtelPluginConfig) null)); + + assertTrue(error.getMessage().contains("config"), error.getMessage()); + } + + @Test + void invocationPluginRejectsANullConfigOnTheProviderBuilderOverload() { + assertThrows(NullPointerException.class, () -> InvocationOtelPlugin.factory(SdkTracerProvider.builder(), null)); + } + + @Test + void executionPluginRejectsANullConfigOnTheProviderBuilderOverload() { + assertThrows(NullPointerException.class, () -> ExecutionOtelPlugin.factory(SdkTracerProvider.builder(), null)); + } + + @Test + void aRejectedConfigLeavesTheBuilderUsable() { + // The check is the first statement of forProviderBuilder, so it precedes the ID-generator and sampler + // installation and the provider build. That ordering matters because a provider built and then thrown away is + // unreachable: its span processors and their worker threads are never shut down. The ordering itself is a + // property of the source rather than something this test can observe -- SdkTracerProviderBuilder exposes no + // getters -- so what is asserted here is the consequence a caller can see: the builder they passed still + // works. + var builder = SdkTracerProvider.builder(); + + assertThrows(NullPointerException.class, () -> InvocationOtelPlugin.factory(builder, null)); + + try (var provider = builder.build()) { + assertNotNull(provider.get("probe")); + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PluginStartupOrderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PluginStartupOrderTest.java new file mode 100644 index 000000000..3f6b574ee --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/PluginStartupOrderTest.java @@ -0,0 +1,64 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.time.Instant; +import java.util.List; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.plugin.PluginRunner; + +class PluginStartupOrderTest { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void earlierInitializerContextIsAvailableToOtelFactory(boolean executionView) { + var carrier = new ThreadLocal(); + var expected = new ExtractedContext( + "6955b900123456789012345678901234", "1234567890123456", ExtractedContext.Sampling.SAMPLED); + DurableExecutionPluginFactory initializer = info -> new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo info) { + carrier.set(expected); + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + carrier.remove(); + } + }; + var exporter = InMemorySpanExporter.create(); + var builder = SdkTracerProvider.builder() + .setSampler(Sampler.alwaysOff()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var config = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(carrier::get) + .build(); + var otel = executionView + ? ExecutionOtelPlugin.factory(builder, config) + : InvocationOtelPlugin.factory(builder, config); + var runner = new PluginRunner(List.of(initializer, otel)); + try { + runner.onInvocationStart(new InvocationInfo("request", "arn", true, Instant.EPOCH)); + runner.onInvocationEnd(new InvocationEndInfo("request", "arn", true, InvocationStatus.SUCCEEDED, null)); + var spans = exporter.getFinishedSpanItems(); + assertEquals(2, spans.size(), "Initializer's sampled context must override the always-off fallback"); + assertTrue(spans.stream().allMatch(span -> expected.traceId().equals(span.getTraceId()))); + assertTrue(spans.stream().allMatch(span -> expected.parentSpanId().equals(span.getParentSpanId()))); + } finally { + carrier.remove(); + runner.releasePlugins(); + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java new file mode 100644 index 000000000..0564f007a --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RootStartProcessorIsolationTest.java @@ -0,0 +1,187 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; + +class RootStartProcessorIsolationTest { + private static final String TRACE = "6955b900123456789012345678901234"; + private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id"; + private static final Instant START = Instant.parse("2026-10-02T00:00:00Z"); + + static Stream cases() { + return Stream.of(true, false) + .flatMap(view -> Stream.of(true, false) + .flatMap(sampled -> Stream.of(true, false) + .flatMap(shared -> Stream.of(true, false) + .flatMap(recordOnly -> Stream.of(true, false) + .map(forwardParent -> Arguments.of( + view, sampled, shared, recordOnly, forwardParent)))))); + } + + @ParameterizedTest + @MethodSource("cases") + void rootStartCallbacksRetainTheirOwnSamplingAndRandomIds( + boolean executionView, + boolean sampled, + boolean sharedGenerator, + boolean rootRecordOnly, + boolean forwardParent) { + var observer = new StartObserver(forwardParent); + var policy = sampled ? Sampler.alwaysOn() : Sampler.alwaysOff(); + var builder = SdkTracerProvider.builder() + .setSampler(rootSampler(rootRecordOnly, policy)) + .addSpanProcessor(observer); + var config = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(() -> new ExtractedContext( + TRACE, + null, + rootRecordOnly ? ExtractedContext.Sampling.UNDECIDED : ExtractedContext.Sampling.SAMPLED)) + .build(); + DurableExecutionPluginFactory plugin = executionView + ? ExecutionOtelPlugin.factory(builder, config) + : InvocationOtelPlugin.factory(builder, config); + // Rebuilding this configured builder shares the plugin-installed generator; the other branch exercises the + // same thread-scoped bridge through an independent generator instance. + try (var unrelated = sharedGenerator + ? builder.build() + : SdkTracerProvider.builder() + .setSampler(DurableSampler.wrap(policy)) + .setIdGenerator(new DeterministicIdGenerator()) + .build()) { + observer.tracer = unrelated.get("processor"); + var original = Context.current(); + assertControl(observer.tracer, sampled); + invoke(plugin, true); + invoke(plugin, false); + assertControl(observer.tracer, sampled); + assertSame(original, Context.current()); + assertObservations(observer, sampled); + } + } + + private static void invoke(DurableExecutionPluginFactory plugin, boolean first) { + var instance = plugin.createPlugin(new InvocationInfo("request", ARN, first, START)); + instance.onInvocationEnd(new InvocationEndInfo("request", ARN, first, InvocationStatus.PENDING, null)); + } + + private static void assertControl(Tracer tracer, boolean sampled) { + var span = tracer.spanBuilder("control").setNoParent().startSpan(); + assertEquals(sampled, span.getSpanContext().isSampled()); + assertEquals(sampled, span.isRecording()); + span.end(); + } + + private static void assertObservations(StartObserver observer, boolean sampled) { + assertEquals(2, observer.roots.size()); + assertEquals(observer.roots.get(0), observer.roots.get(1)); + var root = observer.roots.get(0); + assertEquals(new DeterministicIdGenerator().generateExecutionRootSpanId(ARN), root.getSpanId()); + assertEquals(2, observer.callbacks.size()); + assertEquals(List.of(sampled, sampled), observer.recording); + for (var callback : observer.callbacks) { + assertEquals(sampled, callback.isSampled()); + assertTrue(callback.isValid()); + assertNotEquals(root.getTraceId(), callback.getTraceId()); + assertNotEquals(root.getSpanId(), callback.getSpanId()); + } + assertEquals( + 2, + observer.callbacks.stream() + .map(SpanContext::getTraceId) + .distinct() + .count()); + assertEquals( + 2, + observer.callbacks.stream() + .map(SpanContext::getSpanId) + .distinct() + .count()); + } + + private static Sampler rootSampler(boolean recordOnly, Sampler policy) { + if (!recordOnly) return policy; + return new Sampler() { + @Override + public SamplingResult shouldSample( + Context parent, + String trace, + String name, + SpanKind kind, + Attributes attributes, + List links) { + return name.equals("Workflow") + ? SamplingResult.recordOnly() + : policy.shouldSample(parent, trace, name, kind, attributes, links); + } + + @Override + public String getDescription() { + return "RecordOnlyWorkflow"; + } + }; + } + + private static final class StartObserver implements SpanProcessor { + private Tracer tracer; + private final boolean forwardParent; + + private StartObserver(boolean forwardParent) { + this.forwardParent = forwardParent; + } + + private final List roots = new ArrayList<>(); + private final List callbacks = new ArrayList<>(); + private final List recording = new ArrayList<>(); + + @Override + public void onStart(Context parent, ReadWriteSpan span) { + // The name guard permits exactly one unrelated span per root and prevents processor recursion. + if (!span.getName().equals("DurableExecutionRoot")) return; + roots.add(span.getSpanContext()); + var builder = tracer.spanBuilder("unrelated-onStart"); + var callback = (forwardParent ? builder.setParent(parent) : builder.setNoParent()).startSpan(); + callbacks.add(callback.getSpanContext()); + recording.add(callback.isRecording()); + callback.end(); + } + + @Override + public boolean isStartRequired() { + return true; + } + + @Override + public void onEnd(ReadableSpan span) {} + + @Override + public boolean isEndRequired() { + return false; + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RuntimeHeaderFailureIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RuntimeHeaderFailureIntegrationTest.java new file mode 100644 index 000000000..1e9528488 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/RuntimeHeaderFailureIntegrationTest.java @@ -0,0 +1,140 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazonaws.services.lambda.runtime.Context; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.time.Instant; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.ExecutionDetails; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.client.DurableExecutionClient; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; + +class RuntimeHeaderFailureIntegrationTest { + @Test + void noPluginsDoesNotReadRuntimeTraceHeader() { + executeWithThrowingRuntimeOverride(null, new AssertionError("must not read")); + } + + @ParameterizedTest + @CsvSource({ + "true,0,runtime", + "true,0,assertion", + "true,0,class-linkage", + "true,0,method-linkage", + "true,0,abstract-linkage", + "true,1,runtime", + "true,1,assertion", + "true,1,class-linkage", + "true,1,method-linkage", + "true,1,abstract-linkage", + "false,0,runtime", + "false,0,assertion", + "false,0,class-linkage", + "false,0,method-linkage", + "false,0,abstract-linkage", + "false,1,runtime", + "false,1,assertion", + "false,1,class-linkage", + "false,1,method-linkage", + "false,1,abstract-linkage" + }) + void throwingRuntimeOverrideCannotBorrowStaleGlobalCarrier( + boolean executionView, String sampled, String failureKind) { + var property = "com.amazonaws.xray.traceHeader"; + var previous = System.getProperty(property); + var stale = "Root=1-6955b900-aaaaaaaaaaaaaaaaaaaaaaaa;Sampled=" + sampled; + System.setProperty(property, stale); + try { + var exporter = InMemorySpanExporter.create(); + var builder = SdkTracerProvider.builder() + .setSampler(Sampler.alwaysOn()) + .addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var config = OtelPluginConfig.builder().enableMdc(false).build(); + DurableExecutionPluginFactory plugin = executionView + ? ExecutionOtelPlugin.factory(builder, config) + : InvocationOtelPlugin.factory(builder, config); + executeWithThrowingRuntimeOverride(plugin, accessorFailure(failureKind)); + var spans = exporter.getFinishedSpanItems(); + assertEquals(3, spans.size(), "a failed invocation accessor must not inherit stale sampling"); + // This factory branch already exports its synthetic ancestor in addition to Workflow and Invocation. + assertEquals( + List.of("DurableExecutionRoot", "Invocation", "Workflow"), + spans.stream().map(span -> span.getName()).sorted().toList()); + assertTrue( + spans.stream().allMatch(span -> !"6955b900aaaaaaaaaaaaaaaaaaaaaaaa".equals(span.getTraceId())), + "a failed invocation accessor must not inherit another request's trace"); + assertEquals(stale, System.getProperty(property)); + } finally { + if (previous == null) System.clearProperty(property); + else System.setProperty(property, previous); + } + } + + private static void executeWithThrowingRuntimeOverride(DurableExecutionPluginFactory plugin, Throwable failure) { + var context = mock(RuntimeContext.class); + when(context.getXrayTraceId()).thenAnswer(invocation -> { + throw failure; + }); + when(context.getRemainingTimeInMillis()).thenReturn(30000); + var operation = Operation.builder() + .id("id") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + var input = new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/name/id", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + var builder = DurableConfig.builder().withDurableExecutionClient(mock(DurableExecutionClient.class)); + if (plugin != null) builder.withPlugins(plugin); + var config = builder.build(); + var result = + DurableExecutor.execute(input, context, TypeToken.get(String.class), (value, ctx) -> value, config); + assertEquals(ExecutionStatus.SUCCEEDED, result.status()); + assertEquals("\"input\"", result.result()); + if (plugin == null) verify(context, never()).getXrayTraceId(); + } + + private static Throwable accessorFailure(String kind) { + return switch (kind) { + case "runtime" -> new SecurityException("runtime carrier access denied"); + case "assertion" -> new AssertionError("optional runtime assertion"); + case "class-linkage" -> new NoClassDefFoundError("optional dependency"); + case "method-linkage" -> new NoSuchMethodError("inside available override"); + case "abstract-linkage" -> new AbstractMethodError("inside available override"); + default -> throw new IllegalArgumentException(kind); + }; + } + + private abstract static class RuntimeContext implements Context { + @Override + public String getXrayTraceId() { + return null; + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionStartCleanupTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionStartCleanupTest.java new file mode 100644 index 000000000..e8b8ead5c --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionStartCleanupTest.java @@ -0,0 +1,145 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +@Timeout(10) +class UserFunctionStartCleanupTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void partialStartFailureRestoresContextOnReusableDirectThread(boolean executionView) throws Exception { + var thread = Executors.newSingleThreadExecutor(task -> daemon(task, "reused-direct-probe")); + var fatal = new InternalError("later start hook"); + var starts = new AtomicInteger(); + var ends = new AtomicInteger(); + var settings = OtelPluginConfig.builder() + .enableMdc(false) + .contextExtractor(() -> new ExtractedContext( + "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED)) + .build(); + var otel = executionView + ? ExecutionOtelPlugin.factory(SdkTracerProvider.builder(), settings) + : InvocationOtelPlugin.factory(SdkTracerProvider.builder(), settings); + DurableExecutionPluginFactory observer = info -> new DurableExecutionPlugin() { + public void onUserFunctionStart(UserFunctionStartInfo start) { + starts.incrementAndGet(); + } + + public void onUserFunctionEnd(UserFunctionEndInfo end) { + ends.incrementAndGet(); + } + }; + DurableExecutionPluginFactory faulty = info -> new DurableExecutionPlugin() { + public void onUserFunctionStart(UserFunctionStartInfo start) { + throw fatal; + } + }; + try { + thread.submit(() -> { + assertFalse(Span.current().getSpanContext().isValid()); + assertSame( + fatal, + assertThrows( + InternalError.class, + () -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, ctx) -> { + ctx.stepAsync("trigger", String.class, step -> "unreachable"); + return "done"; + }, + config( + new LocalMemoryExecutionClient(), + new DirectExecutor(), + otel, + observer, + faulty)))); + }) + .get(5, TimeUnit.SECONDS); + assertEquals(1, starts.get()); + assertEquals(1, ends.get()); + var reusedSpan = + thread.submit(() -> Span.current().getSpanContext()).get(2, TimeUnit.SECONDS); + assertFalse(reusedSpan.isValid(), "attempt cleanup must not restore an ended handler span"); + } finally { + thread.shutdownNow(); + thread.awaitTermination(2, TimeUnit.SECONDS); + } + } + + private static DurableConfig config( + LocalMemoryExecutionClient client, ExecutorService workers, DurableExecutionPluginFactory... plugins) { + return DurableConfig.builder() + .withDurableExecutionClient(client) + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(plugins) + .build(); + } + + private static DurableExecutionInput input() { + var operation = Operation.builder() + .id("id") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } + + private static Thread daemon(Runnable task, String name) { + var thread = new Thread(task, name); + thread.setDaemon(true); + thread.setUncaughtExceptionHandler((owner, error) -> {}); + return thread; + } + + private static final class DirectExecutor extends AbstractExecutorService { + public void execute(Runnable task) { + task.run(); + } + + public void shutdown() {} + + public List shutdownNow() { + return List.of(); + } + + public boolean isShutdown() { + return false; + } + + public boolean isTerminated() { + return false; + } + + public boolean awaitTermination(long timeout, TimeUnit unit) { + return true; + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayContextExtractorTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayContextExtractorTest.java index 0a02a66e9..e71a3b45f 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayContextExtractorTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayContextExtractorTest.java @@ -4,12 +4,101 @@ import static org.junit.jupiter.api.Assertions.*; +import java.time.Instant; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.NullAndEmptySource; import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.plugin.InvocationInfo; class XRayContextExtractorTest { + private static final InvocationInfo INFO = new InvocationInfo("request", "arn", true, Instant.EPOCH); + + @ParameterizedTest + @ValueSource(strings = {"0", "1"}) + void invocationHeaderOverridesGlobalCarrier(String sampled) { + System.setProperty("com.amazonaws.xray.traceHeader", "Root=1-ffffffff-ffffffffffffffffffffffff;Sampled=1"); + try { + var header = "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=" + sampled; + var extracted = new XRayContextExtractor().extract(invocation(header)); + assertEquals("6955b900123456789012345678901234", extracted.traceId()); + assertEquals("1234567890123456", extracted.parentSpanId()); + assertEquals( + sampled.equals("1") ? ExtractedContext.Sampling.SAMPLED : ExtractedContext.Sampling.NOT_SAMPLED, + extracted.sampling()); + assertNull(new XRayContextExtractor().extract(invocation("malformed"))); + } finally { + System.clearProperty("com.amazonaws.xray.traceHeader"); + } + } + + @ParameterizedTest + @NullAndEmptySource + void unavailableAccessorFallsBackButCapturedMissingHeaderDoesNot(String header) { + System.setProperty("com.amazonaws.xray.traceHeader", "Root=1-6955b900-123456789012345678901234;Sampled=0"); + try { + var extracted = new XRayContextExtractor().extract(invocation(header)); + if (header == null) assertEquals(ExtractedContext.Sampling.NOT_SAMPLED, extracted.sampling()); + else assertNull(extracted, "an available but empty runtime snapshot must not borrow the global carrier"); + } finally { + System.clearProperty("com.amazonaws.xray.traceHeader"); + } + } + + @Test + void existingCustomExtractorStillReceivesOneCall() { + var expected = + new ExtractedContext("6955b900123456789012345678901234", null, ExtractedContext.Sampling.SAMPLED); + ContextExtractor custom = () -> expected; + assertSame(expected, custom.extract(invocation("ignored"))); + } + + @Test + void existingNoArgumentSubclassOverrideKeepsControlWithInvocationHeader() { + var expected = + new ExtractedContext("6955b900aaaaaaaaaaaaaaaaaaaaaaaa", null, ExtractedContext.Sampling.NOT_SAMPLED); + var extractor = new XRayContextExtractor() { + @Override + public ExtractedContext extract() { + return expected; + } + }; + assertSame( + expected, + extractor.extract( + invocation("Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=1"))); + } + + @Test + void delegatingSubclassUsesInvocationHeaderAndClearsItAfterExtraction() { + var extractor = new XRayContextExtractor() { + @Override + public ExtractedContext extract() { + var extracted = super.extract(); + if (extracted != null && extracted.sampling() == ExtractedContext.Sampling.NOT_SAMPLED) { + throw new IllegalArgumentException("custom extraction failure"); + } + return extracted; + } + }; + var header = "Root=1-6955b900-123456789012345678901234;Sampled="; + assertNotNull(extractor.extract(invocation(header + "1"))); + assertNull(extractor.extract()); + assertThrows(IllegalArgumentException.class, () -> extractor.extract(invocation(header + "0"))); + assertNull(extractor.extract()); + } + + private static InvocationInfo invocation(String header) { + return new InvocationInfo( + INFO.requestId(), + INFO.durableExecutionArn(), + INFO.isFirstInvocation(), + INFO.executionStartTime(), + INFO.executionInput(), + INFO.operations(), + INFO.updatedOperations(), + header); + } @Test void extract_withoutEnvVar_returnsNull() { diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayExtractorBinaryCompatibilityTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayExtractorBinaryCompatibilityTest.java new file mode 100644 index 000000000..eec55724c --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/XRayExtractorBinaryCompatibilityTest.java @@ -0,0 +1,85 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Map; +import javax.tools.ToolProvider; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import software.amazon.lambda.durable.plugin.InvocationInfo; + +class XRayExtractorBinaryCompatibilityTest { + @Test + void subclassCompiledAgainstLegacyApiRetainsOverride(@TempDir Path directory) throws Exception { + var baseline = directory.resolve("baseline"); + var consumer = directory.resolve("consumer"); + var classpath = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")); + compile(baseline, "XRayContextExtractor", """ + package software.amazon.lambda.durable.otel; + public class XRayContextExtractor { + public ExtractedContext extract() { return null; } + } + """, classpath); + compile(consumer, "LegacyExtractor", """ + import software.amazon.lambda.durable.otel.*; + public class LegacyExtractor extends XRayContextExtractor { + public int calls; + @Override public ExtractedContext extract() { + calls++; + return new ExtractedContext("6955b900aaaaaaaaaaaaaaaaaaaaaaaa", null, + ExtractedContext.Sampling.NOT_SAMPLED); + } + } + """, baseline + File.pathSeparator + classpath); + verifyConsumer(consumer); + } + + private void verifyConsumer(Path consumer) throws Exception { + // Only the consumer is loaded from disk: its superclass resolves to the current SDK in the parent loader. + try (var loader = new URLClassLoader( + new URL[] {consumer.toUri().toURL()}, getClass().getClassLoader())) { + var type = loader.loadClass("LegacyExtractor"); + var extractor = (XRayContextExtractor) type.getConstructor().newInstance(); + var runtime = ("Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=1"); + var info = new InvocationInfo("request", "arn", true, Instant.EPOCH, null, Map.of(), Map.of(), runtime); + var extracted = extractor.extract(info); + assertEquals("6955b900aaaaaaaaaaaaaaaaaaaaaaaa", extracted.traceId()); + assertEquals(ExtractedContext.Sampling.NOT_SAMPLED, extracted.sampling()); + assertEquals(1, type.getField("calls").get(extractor)); + } + } + + private static void compile(Path directory, String name, String source, String classpath) throws Exception { + Files.createDirectories(directory); + var file = directory.resolve("src").resolve(name + ".java"); + Files.createDirectories(file.getParent()); + Files.writeString(file, source); + var errors = new ByteArrayOutputStream(); + var compiler = ToolProvider.getSystemJavaCompiler(); + assertNotNull(compiler, "Binary compatibility test requires a JDK"); + assertEquals( + 0, + compiler.run( + null, + null, + errors, + "--release", + "17", + "-classpath", + classpath, + "-d", + directory.toString(), + file.toString()), + errors.toString(StandardCharsets.UTF_8)); + } +} diff --git a/pom.xml b/pom.xml index 739a1f175..03a728a7a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ software.amazon.lambda.durable aws-durable-execution-sdk-java-parent - 2.2.2-SNAPSHOT + 3.0.0-SNAPSHOT pom AWS Lambda Durable Execution SDK diff --git a/sdk-integration-tests/pom.xml b/sdk-integration-tests/pom.xml index e44650c54..d28680a79 100644 --- a/sdk-integration-tests/pom.xml +++ b/sdk-integration-tests/pom.xml @@ -7,7 +7,7 @@ software.amazon.lambda.durable aws-durable-execution-sdk-java-parent - 2.2.2-SNAPSHOT + 3.0.0-SNAPSHOT aws-durable-execution-sdk-java-integration-tests diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/DelayedCheckpointFatalIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/DelayedCheckpointFatalIntegrationTest.java new file mode 100644 index 000000000..104414f34 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/DelayedCheckpointFatalIntegrationTest.java @@ -0,0 +1,138 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.stream.Stream; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.config.StepSemantics; +import software.amazon.lambda.durable.context.BaseContextImpl; +import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.execution.ThreadContext; +import software.amazon.lambda.durable.execution.ThreadType; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +@Timeout(10) +class DelayedCheckpointFatalIntegrationTest { + @SuppressWarnings("removal") + static Stream failures() { + return Stream.of(false, true) + .flatMap(wrapped -> Stream.of(new InternalError("external plugin fatal"), new ThreadDeath()) + .map(fatal -> Arguments.of(wrapped, fatal))); + } + + @ParameterizedTest + @MethodSource("failures") + void externalFatalSettlesDelayedAtMostOnceStartBeforeItsTimer(boolean wrapped, Error fatal) throws Exception { + var startQueued = new CountDownLatch(1); + var startFuture = new AtomicReference>(); + var bodies = new AtomicInteger(); + var backendCalls = new AtomicInteger(); + var workers = Executors.newCachedThreadPool(task -> { + var thread = new Thread(task, "delayed-start-owner"); + thread.setDaemon(true); + thread.setUncaughtExceptionHandler((owner, failure) -> {}); + return thread; + }); + DurableExecutionPluginFactory fault = ignored -> new DurableExecutionPlugin() { + public void onOperationStart(OperationInfo info) { + if (!"trigger".equals(info.name())) return; + if (wrapped) throw new CompletionException(new ExecutionException(fatal)); + throw fatal; + } + }; + var config = DurableConfig.builder() + .withExecutorService(workers) + .withCheckpointDelay(Duration.ofMinutes(1)) + .withDurableExecutionClient(new LocalMemoryExecutionClient() { + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List batch) { + backendCalls.incrementAndGet(); + return super.checkpoint(arn, token, batch); + } + }) + .withPlugins(fault) + .build(); + var input = input(); + var manager = new ExecutionManager(input, config, null) { + public CompletableFuture sendOperationUpdate(OperationUpdate update) { + var result = super.sendOperationUpdate(update); + if ("blocked".equals(update.name()) && update.action() == OperationAction.START) { + startFuture.set(result); + startQueued.countDown(); + } + return result; + } + }; + try { + manager.registerActiveThread(null); + manager.setCurrentThreadContext(new ThreadContext(null, ThreadType.CONTEXT)); + manager.getPluginRunner() + .onInvocationStart(new InvocationInfo("request", input.durableExecutionArn(), true, Instant.EPOCH)); + var context = DurableContextImpl.createRootContext(manager, config, null); + BaseContextImpl.setCurrentContext(context); + context.stepAsync( + "blocked", + String.class, + step -> { + bodies.incrementAndGet(); + return "unexpected"; + }, + StepConfig.builder() + .semanticsPerRetry(StepSemantics.AT_MOST_ONCE_PER_RETRY) + .build()); + assertTrue(startQueued.await(3, TimeUnit.SECONDS)); + assertSame( + fatal, + assertThrows(Error.class, () -> context.stepAsync("trigger", String.class, step -> "unreachable"))); + assertSame( + fatal, + assertThrows( + ExecutionException.class, + () -> startFuture.get().get(500, TimeUnit.MILLISECONDS)) + .getCause()); + assertEquals(0, bodies.get()); + assertEquals(0, backendCalls.get()); + } finally { + // Keep the negative control bounded even before the production abort path exists. + if (startFuture.get() != null) startFuture.get().completeExceptionally(fatal); + workers.shutdown(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + try { + manager.close(); + } catch (CompletionException failure) { + assertSame(fatal, failure.getCause()); + } + BaseContextImpl.setCurrentContext(null); + } + } + + private static DurableExecutionInput input() { + var operation = Operation.builder() + .id("id") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } +} diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/HandlerScopeFinalizationIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/HandlerScopeFinalizationIntegrationTest.java new file mode 100644 index 000000000..eb6db4d0f --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/HandlerScopeFinalizationIntegrationTest.java @@ -0,0 +1,192 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.HandlerScoped; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class HandlerScopeFinalizationIntegrationTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void blockedCleanupReturnsTheOriginalOutcomeThenClosesOnItsOwner(boolean terminate) throws Exception { + var releaseFinally = new CountDownLatch(1); + var enteredFinally = new CountDownLatch(1); + var scopeClosed = new CountDownLatch(1); + var closedAtEnd = new AtomicBoolean(); + var plugin = new ScopedPlugin() { + @Override + public AutoCloseable openHandlerScope() { + var owner = Thread.currentThread(); + return () -> { + assertSame(owner, Thread.currentThread()); + scopeClosed.countDown(); + }; + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + closedAtEnd.set(scopeClosed.getCount() == 0); + } + }; + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + try { + if (terminate) + context.step("terminate", String.class, step -> { + throw new UnrecoverableDurableExecutionException( + ErrorObject.builder() + .errorMessage("retry") + .build(), + true); + }); + else context.wait("pause", Duration.ofSeconds(1)); + return "done"; + } finally { + enteredFinally.countDown(); + try { + if (!releaseFinally.await(5, TimeUnit.SECONDS)) throw new AssertionError("not released"); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new AssertionError(error); + } + } + }, + DurableConfig.builder().withPlugins(info -> plugin).build()); + var caller = Executors.newSingleThreadExecutor(); + try { + var response = caller.submit(() -> runner.run("input")); + assertTrue(enteredFinally.await(5, TimeUnit.SECONDS)); + if (terminate) { + var failure = assertThrows(ExecutionException.class, () -> response.get(2, TimeUnit.SECONDS)); + assertInstanceOf(UnrecoverableDurableExecutionException.class, failure.getCause()); + } else + assertEquals( + ExecutionStatus.PENDING, + response.get(2, TimeUnit.SECONDS).getStatus()); + assertFalse(closedAtEnd.get()); + assertEquals(1L, scopeClosed.getCount()); + } finally { + releaseFinally.countDown(); + assertTrue(scopeClosed.await(5, TimeUnit.SECONDS)); + caller.shutdownNow(); + } + } + + @ParameterizedTest + @CsvSource({"true,false", "false,false", "true,true", "false,true"}) + void finalizationWaitsForAnOpenedScopeWithoutChangingLegacyNoScopeTiming(boolean hasScope, boolean terminate) + throws Exception { + var finallyEntered = new CountDownLatch(1); + var releaseFinally = new CountDownLatch(1); + var handlerExited = new CountDownLatch(1); + var scopeClosed = new AtomicBoolean(); + var endCalled = new AtomicBoolean(); + var scopeClosedAtEnd = new AtomicBoolean(); + var plugin = new ScopedPlugin() { + @Override + public AutoCloseable openHandlerScope() { + if (!hasScope) return null; + var owner = Thread.currentThread(); + return () -> { + assertSame(owner, Thread.currentThread()); + scopeClosed.set(true); + }; + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + scopeClosedAtEnd.set(scopeClosed.get()); + endCalled.set(true); + } + }; + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + try { + if (terminate) { + context.step("terminate", String.class, step -> { + throw new UnrecoverableDurableExecutionException( + ErrorObject.builder() + .errorMessage("retry invocation") + .build(), + true); + }); + } else { + context.wait("pause", Duration.ofSeconds(1)); + } + return "done"; + } finally { + finallyEntered.countDown(); + try { + if (!releaseFinally.await(5, TimeUnit.SECONDS)) + throw new AssertionError("finally not released"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } finally { + handlerExited.countDown(); + } + } + }, + config); + var caller = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "test-invocation-caller"); + thread.setDaemon(true); + return thread; + }); + try { + var response = caller.submit(() -> runner.run("input")); + assertTrue(finallyEntered.await(5, TimeUnit.SECONDS)); + if (hasScope) { + assertThrows(TimeoutException.class, () -> response.get(150, TimeUnit.MILLISECONDS)); + assertFalse(endCalled.get(), "flush must wait for same-thread scope cleanup"); + releaseFinally.countDown(); + } + if (terminate) { + var failure = assertThrows(ExecutionException.class, () -> response.get(5, TimeUnit.SECONDS)); + assertInstanceOf(UnrecoverableDurableExecutionException.class, failure.getCause()); + } else { + assertEquals( + ExecutionStatus.PENDING, + response.get(5, TimeUnit.SECONDS).getStatus()); + } + assertTrue(endCalled.get()); + assertEquals(hasScope, scopeClosedAtEnd.get()); + } finally { + releaseFinally.countDown(); + assertTrue(handlerExited.await(5, TimeUnit.SECONDS)); + caller.shutdownNow(); + } + } + + @HandlerScoped(ScopedPlugin.Opener.class) + private abstract static class ScopedPlugin implements DurableExecutionPlugin { + public abstract AutoCloseable openHandlerScope(); + + public static class Opener implements Function { + public AutoCloseable apply(ScopedPlugin plugin) { + return plugin.openHandlerScope(); + } + } + } +} diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/InvocationEndFatalDispatchIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/InvocationEndFatalDispatchIntegrationTest.java new file mode 100644 index 000000000..2718bbcec --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/InvocationEndFatalDispatchIntegrationTest.java @@ -0,0 +1,190 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.context.BaseContextImpl; +import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.execution.ThreadContext; +import software.amazon.lambda.durable.execution.ThreadType; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +@Timeout(10) +class InvocationEndFatalDispatchIntegrationTest { + @SuppressWarnings("removal") + static Stream failures() { + return Stream.of(false, true) + .flatMap(wrapped -> Stream.of(new InternalError("end hook"), new ThreadDeath()) + .map(fatal -> Arguments.of(wrapped, fatal))); + } + + static Stream cases() { + return Stream.of(false, true) + .flatMap(blocked -> failures().map(args -> Arguments.of(blocked, args.get()[0], args.get()[1]))); + } + + @ParameterizedTest + @MethodSource("cases") + void endHookFatalStopsQueuedAtLeastOnceWorkBeforeShutdown(boolean blockedEnd, boolean wrapped, Error fatal) + throws Exception { + var executor = new QueuedExecutor(); + var enteredEnd = new CountDownLatch(1); + var releaseEnd = new CountDownLatch(blockedEnd ? 1 : 0); + var finalizer = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "end-hook-dispatch"); + thread.setDaemon(true); + return thread; + }); + var bodies = new AtomicInteger(); + var ends = new ArrayList(); + var updates = new CopyOnWriteArrayList(); + DurableExecutionPluginFactory failing = ignored -> new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + ends.add(info); + if (wrapped) throw new CompletionException(new ExecutionException(fatal)); + throw fatal; + } + }; + DurableExecutionPluginFactory healthy = ignored -> new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + ends.add(info); + enteredEnd.countDown(); + await(releaseEnd); + } + }; + var config = DurableConfig.builder() + .withExecutorService(executor) + .withCheckpointDelay(Duration.ZERO) + .withDurableExecutionClient(new LocalMemoryExecutionClient() { + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List batch) { + updates.addAll(batch); + return super.checkpoint(arn, token, batch); + } + }) + .withPlugins(failing, healthy) + .build(); + var input = input(); + var manager = new ExecutionManager(input, config, null); + try { + manager.registerActiveThread(null); + manager.setCurrentThreadContext(new ThreadContext(null, ThreadType.CONTEXT)); + var runner = manager.getPluginRunner(); + runner.onInvocationStart(new InvocationInfo("request", input.durableExecutionArn(), true, Instant.EPOCH)); + var context = DurableContextImpl.createRootContext(manager, config, null); + BaseContextImpl.setCurrentContext(context); + context.stepAsync("queued", String.class, step -> { + bodies.incrementAndGet(); + return "unexpected"; + }); + // Model a handler that returns with accepted async work still waiting for its worker. + var end = new InvocationEndInfo( + "request", input.durableExecutionArn(), true, InvocationStatus.SUCCEEDED, null); + var dispatch = finalizer.submit(() -> runner.onInvocationEnd(end)); + assertTrue(enteredEnd.await(3, TimeUnit.SECONDS)); + try { + if (!blockedEnd) + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> dispatch.get(3, TimeUnit.SECONDS)) + .getCause()); + else assertFalse(dispatch.isDone(), "the later exporter is still flushing"); + assertSame(fatal, assertThrows(Error.class, executor::runNext)); + } finally { + releaseEnd.countDown(); + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> dispatch.get(3, TimeUnit.SECONDS)) + .getCause()); + } + assertEquals(List.of(end, end), ends, "remaining plugins retain their single shared end snapshot"); + assertEquals(0, bodies.get(), "invocation-end fatals must stop queued user bodies"); + assertTrue(updates.isEmpty(), "no queued operation checkpoint may reach the backend"); + } finally { + releaseEnd.countDown(); + finalizer.shutdownNow(); + try { + manager.close(); + } catch (CompletionException failure) { + assertSame(fatal, failure.getCause()); + } + BaseContextImpl.setCurrentContext(null); + } + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + throw new AssertionError(failure); + } + } + + private static DurableExecutionInput input() { + var operation = Operation.builder() + .id("id") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } + + private static final class QueuedExecutor extends AbstractExecutorService { + private final ArrayDeque tasks = new ArrayDeque<>(); + + public void execute(Runnable task) { + tasks.addLast(task); + } + + void runNext() { + tasks.removeFirst().run(); + } + + public void shutdown() {} + + public List shutdownNow() { + return List.of(); + } + + public boolean isShutdown() { + return false; + } + + public boolean isTerminated() { + return false; + } + + public boolean awaitTermination(long timeout, TimeUnit unit) { + return true; + } + } +} diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/InvocationFailureCompatibilityTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/InvocationFailureCompatibilityTest.java new file mode 100644 index 000000000..16b5ed3ad --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/InvocationFailureCompatibilityTest.java @@ -0,0 +1,97 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Instant; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.DurableExecutionOutput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class InvocationFailureCompatibilityTest { + @ParameterizedTest + @ValueSource(strings = {"handler", "input", "output"}) + void ordinaryExecutionExceptionKeepsItsIdentityAndSerializedMeaning(String stage) { + var failure = new ExecutionException("application wrapper", new IllegalArgumentException("inner cause")); + var serializedFailure = new AtomicReference(); + var endInfo = new AtomicReference(); + var ends = new AtomicInteger(); + SerDes serDes = new SerDes() { + private final JacksonSerDes delegate = new JacksonSerDes(); + + public String serialize(Object value) { + if (stage.equals("output") && "result".equals(value)) ExceptionHelper.sneakyThrow(failure); + if (value instanceof Throwable) serializedFailure.set(value); + return delegate.serialize(value); + } + + public T deserialize(String value, TypeToken type) { + if (stage.equals("input")) ExceptionHelper.sneakyThrow(failure); + return delegate.deserialize(value, type); + } + }; + var config = DurableConfig.builder() + .withDurableExecutionClient(new LocalMemoryExecutionClient()) + .withSerDes(serDes) + .withPlugins(info -> new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo end) { + ends.incrementAndGet(); + endInfo.set(end); + } + }) + .build(); + if (stage.equals("output")) { + assertSame(failure, assertThrows(ExecutionException.class, () -> execute(stage, failure, config))); + assertEquals(InvocationStatus.RETRYING, endInfo.get().invocationStatus()); + } else { + var output = execute(stage, failure, config); + assertEquals(ExecutionStatus.FAILED, output.status()); + assertEquals(ExecutionException.class.getName(), output.error().errorType()); + assertEquals("application wrapper", output.error().errorMessage()); + assertSame(failure, serializedFailure.get()); + assertEquals(InvocationStatus.FAILED, endInfo.get().invocationStatus()); + } + assertEquals(1, ends.get()); + assertSame(failure, endInfo.get().executionError()); + } + + private static DurableExecutionOutput execute(String stage, Throwable failure, DurableConfig config) { + return DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> { + if (stage.equals("handler")) ExceptionHelper.sneakyThrow(failure); + return "result"; + }, + config); + } + + private static DurableExecutionInput input() { + var operation = Operation.builder() + .id("id") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } +} diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/OperationHookFatalIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/OperationHookFatalIntegrationTest.java new file mode 100644 index 000000000..1b995e6da --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/OperationHookFatalIntegrationTest.java @@ -0,0 +1,314 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.config.StepSemantics; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.operation.BaseDurableOperation; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.retry.RetryDecision; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +class OperationHookFatalIntegrationTest { + @SuppressWarnings("removal") + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void terminalCheckpointFatalStopsReleasedWaiterBeforeItsNextUserBody(boolean threadDeath) throws Exception { + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("checkpoint observer fatal"); + var changeEntered = new CountDownLatch(1); + var releaseChange = new CountDownLatch(1); + var firstRegistered = new CountDownLatch(1); + var nextStart = new CountDownLatch(1); + var releaseNext = new CountDownLatch(1); + var fatalObserved = new CountDownLatch(1); + var first = new AtomicReference(); + var effectsAfterFatal = new AtomicInteger(); + DurableExecutionPluginFactory faulty = info -> new DurableExecutionPlugin() { + public void onOperationChange(OperationChangeInfo value) { + if (value.updatedOperations().values().stream() + .noneMatch(op -> "first".equals(op.name()) && op.status() == OperationStatus.SUCCEEDED)) return; + Thread.currentThread().setUncaughtExceptionHandler((owner, failure) -> { + if (failure == fatal) fatalObserved.countDown(); + }); + changeEntered.countDown(); + await(releaseChange); + throw fatal; + } + + public void onUserFunctionStart(UserFunctionStartInfo value) { + if (!"next".equals(value.name())) return; + nextStart.countDown(); + await(releaseNext); + } + }; + var workers = Executors.newCachedThreadPool(); + var caller = Executors.newSingleThreadExecutor(); + var config = DurableConfig.builder() + .withDurableExecutionClient(new LocalMemoryExecutionClient()) + .withCheckpointDelay(Duration.ZERO) + .withExecutorService(workers) + .withPlugins(faulty) + .build(); + var atLeastOnce = StepConfig.builder() + .semanticsPerRetry(StepSemantics.AT_LEAST_ONCE_PER_RETRY) + .build(); + try { + var response = caller.submit(() -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> { + var initial = context.stepAsync("first", String.class, step -> "first", atLeastOnce); + first.set((BaseDurableOperation) initial); + firstRegistered.countDown(); + initial.get(); + return context.step( + "next", + String.class, + step -> { + if (fatalObserved.getCount() == 0) effectsAfterFatal.incrementAndGet(); + return "next"; + }, + atLeastOnce); + }, + config)); + assertTrue(changeEntered.await(5, TimeUnit.SECONDS)); + assertTrue(firstRegistered.await(5, TimeUnit.SECONDS)); + // Coordinate the broken ordering without assuming timing: if the waiter was already released, + // hold the next user-function hook until the fatal has definitely been reported by its owner. + if (first.get().getCompletionFuture().isDone()) assertTrue(nextStart.await(5, TimeUnit.SECONDS)); + releaseChange.countDown(); + assertTrue(fatalObserved.await(5, TimeUnit.SECONDS)); + releaseNext.countDown(); + var error = assertThrows(ExecutionException.class, () -> response.get(5, TimeUnit.SECONDS)); + assertSame(fatal, error.getCause()); + assertEquals( + 0, effectsAfterFatal.get(), "work released by this checkpoint must not run past its fatal barrier"); + } finally { + releaseChange.countDown(); + releaseNext.countDown(); + workers.shutdownNow(); + caller.shutdownNow(); + } + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(failure); + } + } + + @SuppressWarnings("removal") + static Stream cases() { + return Stream.of( + "step-start", + "step-end", + "child-start", + "nested-operation-start", + "operation-end", + "operation-change") + .flatMap(stage -> Stream.of(false, true) + .flatMap(wrapped -> Stream.of(new InternalError("plugin fatal"), new ThreadDeath()) + .map(fatal -> Arguments.of(stage, wrapped, fatal)))); + } + + @ParameterizedTest + @MethodSource("cases") + void pluginFatalEscapesWithoutOperationRetryOrFailureCheckpoint(String stage, boolean wrapped, Error fatal) + throws Exception { + var fired = new AtomicBoolean(); + var hookWorker = new AtomicReference(); + var escapedWorker = new AtomicReference(); + var uncaught = new AtomicReference(); + var escaped = new CountDownLatch(1); + var retries = new AtomicInteger(); + var endCalls = new AtomicInteger(); + var end = new AtomicReference(); + var updates = new CopyOnWriteArrayList(); + var client = new LocalMemoryExecutionClient() { + @Override + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List batch) { + updates.addAll(batch); + return super.checkpoint(arn, token, batch); + } + }; + DurableExecutionPluginFactory observer = info -> new DurableExecutionPlugin() { + @Override + public void onInvocationEnd(InvocationEndInfo value) { + endCalls.incrementAndGet(); + end.set(value); + } + }; + DurableExecutionPluginFactory faulty = info -> new DurableExecutionPlugin() { + private void failAt(String point) { + if (!stage.equals(point) || !fired.compareAndSet(false, true)) return; + hookWorker.set(Thread.currentThread()); + Thread.currentThread().setUncaughtExceptionHandler((owner, failure) -> { + escapedWorker.set(owner); + uncaught.set(failure); + escaped.countDown(); + }); + if (wrapped) throw new CompletionException(new ExecutionException(fatal)); + throw fatal; + } + + public void onUserFunctionStart(UserFunctionStartInfo value) { + if ("work".equals(value.name())) failAt("step-start"); + if ("child".equals(value.name())) failAt("child-start"); + } + + public void onUserFunctionEnd(UserFunctionEndInfo value) { + if ("work".equals(value.name())) failAt("step-end"); + } + + public void onOperationStart(OperationInfo value) { + if ("work".equals(value.name())) failAt("nested-operation-start"); + } + + public void onOperationEnd(OperationEndInfo value) { + if ("work".equals(value.name())) failAt("operation-end"); + } + + public void onOperationChange(OperationChangeInfo value) { + if (value.updatedOperations().values().stream().anyMatch(op -> op.status() == OperationStatus.STARTED)) + failAt("operation-change"); + } + }; + var workers = Executors.newCachedThreadPool(task -> { + var thread = new Thread(task, "operation-fatal-owner"); + thread.setDaemon(true); + return thread; + }); + var callers = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "operation-fatal-caller"); + thread.setDaemon(true); + return thread; + }); + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ZERO) + .withExecutorService(workers) + .withPlugins(observer, faulty) + .build(); + var step = StepConfig.builder() + // Await START for the checkpoint-hook case so its fatal is known before end-hook dispatch. + .semanticsPerRetry( + stage.equals("operation-change") + ? StepSemantics.AT_MOST_ONCE_PER_RETRY + : StepSemantics.AT_LEAST_ONCE_PER_RETRY) + .retryStrategy((error, attempt) -> { + retries.incrementAndGet(); + return RetryDecision.fail(); + }) + .build(); + try { + var response = callers.submit(() -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> { + if (stage.equals("child-start") || stage.equals("nested-operation-start")) + return context.runInChildContext( + "child", + String.class, + child -> child.step("work", String.class, childStep -> "ok", step)); + return context.step("work", String.class, stepContext -> "ok", step); + }, + config)); + var thrown = assertThrows(ExecutionException.class, () -> response.get(5, TimeUnit.SECONDS)); + assertSame(fatal, thrown.getCause(), "plugin fatal identity must reach the invocation caller"); + assertTrue(fired.get()); + assertTrue(escaped.await(2, TimeUnit.SECONDS), "fatal must escape the actual hook worker"); + assertSame(hookWorker.get(), escapedWorker.get()); + assertSame(fatal, uncaught.get()); + assertEquals(0, retries.get(), "plugin fatal must bypass the user operation retry strategy"); + assertTrue( + updates.stream() + .noneMatch(update -> update.action() == OperationAction.FAIL + || update.action() == OperationAction.RETRY), + "plugin fatal must not become a persisted user failure"); + assertEquals(1, endCalls.get()); + assertEquals(InvocationStatus.RETRYING, end.get().invocationStatus()); + assertSame(fatal, end.get().executionError()); + } finally { + workers.shutdownNow(); + callers.shutdownNow(); + } + } + + @Test + void legacyUserBodyFatalStillUsesItsExistingOperationFailurePath() { + var original = new InternalError("user body, not plugin instrumentation"); + var retries = new AtomicInteger(); + var updates = new CopyOnWriteArrayList(); + var client = new LocalMemoryExecutionClient() { + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List batch) { + updates.addAll(batch); + return super.checkpoint(arn, token, batch); + } + }; + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ZERO) + .build(); + var step = StepConfig.builder() + .retryStrategy((error, attempt) -> { + retries.incrementAndGet(); + return RetryDecision.fail(); + }) + .build(); + var actual = assertThrows( + InternalError.class, + () -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> context.step( + "work", + String.class, + stepContext -> { + throw original; + }, + step), + config)); + assertNotSame(original, actual, "legacy body error is restored from its stored user-operation failure"); + assertEquals(1, retries.get()); + assertTrue(updates.stream().anyMatch(update -> update.action() == OperationAction.FAIL)); + } + + private static DurableExecutionInput input() { + var operation = Operation.builder() + .id("id") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } +} diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/OperationWorkerFailureRegressionTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/OperationWorkerFailureRegressionTest.java new file mode 100644 index 000000000..c93a1dd0f --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/OperationWorkerFailureRegressionTest.java @@ -0,0 +1,313 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.model.WaitForConditionResult; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +class OperationWorkerFailureRegressionTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void rejectedOperationSubmissionSettlesShutdown(boolean direct) throws Exception { + ExecutorService workers = direct + ? new DirectExecutor(true) + : new ThreadPoolExecutor( + 1, + 1, + 0, + TimeUnit.MILLISECONDS, + new SynchronousQueue<>(), + task -> daemon(task, "bounded-owner"), + new ThreadPoolExecutor.AbortPolicy()); + var callers = Executors.newSingleThreadExecutor(task -> daemon(task, "rejection-caller")); + var bodyCalled = new AtomicBoolean(); + try { + var config = DurableConfig.builder() + .withExecutorService(workers) + .withDurableExecutionClient(new LocalMemoryExecutionClient()) + .build(); + var result = callers.submit(() -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> context.step("rejected", String.class, step -> { + bodyCalled.set(true); + return "unexpected"; + }), + config)) + .get(2, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.FAILED, result.status()); + assertEquals( + RejectedExecutionException.class.getName(), result.error().errorType()); + assertFalse(bodyCalled.get()); + } finally { + workers.shutdownNow(); + callers.shutdownNow(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void handledRejectionDoesNotLeaveAPhantomActiveOperation(boolean direct) throws Exception { + ExecutorService workers = direct + ? new DirectExecutor(true) + : new ThreadPoolExecutor( + 1, + 1, + 0, + TimeUnit.MILLISECONDS, + new SynchronousQueue<>(), + task -> daemon(task, "bounded-owner"), + new ThreadPoolExecutor.AbortPolicy()); + var callers = Executors.newSingleThreadExecutor(task -> daemon(task, "rejection-caller")); + try { + var config = DurableConfig.builder() + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withDurableExecutionClient(new LocalMemoryExecutionClient()) + .build(); + var result = callers.submit(() -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> { + assertThrows( + RejectedExecutionException.class, + () -> context.step("rejected", String.class, step -> "unexpected")); + context.wait("pause", Duration.ofSeconds(1)); + return "done"; + }, + config)) + .get(2, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.PENDING, result.status(), "rejected work is not an active handler"); + } finally { + workers.shutdownNow(); + callers.shutdownNow(); + } + } + + @SuppressWarnings("removal") + static Stream hookCases() { + return Stream.of(false, true) + .flatMap(direct -> Stream.of(false, true) + .flatMap(end -> Stream.of(false, true) + .flatMap(wrapped -> Stream.of(new InternalError("plugin fatal"), new ThreadDeath()) + .map(fatal -> Arguments.of(direct, end, wrapped, fatal))))); + } + + @ParameterizedTest + @MethodSource("hookCases") + void waitHookFatalBypassesSerializationAndEscapesItsOwner(boolean direct, boolean end, boolean wrapped, Error fatal) + throws Exception { + runWaitFatal(direct, end, wrapped, fatal, Duration.ZERO); + } + + @SuppressWarnings("removal") + @Test + void waitHookFatalDoesNotWaitForTheFailureCheckpointDelay() throws Exception { + runWaitFatal(false, false, false, new ThreadDeath(), Duration.ofSeconds(3)); + } + + private static void runWaitFatal(boolean direct, boolean atEnd, boolean wrapped, Error fatal, Duration delay) + throws Exception { + var hookThread = new AtomicReference(); + var callerThread = new AtomicReference(); + var escaped = new CountDownLatch(1); + var serializedFatal = new AtomicInteger(); + var ends = new AtomicInteger(); + var endInfo = new AtomicReference(); + var updates = new CopyOnWriteArrayList(); + ExecutorService workers = direct + ? new DirectExecutor(false) + : Executors.newCachedThreadPool(task -> { + var thread = daemon(task, "wait-owner"); + thread.setUncaughtExceptionHandler((owner, error) -> { + if (owner == hookThread.get() && error == fatal) escaped.countDown(); + }); + return thread; + }); + var callers = Executors.newSingleThreadExecutor(task -> daemon(task, "wait-caller")); + var client = new LocalMemoryExecutionClient() { + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List batch) { + updates.addAll(batch); + return super.checkpoint(arn, token, batch); + } + }; + var serdes = new JacksonSerDes() { + public String serialize(Object value) { + if (value == fatal) serializedFatal.incrementAndGet(); + return super.serialize(value); + } + }; + DurableExecutionPluginFactory factory = ignored -> new DurableExecutionPlugin() { + private void fail() { + hookThread.set(Thread.currentThread()); + if (wrapped) throw new CompletionException(new ExecutionException(fatal)); + throw fatal; + } + + public void onUserFunctionStart(UserFunctionStartInfo info) { + if (!atEnd) fail(); + } + + public void onUserFunctionEnd(UserFunctionEndInfo info) { + if (atEnd) fail(); + } + + public void onInvocationEnd(InvocationEndInfo info) { + ends.incrementAndGet(); + endInfo.set(info); + } + }; + try { + var config = DurableConfig.builder() + .withExecutorService(workers) + .withDurableExecutionClient(client) + .withSerDes(serdes) + .withCheckpointDelay(delay) + .withPlugins(factory) + .build(); + var response = callers.submit(() -> { + callerThread.set(Thread.currentThread()); + return DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> context.waitForCondition( + "work", + String.class, + (state, step) -> WaitForConditionResult.stopPolling("done"), + WaitForConditionConfig.builder().build()), + config); + }); + var error = assertThrows(ExecutionException.class, () -> response.get(2, TimeUnit.SECONDS)); + assertSame(fatal, error.getCause()); + assertEquals(0, serializedFatal.get(), "a plugin fatal must not invoke user exception serialization"); + assertTrue(updates.stream().noneMatch(update -> update.action() == OperationAction.FAIL)); + assertEquals(1, ends.get()); + assertEquals(InvocationStatus.RETRYING, endInfo.get().invocationStatus()); + assertSame(fatal, endInfo.get().executionError()); + if (direct) assertSame(callerThread.get(), hookThread.get()); + else assertTrue(escaped.await(2, TimeUnit.SECONDS), "the identical fatal must escape the hook worker"); + } finally { + workers.shutdownNow(); + callers.shutdownNow(); + } + } + + @Test + void legacyWaitBodyFatalStillUsesItsExistingFailurePath() { + var fatal = new InternalError("user condition body"); + var serialized = new AtomicInteger(); + var updates = new CopyOnWriteArrayList(); + var client = new LocalMemoryExecutionClient() { + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List batch) { + updates.addAll(batch); + return super.checkpoint(arn, token, batch); + } + }; + var serdes = new JacksonSerDes() { + public String serialize(Object value) { + if (value == fatal) serialized.incrementAndGet(); + return super.serialize(value); + } + }; + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withSerDes(serdes) + .withCheckpointDelay(Duration.ZERO) + .build(); + var actual = assertThrows( + InternalError.class, + () -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> context.waitForCondition( + "work", + String.class, + (state, step) -> { + throw fatal; + }, + WaitForConditionConfig.builder().build()), + config)); + assertNotSame(fatal, actual); + assertEquals(1, serialized.get()); + assertTrue(updates.stream().anyMatch(update -> update.action() == OperationAction.FAIL)); + } + + private static Thread daemon(Runnable task, String name) { + var thread = new Thread(task, name); + thread.setDaemon(true); + return thread; + } + + private static DurableExecutionInput input() { + var operation = Operation.builder() + .id("id") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } + + private static class DirectExecutor extends AbstractExecutorService { + private final boolean rejectOperations; + private int submitted; + + DirectExecutor(boolean rejectOperations) { + this.rejectOperations = rejectOperations; + } + + public void execute(Runnable task) { + if (rejectOperations && submitted++ > 0) throw new RejectedExecutionException("operation rejected"); + task.run(); + } + + public void shutdown() {} + + public List shutdownNow() { + return List.of(); + } + + public boolean isShutdown() { + return false; + } + + public boolean isTerminated() { + return false; + } + + public boolean awaitTermination(long timeout, TimeUnit unit) { + return true; + } + } +} diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalDispatchIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalDispatchIntegrationTest.java new file mode 100644 index 000000000..67c36f960 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalDispatchIntegrationTest.java @@ -0,0 +1,251 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.stream.Stream; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +@Timeout(10) +class PluginFatalDispatchIntegrationTest { + @SuppressWarnings("removal") + static Stream failures() { + return Stream.of(false, true) + .flatMap(wrapped -> Stream.of(new InternalError("plugin start"), new ThreadDeath()) + .map(fatal -> Arguments.of(wrapped, fatal))); + } + + @SuppressWarnings("removal") + static Stream asyncCases() { + return Stream.of(false, true) + .flatMap(queued -> failures().map(args -> Arguments.of(queued, args.get()[0], args.get()[1]))); + } + + @ParameterizedTest + @MethodSource("asyncCases") + void fatalPreventsNewAndQueuedAtLeastOnceBodies(boolean queued, boolean wrapped, Error fatal) throws Exception { + var releaseQueued = new CountDownLatch(1); + var queuedFinished = new CountDownLatch(1); + var fatalObserved = new CountDownLatch(1); + var dispatchFinished = new CountDownLatch(1); + var bodies = new AtomicInteger(); + var lateStarts = new AtomicInteger(); + var endCalls = new AtomicInteger(); + var endInfo = new AtomicReference(); + var dispatchFailure = new AtomicReference(); + var updates = new CopyOnWriteArrayList(); + var workers = + new ThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + 60, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + task -> daemon(task, "fatal-dispatch-worker")) { + private final AtomicInteger submissions = new AtomicInteger(); + + public void execute(Runnable task) { + // The root is first and the held step is second. Leave the failing third task free to run. + if (queued && submissions.incrementAndGet() == 2) { + super.execute(() -> { + try { + await(releaseQueued); + task.run(); + } finally { + queuedFinished.countDown(); + } + }); + } else super.execute(task); + } + }; + DurableExecutionPluginFactory plugin = ignored -> new DurableExecutionPlugin() { + public void onUserFunctionStart(UserFunctionStartInfo info) { + if ("late".equals(info.name())) lateStarts.incrementAndGet(); + if ("trigger".equals(info.name())) fail(wrapped, fatal); + } + + public void onInvocationEnd(InvocationEndInfo info) { + endCalls.incrementAndGet(); + endInfo.set(info); + fatalObserved.countDown(); + releaseQueued.countDown(); + await(queued ? queuedFinished : dispatchFinished); + } + }; + var config = DurableConfig.builder() + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withDurableExecutionClient(new LocalMemoryExecutionClient() { + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List batch) { + updates.addAll(batch); + return super.checkpoint(arn, token, batch); + } + }) + .withPlugins(plugin) + .build(); + try { + assertSame( + fatal, + assertThrows( + Error.class, + () -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> { + if (queued) + context.stepAsync("late", String.class, step -> { + bodies.incrementAndGet(); + return "unexpected"; + }); + context.stepAsync("trigger", String.class, step -> "unreachable"); + await(fatalObserved); + if (!queued) { + try { + context.stepAsync("late", String.class, step -> { + bodies.incrementAndGet(); + return "unexpected"; + }); + } catch (Error error) { + dispatchFailure.set(error); + } finally { + dispatchFinished.countDown(); + } + } + return "done"; + }, + config))); + assertEquals(0, bodies.get(), "no new user body may run after the fatal was recorded"); + assertEquals(0, lateStarts.get(), "a queued handler must stop before opening plugin attempt scopes"); + assertTrue(updates.stream().noneMatch(update -> "late".equals(update.name()))); + if (!queued) assertSame(fatal, dispatchFailure.get()); + assertEquals(1, endCalls.get()); + assertEquals(InvocationStatus.RETRYING, endInfo.get().invocationStatus()); + assertSame(fatal, endInfo.get().executionError()); + } finally { + releaseQueued.countDown(); + workers.shutdownNow(); + assertTrue(workers.awaitTermination(2, TimeUnit.SECONDS)); + } + } + + @ParameterizedTest + @MethodSource("failures") + void inlineFatalEscapesTheOperationCallBeforeMoreHandlerCodeRuns(boolean wrapped, Error fatal) { + var continued = new AtomicBoolean(); + var bodyCalled = new AtomicBoolean(); + var endCalls = new AtomicInteger(); + var endInfo = new AtomicReference(); + DurableExecutionPluginFactory plugin = ignored -> new DurableExecutionPlugin() { + public void onUserFunctionStart(UserFunctionStartInfo info) { + fail(wrapped, fatal); + } + + public void onInvocationEnd(InvocationEndInfo info) { + endCalls.incrementAndGet(); + endInfo.set(info); + } + }; + var config = DurableConfig.builder() + .withDurableExecutionClient(new LocalMemoryExecutionClient()) + .withExecutorService(new DirectExecutor()) + .withPlugins(plugin) + .build(); + assertSame( + fatal, + assertThrows( + Error.class, + () -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> { + context.stepAsync("trigger", String.class, step -> { + bodyCalled.set(true); + return "unexpected"; + }); + continued.set(true); + return "done"; + }, + config))); + assertFalse(continued.get(), "an inline fatal must leave the operation call exceptionally"); + assertFalse(bodyCalled.get()); + assertEquals(1, endCalls.get()); + assertEquals(InvocationStatus.RETRYING, endInfo.get().invocationStatus()); + assertSame(fatal, endInfo.get().executionError()); + } + + private static void fail(boolean wrapped, Error fatal) { + if (wrapped) throw new CompletionException(new ExecutionException(fatal)); + throw fatal; + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(3, TimeUnit.SECONDS), "worker coordination timed out"); + } catch (InterruptedException failure) { + throw new AssertionError(failure); + } + } + + private static Thread daemon(Runnable task, String name) { + var thread = new Thread(task, name); + thread.setDaemon(true); + thread.setUncaughtExceptionHandler((owner, failure) -> {}); + return thread; + } + + private static DurableExecutionInput input() { + var operation = Operation.builder() + .id("id") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } + + private static final class DirectExecutor extends AbstractExecutorService { + public void execute(Runnable task) { + task.run(); + } + + public void shutdown() {} + + public List shutdownNow() { + return List.of(); + } + + public boolean isShutdown() { + return false; + } + + public boolean isTerminated() { + return false; + } + + public boolean awaitTermination(long timeout, TimeUnit unit) { + return true; + } + } +} diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalQueuedCheckpointTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalQueuedCheckpointTest.java new file mode 100644 index 000000000..75e66d6c4 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalQueuedCheckpointTest.java @@ -0,0 +1,142 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.config.StepConfig; +import software.amazon.lambda.durable.config.StepSemantics; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; + +class PluginFatalQueuedCheckpointTest { + @SuppressWarnings("removal") + static Stream fatalCases() { + return Stream.of(false, true) + .flatMap(wrapped -> Stream.of(new InternalError("concurrent hook"), new ThreadDeath()) + .map(fatal -> Arguments.of(wrapped, fatal))); + } + + @ParameterizedTest + @MethodSource("fatalCases") + void queuedAtMostOnceStartCannotRunItsBodyAfterAnotherHookFails(boolean wrapped, Error fatal) throws Exception { + var checkpointEntered = new CountDownLatch(1); + var releaseCheckpoint = new CountDownLatch(1); + var queuedBodies = new AtomicInteger(); + var ends = new AtomicInteger(); + var updates = new CopyOnWriteArrayList(); + var client = new LocalMemoryExecutionClient() { + public CheckpointDurableExecutionResponse checkpoint( + String arn, String token, List batch) { + updates.addAll(batch); + if (batch.stream() + .anyMatch(update -> + "barrier".equals(update.name()) && update.action() == OperationAction.START)) { + checkpointEntered.countDown(); + await(releaseCheckpoint); + } + return super.checkpoint(arn, token, batch); + } + }; + DurableExecutionPluginFactory fault = info -> new DurableExecutionPlugin() { + public void onUserFunctionStart(UserFunctionStartInfo start) { + if (!"trigger".equals(start.name())) return; + if (wrapped) throw new CompletionException(new ExecutionException(fatal)); + throw fatal; + } + + public void onInvocationEnd(InvocationEndInfo end) { + ends.incrementAndGet(); + releaseCheckpoint.countDown(); + } + }; + var callers = Executors.newSingleThreadExecutor(task -> daemon(task, "queued-checkpoint-caller")); + var workers = Executors.newCachedThreadPool(task -> daemon(task, "queued-checkpoint-worker")); + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(fault) + .build(); + var atMostOnce = StepConfig.builder() + .semanticsPerRetry(StepSemantics.AT_MOST_ONCE_PER_RETRY) + .build(); + try { + var response = callers.submit(() -> DurableExecutor.execute( + input(), + null, + TypeToken.get(String.class), + (value, context) -> { + context.stepAsync("barrier", String.class, step -> "barrier", atMostOnce); + await(checkpointEntered); + var queued = context.stepAsync( + "queued", + String.class, + step -> { + queuedBodies.incrementAndGet(); + return "must not run"; + }, + atMostOnce); + context.stepAsync("trigger", String.class, step -> "unreachable"); + return queued.get(); + }, + config)); + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> response.get(5, TimeUnit.SECONDS)) + .getCause()); + assertEquals(1, ends.get()); + assertTrue( + updates.stream() + .noneMatch(update -> + "queued".equals(update.name()) && update.action() == OperationAction.START), + "the queued START must never reach the backend after the fatal"); + assertEquals(0, queuedBodies.get(), "a skipped START must not authorize the at-most-once user function"); + } finally { + releaseCheckpoint.countDown(); + workers.shutdownNow(); + callers.shutdownNow(); + } + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + } + + private static Thread daemon(Runnable task, String name) { + var thread = new Thread(task, name); + thread.setDaemon(true); + return thread; + } + + private static DurableExecutionInput input() { + var operation = Operation.builder() + .id("id") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.EPOCH) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } +} diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginIntegrationTest.java index f8e3bceb6..21ba26b45 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginIntegrationTest.java @@ -39,10 +39,10 @@ void pluginsFromConfigurationAndEnvironment_receiveLifecycleEvents() { var configuredPlugin = new RecordingPlugin(); var dynamicPlugin = new RecordingPlugin(); var provider = new RecordingPluginProvider(dynamicPlugin); - var plugins = - DynamicPluginLoader.loadConfiguredPlugins("recording", List.of(provider), List.of(configuredPlugin)); + var factories = DynamicPluginLoader.loadConfiguredPluginFactories( + "recording", List.of(provider), List.of(info -> configuredPlugin)); var config = DurableConfig.builder() - .withPlugins(plugins.toArray(DurableExecutionPlugin[]::new)) + .withPlugins(factories.toArray(DurableExecutionPluginFactory[]::new)) .build(); var runner = LocalDurableTestRunner.create( @@ -60,7 +60,7 @@ void pluginsFromConfigurationAndEnvironment_receiveLifecycleEvents() { @Test void plugin_receivesInvocationStartAndEnd_onSuccessfulExecution() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -84,7 +84,7 @@ void plugin_receivesInvocationStartAndEnd_onSuccessfulExecution() { @Test void plugin_receivesInvocationEnd_withPendingStatus_onSuspension() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -106,7 +106,7 @@ void plugin_receivesInvocationEnd_withPendingStatus_onSuspension() { @Test void plugin_invocationSnapshots_trackReplayAcrossSuspension() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -168,7 +168,7 @@ void plugin_invocationSnapshots_trackReplayAcrossSuspension() { @Test void plugin_receivesInvocationEnd_withFailedStatus_onError() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -195,7 +195,7 @@ void plugin_receivesInvocationEnd_withFailedStatus_onError() { @Test void plugin_invocationHooks_carryExecutionInputAndResult_onSuccess() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -217,7 +217,7 @@ void plugin_invocationHooks_carryExecutionInputAndResult_onSuccess() { @Test void plugin_invocationEnd_omitsExecutionResult_onFailure() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -244,7 +244,7 @@ void plugin_invocationEnd_omitsExecutionResult_onFailure() { @Test void plugin_invocationEnd_omitsExecutionResult_onSuspension() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -268,8 +268,10 @@ void plugin_invocationEnd_omitsExecutionResult_onSuspension() { void plugin_executionInput_isDeserializedOnce_andSharedWithHandler() { var serDes = new CountingSerDes(); var plugin = new RecordingPlugin(); - var config = - DurableConfig.builder().withPlugins(plugin).withSerDes(serDes).build(); + var config = DurableConfig.builder() + .withPlugins(info -> plugin) + .withSerDes(serDes) + .build(); var handlerInput = new AtomicReference(); var runner = LocalDurableTestRunner.create( @@ -318,7 +320,7 @@ int inputDeserializations(String value) { void plugin_hooksStayPaired_whenSerDesSneakyThrowsCheckedException() { var plugin = new RecordingPlugin(); var config = DurableConfig.builder() - .withPlugins(plugin) + .withPlugins(info -> plugin) .withSerDes(new SneakyThrowingSerDes()) .build(); @@ -353,12 +355,100 @@ public T deserialize(String data, TypeToken typeToken) { } } + @Test + void plugin_hooksStayPaired_whenTheResultCannotBeSerialized() { + var plugin = new RecordingPlugin(); + var config = DurableConfig.builder() + .withPlugins(info -> plugin) + .withSerDes(new ResultRejectingSerDes()) + .build(); + + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> "unserializable", config); + + // The invocation fails on the way out, after the handler has already returned. + assertThrows(Exception.class, () -> runner.run("input")); + + // The end hook is the only point at which a plugin can flush: releasePlugins() calls nothing on the + // instances it drops and the contract has no close(). An exit that skips it therefore discards the whole + // invocation's telemetry -- Insight's record and every exporter's flush, and both OTel plugins' spans -- + // and can leave a record queued for a pump that exports it after this invocation has returned. + assertEquals(1, plugin.invocationStarts.size()); + assertEquals(1, plugin.invocationEnds.size(), "a start hook must not be left without its end hook"); + // RETRYING, not SUCCEEDED: the result never reached the backend, so the execution is not finished. + assertEquals(InvocationStatus.RETRYING, plugin.invocationEnds.get(0).invocationStatus()); + assertNotNull( + plugin.invocationEnds.get(0).executionError(), "the plugin must be told why the invocation ended"); + } + + /** SerDes that refuses to serialize the handler's result, as JacksonSerDes does for an unwritable value. */ + static class ResultRejectingSerDes implements SerDes { + private final JacksonSerDes delegate = new JacksonSerDes(); + + @Override + public String serialize(Object value) { + if ("unserializable".equals(value)) { + throw new IllegalStateException("cannot serialize the result"); + } + return delegate.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return delegate.deserialize(data, typeToken); + } + } + + @Test + void plugin_seesTheUnderlyingFailure_whenResultDeliveryFailsWrapped() { + // handleLargePayload waits with join(), so a failed checkpoint of an oversized result reaches the same catch + // wrapped in a CompletionException. Plugins are told what failed, not how it was delivered. + var plugin = new RecordingPlugin(); + var cause = new IllegalStateException("underlying delivery failure"); + var config = DurableConfig.builder() + .withPlugins(info -> plugin) + .withSerDes(new WrappedFailureSerDes(cause)) + .build(); + + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> "unserializable", config); + + assertThrows(Exception.class, () -> runner.run("input")); + + assertEquals(1, plugin.invocationEnds.size()); + assertSame( + cause, + plugin.invocationEnds.get(0).executionError(), + "the plugin must be told the underlying failure, not the CompletionException wrapper"); + } + + /** SerDes whose result failure arrives wrapped, as a failed oversized-result checkpoint does. */ + static class WrappedFailureSerDes implements SerDes { + private final JacksonSerDes delegate = new JacksonSerDes(); + private final Throwable cause; + + WrappedFailureSerDes(Throwable cause) { + this.cause = cause; + } + + @Override + public String serialize(Object value) { + if ("unserializable".equals(value)) { + throw new CompletionException(cause); + } + return delegate.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return delegate.deserialize(data, typeToken); + } + } + // ─── Operation-level hooks ─────────────────────────────────────────── @Test void plugin_receivesOperationStartAndEnd_forStep() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, (input, context) -> context.step("my-step", String.class, stepCtx -> "result"), config); @@ -379,7 +469,7 @@ void plugin_receivesOperationStartAndEnd_forStep() { @Test void plugin_receivesOperationStart_forMultipleSteps() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -402,7 +492,7 @@ void plugin_receivesOperationStart_forMultipleSteps() { @Test void plugin_operationEnd_notFiredOnReplay() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -436,7 +526,7 @@ void plugin_operationEnd_notFiredOnReplay() { @Test void plugin_operationEnd_firedForOperationCompletedDuringSuspension() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -473,7 +563,7 @@ void plugin_operationEnd_firedForOperationCompletedDuringSuspension() { @Test void plugin_operationEnd_firedOnceForStepCompletingInCurrentInvocation() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -496,7 +586,7 @@ void plugin_operationEnd_firedOnceForStepCompletingInCurrentInvocation() { @Test void plugin_operationEnd_includesError_whenInvokeFailsDuringSuspension() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -541,7 +631,7 @@ void plugin_operationEnd_includesError_whenInvokeFailsDuringSuspension() { @Test void plugin_operationEnd_includesError_whenStepFailsViaCheckpoint() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -572,7 +662,7 @@ void plugin_operationEnd_includesError_whenStepFailsViaCheckpoint() { @Test void plugin_operationEnd_noError_whenOperationSucceeds() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, (input, context) -> context.step("ok-step", String.class, stepCtx -> "success"), config); @@ -590,7 +680,7 @@ void plugin_operationEnd_noError_whenOperationSucceeds() { @Test void plugin_operationEnd_includesResult_whenStepSucceeds() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, (input, context) -> context.step("my-step", String.class, stepCtx -> "task-a"), config); @@ -611,7 +701,7 @@ void plugin_operationStartAndEnd_balanced_forEmptyMap() { var plugin = new RecordingPlugin(); // withCheckpointEmptyMap is a temporary flag expected to be removed in a future major version. var config = DurableConfig.builder() - .withPlugins(plugin) + .withPlugins(info -> plugin) .withCheckpointEmptyMap(true) .build(); @@ -643,7 +733,7 @@ void plugin_operationStartAndEnd_balanced_forEmptyMap() { @Test void plugin_receivesOperationChange_forStep() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, (input, context) -> context.step("my-step", String.class, stepCtx -> "result"), config); @@ -666,7 +756,7 @@ void plugin_receivesOperationChange_forStep() { @Test void plugin_operationChange_includesErrorAndStatus_whenStepFails() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -700,7 +790,7 @@ void plugin_operationChange_includesErrorAndStatus_whenStepFails() { @Test void plugin_receivesUserFunctionStartAndEnd_forStep() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, (input, context) -> context.step("compute", String.class, stepCtx -> "42"), config); @@ -720,7 +810,7 @@ void plugin_receivesUserFunctionStartAndEnd_forStep() { @Test void plugin_userFunctionEnd_reportsFailed_whenStepFails() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); // When a step's user function throws, the exception propagates through the user-function hook // boundary, so onUserFunctionEnd reports FAILED with the error. Retry/checkpoint @@ -762,7 +852,7 @@ void plugin_userFunctionEnd_reportsFailed_whenStepFails() { void plugin_userFunctionStart_includesAttemptNumber_forRetries() { var attemptCounter = new AtomicInteger(0); var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -797,7 +887,9 @@ void plugin_userFunctionStart_includesAttemptNumber_forRetries() { void multiplePlugins_allReceiveHooks() { var plugin1 = new RecordingPlugin(); var plugin2 = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin1, plugin2).build(); + var config = DurableConfig.builder() + .withPlugins(info -> plugin1, info -> plugin2) + .build(); var runner = LocalDurableTestRunner.create( String.class, (input, context) -> context.step("step", String.class, stepCtx -> "result"), config); @@ -816,7 +908,7 @@ void throwingPlugin_doesNotDisruptExecution() { var throwingPlugin = new ThrowingPlugin(); var recordingPlugin = new RecordingPlugin(); var config = DurableConfig.builder() - .withPlugins(throwingPlugin, recordingPlugin) + .withPlugins(info -> throwingPlugin, info -> recordingPlugin) .build(); var runner = LocalDurableTestRunner.create( @@ -833,12 +925,63 @@ void throwingPlugin_doesNotDisruptExecution() { assertFalse(recordingPlugin.invocationEnds.isEmpty()); } + @Test + void factoryThrowingLinkageError_doesNotDisruptExecution() { + // A provider whose optional dependency is missing from the deployment package fails this way. A LinkageError is + // an Error, not an Exception, so containment that catches only Exception lets it escape onInvocationStart and + // fail the whole execution. + var recordingPlugin = new RecordingPlugin(); + var config = DurableConfig.builder() + .withPlugins( + info -> { + throw new NoClassDefFoundError("software/amazon/example/OptionalExporter"); + }, + info -> recordingPlugin) + .build(); + + var runner = LocalDurableTestRunner.create( + String.class, (input, context) -> context.step("step", String.class, stepCtx -> "safe"), config); + + var result = runner.runUntilComplete("input"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("safe", result.getResult(String.class)); + assertFalse(recordingPlugin.invocationStarts.isEmpty(), "the surviving plugin must still receive its hooks"); + assertFalse(recordingPlugin.invocationEnds.isEmpty()); + } + + @Test + void factoryThrowingAbstractMethodError_doesNotDisruptExecution() { + // What a provider compiled against an earlier version of the factory interface throws the first time the SDK + // invokes the method it does not implement — the exact failure this SDK's factory-only plugin contract creates + // for a provider that has not been recompiled. + var recordingPlugin = new RecordingPlugin(); + var config = DurableConfig.builder() + .withPlugins( + info -> { + throw new AbstractMethodError( + "software.amazon.example.LegacyProvider.createPlugin(InvocationInfo)"); + }, + info -> recordingPlugin) + .build(); + + var runner = LocalDurableTestRunner.create( + String.class, (input, context) -> context.step("step", String.class, stepCtx -> "safe"), config); + + var result = runner.runUntilComplete("input"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("safe", result.getResult(String.class)); + assertFalse(recordingPlugin.invocationStarts.isEmpty(), "the surviving plugin must still receive its hooks"); + assertFalse(recordingPlugin.invocationEnds.isEmpty()); + } + // ─── Child context hooks ───────────────────────────────────────────── @Test void plugin_receivesHooks_forChildContextOperations() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -861,7 +1004,7 @@ void plugin_receivesHooks_forChildContextOperations() { void plugin_receivesAttemptNumbers_forWaitForCondition() { var checkCount = new AtomicInteger(0); var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -892,7 +1035,7 @@ void plugin_receivesAttemptNumbers_forWaitForCondition() { void plugin_reportsFailedThenSucceededAttempts_forRetriedStep() { var attempts = new AtomicInteger(0); var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -948,7 +1091,7 @@ void plugin_reportsFailedThenSucceededAttempts_forRetriedStep() { @Test void plugin_userFunctionEnd_reportsSuspension_asIncomplete() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); // A child context whose body suspends (on a wait) throws SuspendExecutionException through the // user-function boundary, so onUserFunctionEnd fires with INCOMPLETE and the suspend exception. @@ -976,7 +1119,7 @@ void plugin_userFunctionEnd_reportsSuspension_asIncomplete() { @Test void plugin_userFunctionEnd_unwrapsCompletionExceptionForSuspension() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var suspension = new SuspendExecutionException(); var runner = LocalDurableTestRunner.create( @@ -1000,7 +1143,7 @@ void plugin_userFunctionEnd_unwrapsCompletionExceptionForSuspension() { @Test void plugin_parallelBranches_emitUserFunctionHooks_butConsumerDoesNot() { var plugin = new RecordingPlugin(); - var config = DurableConfig.builder().withPlugins(plugin).build(); + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -1037,7 +1180,15 @@ void plugin_parallelBranches_emitUserFunctionHooks_butConsumerDoesNot() { // ─── Test helper classes ───────────────────────────────────────────── - /** Plugin that records all hook invocations for assertions. */ + /** + * Plugin that records all hook invocations for assertions. + * + *

    Registered as {@code withPlugins(info -> plugin)}, so every invocation of a test's execution is handed the + * same recorder. The SDK creates a plugin instance per invocation, and several tests here span two invocations (a + * suspension and its resume, or a retry with a delay); handing all of them one recorder is what lets those tests + * assert on what the whole execution observed, e.g. that {@code step1}'s operation-end fired exactly once across + * both invocations. Production plugins return a fresh instance instead. + */ private static class RecordingPlugin implements DurableExecutionPlugin { final List invocationStarts = Collections.synchronizedList(new ArrayList<>()); final List invocationEnds = Collections.synchronizedList(new ArrayList<>()); @@ -1089,18 +1240,12 @@ public String getName() { return "recording"; } + /** + * Hands every invocation the same recorder so the assertions can read what all of them observed; a real + * provider would build a fresh instance here. + */ @Override - public int getApiVersion() { - return API_VERSION; - } - - @Override - public Class getPluginType() { - return RecordingPlugin.class; - } - - @Override - public DurableExecutionPlugin createPlugin() { + public DurableExecutionPlugin createPlugin(InvocationInfo invocationInfo) { return plugin; } } diff --git a/sdk-testing/pom.xml b/sdk-testing/pom.xml index 3d87ee01c..ff82615e0 100644 --- a/sdk-testing/pom.xml +++ b/sdk-testing/pom.xml @@ -7,7 +7,7 @@ software.amazon.lambda.durable aws-durable-execution-sdk-java-parent - 2.2.2-SNAPSHOT + 3.0.0-SNAPSHOT aws-durable-execution-sdk-java-testing diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index 1a5c47c88..b2c743ef0 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -21,7 +21,6 @@ import software.amazon.lambda.durable.execution.DurableExecutor; import software.amazon.lambda.durable.model.DurableExecutionInput; import software.amazon.lambda.durable.model.ExecutionStatus; -import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; import software.amazon.lambda.durable.testing.local.OperationResult; @@ -64,16 +63,8 @@ private LocalDurableTestRunner( // Create config that uses customer's configuration but overrides the client with in-memory storage if (customerConfig != null) { // Use customer's config but override the client with our in-memory implementation - this.customerConfig = DurableConfig.builder() + this.customerConfig = customerConfig.toBuilder() .withDurableExecutionClient(storage) - .withSerDes(customerConfig.getSerDes()) - .withExecutorService(customerConfig.getExecutorService()) - .withPollingStrategy(customerConfig.getPollingStrategy()) - .withCheckpointDelay(customerConfig.getCheckpointDelay()) - .withLoggerConfig(customerConfig.getLoggerConfig()) - // Temporary: remove along with the checkpointEmptyMap flag in a future major version. - .withCheckpointEmptyMap(customerConfig.shouldCheckpointEmptyMap()) - .withPlugins(customerConfig.getPluginRunner().getPlugins().toArray(new DurableExecutionPlugin[0])) .build(); } else { // Fallback to default config with in-memory client diff --git a/sdk-testing/src/test/compatibility/TestingSdkCompatibilityProbe.java b/sdk-testing/src/test/compatibility/TestingSdkCompatibilityProbe.java new file mode 100644 index 000000000..88ef8264b --- /dev/null +++ b/sdk-testing/src/test/compatibility/TestingSdkCompatibilityProbe.java @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import java.lang.reflect.Proxy; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.client.DurableExecutionClient; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +public class TestingSdkCompatibilityProbe { + public static void main(String[] args) { + var unused=(DurableExecutionClient)Proxy.newProxyInstance(DurableExecutionClient.class.getClassLoader(), + new Class[]{DurableExecutionClient.class}, (p,m,a)->{throw new AssertionError("unexpected client call");}); + var config=DurableConfig.builder().withDurableExecutionClient(unused).build(); + var result=LocalDurableTestRunner.create(String.class,(input,context)->input,config).runUntilComplete("ok"); + if(result.getStatus()!=ExecutionStatus.SUCCEEDED || !"ok".equals(result.getResult(String.class)))throw new AssertionError(result); + System.out.println("PASS testing-sdk on released core"); + } +} diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/ConfigurationCopyCompatibilityTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/ConfigurationCopyCompatibilityTest.java new file mode 100644 index 000000000..16c423de8 --- /dev/null +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/ConfigurationCopyCompatibilityTest.java @@ -0,0 +1,46 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.testing; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; + +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.client.DurableExecutionClient; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationInfo; + +class ConfigurationCopyCompatibilityTest { + @Test + void copyPreservesFactoryIdentityAndConfiguredBehavior() { + var starts = new AtomicInteger(); + DurableExecutionPluginFactory factory = info -> new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo info) { + starts.incrementAndGet(); + } + }; + var configured = DurableConfig.builder() + .withDurableExecutionClient(mock(DurableExecutionClient.class)) + .withPlugins(factory) + .withDeserializeAfterSerialization(false) + .build(); + var copied = configured.toBuilder() + .withDurableExecutionClient(mock(DurableExecutionClient.class)) + .build(); + assertSame(factory, copied.getPluginFactories().get(0)); + assertFalse(copied.shouldDeserializeAfterSerialization()); + var result = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.step("copy-check", String.class, step -> input), + copied) + .runUntilComplete("ok"); + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("ok", result.getResult(String.class)); + assertEquals(1, starts.get()); + } +} diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 36f1bbced..23a30a141 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -95,7 +95,8 @@ public void onInvocationStart(InvocationInfo info) { executionStartTimes.add(info.executionStartTime()); } }; - var config = DurableConfig.builder().withPlugins(plugin).build(); + // One instance for both invocations, so the assertion below still compares what two invocations observed. + var config = DurableConfig.builder().withPlugins(info -> plugin).build(); var runner = LocalDurableTestRunner.create( String.class, (input, context) -> { diff --git a/sdk/pom.xml b/sdk/pom.xml index d7dac4c94..7712d08e2 100644 --- a/sdk/pom.xml +++ b/sdk/pom.xml @@ -7,7 +7,7 @@ software.amazon.lambda.durable aws-durable-execution-sdk-java-parent - 2.2.2-SNAPSHOT + 3.0.0-SNAPSHOT aws-durable-execution-sdk-java diff --git a/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java index 5101b9fda..ff95a1a33 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java @@ -23,7 +23,7 @@ import software.amazon.lambda.durable.client.DurableExecutionClient; import software.amazon.lambda.durable.client.LambdaDurableFunctionsClient; import software.amazon.lambda.durable.logging.LoggerConfig; -import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; import software.amazon.lambda.durable.plugin.PluginRunner; import software.amazon.lambda.durable.retry.PollingStrategies; import software.amazon.lambda.durable.retry.PollingStrategy; @@ -100,10 +100,13 @@ public final class DurableConfig { private final Duration checkpointDelay; private final boolean deserializeAfterSerialization; private final boolean checkpointEmptyMap; - private final PluginRunner pluginRunner; + private final List pluginFactories; private DurableConfig(Builder builder) { - var plugins = DynamicPluginLoader.loadConfiguredPlugins(builder.plugins); + this.pluginFactories = builder.loadDynamicPlugins + ? DynamicPluginLoader.loadConfiguredPluginFactories(builder.pluginFactories) + : List.copyOf(builder.pluginFactories); + PluginRunner.validateExclusiveGroups(this.pluginFactories); this.durableExecutionClient = Objects.requireNonNullElseGet( builder.durableExecutionClient, DurableConfig::createDefaultDurableExecutionClient); this.serDes = Objects.requireNonNullElseGet(builder.serDes, JacksonSerDes::new); @@ -114,7 +117,6 @@ private DurableConfig(Builder builder) { this.checkpointDelay = Objects.requireNonNullElseGet(builder.checkpointDelay, () -> Duration.ofSeconds(0)); this.deserializeAfterSerialization = builder.deserializeAfterSerialization; this.checkpointEmptyMap = builder.checkpointEmptyMap; - this.pluginRunner = plugins.isEmpty() ? PluginRunner.noOp() : new PluginRunner(plugins); validateConfiguration(); } @@ -215,14 +217,31 @@ public boolean shouldCheckpointEmptyMap() { } /** - * Gets the plugin runner that dispatches lifecycle events to registered plugins. + * Gets the plugin factories registered via the builder or loaded dynamically, in dispatch order. * - *

    Returns a no-op runner if no plugins were registered via the builder or loaded dynamically. + *

    Each factory is called once per Lambda invocation to create that invocation's plugin instance; the SDK never + * shares a plugin instance across invocations. * - * @return PluginRunner instance (never null) + * @return immutable list of plugin factories (never null, possibly empty) */ - public PluginRunner getPluginRunner() { - return pluginRunner; + public List getPluginFactories() { + return pluginFactories; + } + + /** Copies effective factory registrations without repeating environment discovery. */ + public Builder toBuilder() { + var builder = new Builder() + .withDurableExecutionClient(durableExecutionClient) + .withSerDes(serDes) + .withExecutorService(executorService) + .withLoggerConfig(loggerConfig) + .withPollingStrategy(pollingStrategy) + .withCheckpointDelay(checkpointDelay) + .withDeserializeAfterSerialization(deserializeAfterSerialization) + .withCheckpointEmptyMap(checkpointEmptyMap); + builder.pluginFactories = new ArrayList<>(pluginFactories); + builder.loadDynamicPlugins = false; + return builder; } public void validateConfiguration() { @@ -321,7 +340,8 @@ public static final class Builder { private Duration checkpointDelay; private boolean deserializeAfterSerialization = true; private boolean checkpointEmptyMap = false; - private List plugins = new ArrayList<>(); + private List pluginFactories = new ArrayList<>(); + private boolean loadDynamicPlugins = true; public Builder() {} @@ -459,24 +479,30 @@ public Builder withCheckpointEmptyMap(boolean checkpointEmptyMap) { } /** - * Registers one or more plugins for lifecycle event instrumentation. + * Registers one or more plugin factories for lifecycle event instrumentation. + * + *

    Each factory is called once per Lambda invocation, with that invocation's {@code InvocationInfo}, and the + * instance it returns receives only that invocation's hooks. Plugin instances can therefore keep per-invocation + * state in plain fields even when the execution environment runs several executions concurrently. * - *

    Plugins receive hooks at invocation, operation, and user function boundaries. Errors thrown by plugins are - * isolated and never disrupt SDK execution. + *

    Plugins receive hooks at invocation, operation, and user function boundaries. Non-fatal factory/hook + * failures and null factory results are logged and skipped. {@link VirtualMachineError} and {@link ThreadDeath} + * propagate, including when wrapped by asynchronous completion/future exceptions. * - *

    Calling this method replaces any previously registered plugins. Plugins are called in registration order. + *

    Calling this method replaces any previously registered factories. Plugins are called in registration + * order. * - * @param plugins the plugins to register + * @param pluginFactories the plugin factories to register * @return This builder - * @throws NullPointerException if any plugin is null + * @throws NullPointerException if any factory is null */ - public Builder withPlugins(DurableExecutionPlugin... plugins) { - Objects.requireNonNull(plugins, "Plugins array cannot be null"); - var newPlugins = new ArrayList(plugins.length); - for (var plugin : plugins) { - newPlugins.add(Objects.requireNonNull(plugin, "Plugin cannot be null")); + public Builder withPlugins(DurableExecutionPluginFactory... pluginFactories) { + Objects.requireNonNull(pluginFactories, "Plugins array cannot be null"); + var newFactories = new ArrayList(pluginFactories.length); + for (var pluginFactory : pluginFactories) { + newFactories.add(Objects.requireNonNull(pluginFactory, "Plugin cannot be null")); } - this.plugins = newPlugins; + this.pluginFactories = newFactories; return this; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/DynamicPluginLoader.java b/sdk/src/main/java/software/amazon/lambda/durable/DynamicPluginLoader.java index efd2e86f0..7d596f09e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/DynamicPluginLoader.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/DynamicPluginLoader.java @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable; +import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -11,45 +12,59 @@ import java.util.ServiceConfigurationError; import java.util.ServiceLoader; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider; +import software.amazon.lambda.durable.plugin.InvocationInfo; final class DynamicPluginLoader { static final String PLUGINS_ENVIRONMENT_VARIABLE = "DURABLE_EXECUTION_PLUGINS"; + /** + * Closing sentences shared by the failures a stale provider JAR produces. A provider distributed as a Lambda layer + * has its own version and its own deployment, so raising the function's SDK dependency leaves the deployed provider + * untouched. An operator who does not know that reads "rebuild the provider" as something the function build + * already did, so the remedy names the layer explicitly. + */ + private static final String REBUILD_PROVIDER_REMEDY = + "Rebuild the provider against this SDK version and redeploy it. A provider shipped as a Lambda layer is " + + "versioned and deployed separately from the function package, so upgrading the function's SDK " + + "dependency does not update the layer."; + private DynamicPluginLoader() {} - static List loadConfiguredPlugins(List explicitPlugins) { + static List loadConfiguredPluginFactories( + List explicitFactories) { var configuredNames = System.getenv(PLUGINS_ENVIRONMENT_VARIABLE); if (configuredNames == null || configuredNames.isBlank()) { - return List.copyOf(explicitPlugins); + return List.copyOf(explicitFactories); } var classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { classLoader = DurableExecutionPluginProvider.class.getClassLoader(); } - return loadConfiguredPlugins( + return loadConfiguredPluginFactories( configuredNames, ServiceLoader.load(DurableExecutionPluginProvider.class, classLoader), - explicitPlugins); + explicitFactories); } - static List loadConfiguredPlugins( + static List loadConfiguredPluginFactories( String configuredNames, Iterable providers, - List explicitPlugins) { + List explicitFactories) { if (configuredNames == null || configuredNames.isBlank()) { - return List.copyOf(explicitPlugins); + return List.copyOf(explicitFactories); } var requestedNames = parseProviderNames(configuredNames); var providersByName = indexProviders(providers); - var plugins = new ArrayList(); + var factories = new ArrayList(); for (var name : requestedNames) { - addPlugin(name, getProvider(name, providersByName), plugins); + factories.add(getProvider(name, providersByName)); } - plugins.addAll(explicitPlugins); - return List.copyOf(plugins); + factories.addAll(explicitFactories); + return List.copyOf(factories); } private static List parseProviderNames(String configuredNames) { @@ -118,59 +133,135 @@ private static DurableExecutionPluginProvider getProvider( throw configurationError("No DurableExecutionPluginProvider named '" + name + "' was found on the application class path. Available providers: " + available); } + requireCreatePluginImplementation(name, provider); return provider; } - private static void addPlugin( - String name, DurableExecutionPluginProvider provider, List plugins) { - var pluginType = validateProvider(name, provider); - var plugin = createPlugin(name, provider); - if (!pluginType.isInstance(plugin)) { - throw configurationError("Plugin provider '" + name + "' declared type '" + pluginType.getName() - + "' but created '" + plugin.getClass().getName() + "'"); + /** + * Fails when a selected provider does not implement + * {@link DurableExecutionPluginFactory#createPlugin(InvocationInfo)}. + * + *

    A provider JAR compiled against an SDK version whose provider interface declared a different + * {@code createPlugin} method still loads. Its class file references nothing this version removed, so + * {@link ServiceLoader} instantiates it, {@link DurableExecutionPluginProvider#getName()} returns its name, and + * selection by name succeeds. The first call to {@code createPlugin(InvocationInfo)} then throws + * {@link AbstractMethodError}, which is contained per invocation and logged as a warning. Without this check the + * function keeps succeeding while the provider emits nothing, and the only signal is one warning per invocation. + * This check reports the condition as a startup failure instead, which is how every other provider configuration + * problem on this path is already reported. + * + *

    What is checked is the condition {@code invokeinterface} itself needs: a public, non-static, non-abstract + * method named {@code createPlugin} taking this SDK's {@link InvocationInfo} and returning exactly + * {@link DurableExecutionPlugin}, which is the erased descriptor the interface declares. Checking anything looser + * accepts class files the call cannot dispatch to. A concrete {@code MyPlugin createPlugin(InvocationInfo)} that + * overrides nothing -- which is what a class compiled against an older interface declares -- is such a file: its + * return type is a {@link DurableExecutionPlugin} subtype, so an assignability test passes it, while the interface + * call still finds no matching descriptor and throws. + * + *

    Requiring the exact descriptor cannot reject a provider that would have worked, because the descriptor is what + * dispatch resolves. A covariant override compiles to the specific method plus a bridge that returns + * {@link DurableExecutionPlugin}, and it is the bridge the interface call reaches; a compiler that emitted no + * bridge would produce a class the JVM cannot dispatch to either. The whole public method set is examined rather + * than the one {@link Class#getMethod} resolves, because that resolution prefers the most specific return type and + * so hides the bridge behind the covariant declaration, and because it searches the class before the interfaces and + * so returns a static same-signature helper in preference to the interface's declaration. + * + *

    Every property read is a class-file property, so this runs no provider code. + * + *

    {@link Class#getMethods} can raise a {@link LinkageError} while resolving a method's parameter or return type + * against a class path that cannot supply it. That is a class path problem with the same remedy, so it is reported + * as this configuration failure rather than escaping as an unexplained {@code NoClassDefFoundError}. + */ + private static void requireCreatePluginImplementation(String name, DurableExecutionPluginProvider provider) { + var providerClass = provider.getClass(); + String reason; + try { + reason = undispatchableCreatePluginReason(providerClass); + } catch (LinkageError e) { + throw configurationError( + "Plugin provider '" + name + "' (" + describe(providerClass) + + ") declares a createPlugin method whose types this class path cannot resolve. " + + REBUILD_PROVIDER_REMEDY, + e); + } + if (reason != null) { + throw configurationError("Plugin provider '" + name + "' (" + describe(providerClass) + + ") does not implement createPlugin(InvocationInfo): " + reason + + ". It was compiled against an older Durable Execution SDK whose provider interface declared a " + + "different createPlugin method. " + REBUILD_PROVIDER_REMEDY); } - plugins.add(plugin); } - private static Class validateProvider( - String name, DurableExecutionPluginProvider provider) { - int apiVersion; - Class pluginType; - try { - apiVersion = provider.getApiVersion(); - pluginType = provider.getPluginType(); - } catch (RuntimeException | LinkageError e) { - throw configurationError( - "Plugin provider '" + name + "' is not compatible with this Durable Execution SDK version", e); + /** + * Returns why no public method can serve the interface call, or null when one can. + * + *

    The reason names what was found instead, because an operator reading the failure has to be able to tell a + * provider that predates the current interface from a class path that resolves {@link InvocationInfo} to two + * different classes. + */ + private static String undispatchableCreatePluginReason(Class providerClass) { + var abstractOn = (Class) null; + var staticFound = false; + var otherReturnType = (Class) null; + for (var method : providerClass.getMethods()) { + if (!isCreatePluginCandidate(method)) { + continue; + } + var modifiers = method.getModifiers(); + if (Modifier.isStatic(modifiers)) { + staticFound = true; + } else if (Modifier.isAbstract(modifiers)) { + abstractOn = method.getDeclaringClass(); + } else if (method.getReturnType() == DurableExecutionPlugin.class) { + return null; + } else { + otherReturnType = method.getReturnType(); + } } - if (apiVersion != DurableExecutionPluginProvider.API_VERSION) { - throw configurationError("Plugin provider '" + name + "' uses provider API version " + apiVersion - + ", but this SDK requires version " + DurableExecutionPluginProvider.API_VERSION); + if (otherReturnType != null) { + return "its createPlugin(InvocationInfo) returns " + otherReturnType.getName() + + " and the class carries no method returning " + DurableExecutionPlugin.class.getName() + + ", so it overrides nothing the interface call can dispatch to"; } - if (pluginType == null - || pluginType.isInterface() - || Modifier.isAbstract(pluginType.getModifiers()) - || !DurableExecutionPlugin.class.isAssignableFrom(pluginType)) { - throw configurationError( - "Plugin provider '" + name + "' must declare a concrete DurableExecutionPlugin type"); + if (staticFound) { + return "the createPlugin(InvocationInfo) it declares is static, so it cannot implement the interface's " + + "instance method"; } - return pluginType; + if (abstractOn != null) { + return "the only declaration is the abstract one on " + abstractOn.getName(); + } + return "it declares no createPlugin method taking this SDK's " + InvocationInfo.class.getName(); + } + + /** Whether a method is named and parameterized like the factory method, whatever it returns. */ + private static boolean isCreatePluginCandidate(Method method) { + return "createPlugin".equals(method.getName()) + && method.getParameterCount() == 1 + && method.getParameterTypes()[0] == InvocationInfo.class; + } + + /** Returns the provider class name, with the artifact it was loaded from when the JVM reports one. */ + private static String describe(Class providerClass) { + var location = codeSourceLocation(providerClass); + return location == null ? providerClass.getName() : providerClass.getName() + " from " + location; } - private static DurableExecutionPlugin createPlugin(String name, DurableExecutionPluginProvider provider) { - DurableExecutionPlugin plugin; + /** + * Returns the location of the artifact a class was loaded from, or null when the JVM does not report one. + * + *

    A class defined by a loader that supplies no code source has no location, and a security manager can refuse + * the protection domain. Neither case says anything about whether the provider is usable, so neither may replace + * the configuration failure being reported. Both are therefore reported as an absent location. + */ + private static String codeSourceLocation(Class providerClass) { try { - plugin = provider.createPlugin(); - } catch (RuntimeException | LinkageError e) { - throw configurationError( - "Plugin provider '" + name + "' failed to create its plugin. " - + "Verify its settings and compatibility with this Durable Execution SDK version", - e); - } - if (plugin == null) { - throw configurationError("Plugin provider '" + name + "' returned a null plugin"); + var protectionDomain = providerClass.getProtectionDomain(); + var codeSource = protectionDomain == null ? null : protectionDomain.getCodeSource(); + var location = codeSource == null ? null : codeSource.getLocation(); + return location == null ? null : location.toString(); + } catch (RuntimeException e) { + return null; } - return plugin; } private static IllegalStateException configurationError(String message) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ApiRequestDelayedBatcher.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ApiRequestDelayedBatcher.java index 3b0def81f..50f5248c9 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ApiRequestDelayedBatcher.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ApiRequestDelayedBatcher.java @@ -7,10 +7,13 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; /** * Batches API requests to optimize throughput by grouping individual calls into batch operations. Batches are flushed @@ -34,6 +37,9 @@ public class ApiRequestDelayedBatcher { /** Executes the batch operation */ private final Consumer> executeBatch; + private final Supplier pluginFatal; + private final AtomicReference terminalFailure = new AtomicReference<>(); + /** Accumulated requests to be executed in future */ private final List> delayedBatch; @@ -64,6 +70,16 @@ public ApiRequestDelayedBatcher( int maxBatchBytes, Function calculateItemSize, Consumer> executeBatch) { + this(maxItemCount, maxBatchBytes, calculateItemSize, executeBatch, () -> null); + } + + ApiRequestDelayedBatcher( + int maxItemCount, + int maxBatchBytes, + Function calculateItemSize, + Consumer> executeBatch, + Supplier pluginFatal) { + this.pluginFatal = pluginFatal; this.maxItemCount = maxItemCount; this.maxBatchBytes = maxBatchBytes; this.calculateItemSize = calculateItemSize; @@ -85,6 +101,8 @@ public ApiRequestDelayedBatcher( */ CompletableFuture submit(T request, Duration flushDelay) { synchronized (delayedBatch) { + var fatal = getFatalFailure(); + if (fatal != null) return CompletableFuture.failedFuture(fatal); // add the request to the current batch CompletableFuture future = new CompletableFuture<>(); delayedBatch.add(new Item<>(request, future)); @@ -113,6 +131,8 @@ void shutdown() { // wait for previous batches to be flushed flushingQueueFuture.join(); + var fatal = getFatalFailure(); + if (fatal != null) throw new CompletionException(fatal); } /** clear the current batch and creates a new batch */ @@ -144,8 +164,31 @@ private void flushDelayedBatch() { // Schedule a new flushing future. If the items in this batch have been executed by the previous flushQueue // future, // the new future will just do nothing. - flushingQueueFuture = flushingQueueFuture.thenRunAsync(this::flushQueue, InternalExecutor.INSTANCE); + flushingQueueFuture = flushingQueueFuture.thenCompose(ignored -> flushObservedAsync()); + } + + private CompletableFuture flushObservedAsync() { + var completion = new CompletableFuture(); + // Keep sequencing through the future, but let reported plugin fatals escape the actual worker. + // CompletableFuture async stages would otherwise catch the fatal again after flushQueue rethrows it. + Runnable flush = (Runnable & CompletableFuture.AsynchronousCompletionTask) () -> { + try { + flushQueue(); + completion.complete(null); + } catch (Throwable failure) { + completion.completeExceptionally(failure); + var fatal = terminalFailure.get(); + if (fatal != null) throw fatal; + } + }; + try { + InternalExecutor.INSTANCE.execute(flush); + } catch (Throwable failure) { + completion.completeExceptionally(failure); + } + return completion; } + /** Call checkpoint API with items in the flushing queue */ private void flushQueue() { // There could be more items to flush because @@ -171,7 +214,10 @@ private void flushQueue() { break; } - flushingItems.add(flushingQueue.poll()); + // An external fatal publisher can drain the queue between the peek and this poll. + var next = flushingQueue.poll(); + if (next == null) break; + flushingItems.add(next); flushingSize += itemSizeInByte; } if (!flushingItems.isEmpty()) { @@ -181,16 +227,50 @@ private void flushQueue() { .map(Item::request) .filter(Objects::nonNull) .toList(); + // Another worker may already have failed the invocation, or may do so during this call. + // A skipped checkpoint must never look like a successful START to an at-most-once step. + rethrowPluginFatalIfPresent(); executeBatch.accept(requests); + rethrowPluginFatalIfPresent(); for (Item item : flushingItems) { item.result().complete(null); } } catch (Throwable ex) { + var fatal = getFatalFailure(); for (Item item : flushingItems) { - item.result().completeExceptionally(ex); + item.result().completeExceptionally(fatal == null ? ex : fatal); + } + if (fatal != null) { + abortPending(fatal); + throw fatal; } } } } } + + Error getFatalFailure() { + var fatal = terminalFailure.get(); + return fatal == null ? pluginFatal.get() : fatal; + } + + private void rethrowPluginFatalIfPresent() { + var fatal = getFatalFailure(); + if (fatal != null) throw fatal; + } + + /** Settles delayed and queued requests without waiting for their timer or an in-flight backend call. */ + void abortPending(Error fatal) { + terminalFailure.compareAndSet(null, fatal); + var original = terminalFailure.get(); + var pending = new ArrayList>(); + synchronized (delayedBatch) { + delayedBatchFlushTimer.cancel(false); + pending.addAll(delayedBatch); + delayedBatch.clear(); + Item item; + while ((item = flushingQueue.poll()) != null) pending.add(item); + } + pending.forEach(item -> item.result().completeExceptionally(original)); + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java index 34f44134c..e7196342d 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java @@ -13,6 +13,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.BooleanSupplier; import java.util.function.Consumer; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.awscore.exception.AwsServiceException; @@ -60,6 +61,24 @@ class CheckpointManager { Consumer> callback, BooleanSupplier tryStartCheckpointProcessing, Runnable finishCheckpointProcessing) { + this( + config, + durableExecutionArn, + checkpointToken, + callback, + tryStartCheckpointProcessing, + finishCheckpointProcessing, + () -> null); + } + + CheckpointManager( + DurableConfig config, + String durableExecutionArn, + String checkpointToken, + Consumer> callback, + BooleanSupplier tryStartCheckpointProcessing, + Runnable finishCheckpointProcessing, + Supplier pluginFatal) { this.config = config; this.durableExecutionArn = durableExecutionArn; this.callback = callback; @@ -67,7 +86,11 @@ class CheckpointManager { this.tryStartCheckpointProcessing = tryStartCheckpointProcessing; this.finishCheckpointProcessing = finishCheckpointProcessing; this.checkpointApiRequestDelayedBatcher = new ApiRequestDelayedBatcher<>( - MAX_ITEM_COUNT, MAX_BATCH_SIZE_BYTES, CheckpointManager::estimateSize, this::checkpointBatch); + MAX_ITEM_COUNT, + MAX_BATCH_SIZE_BYTES, + CheckpointManager::estimateSize, + this::checkpointBatch, + pluginFatal); } /** @@ -132,20 +155,31 @@ private CompletableFuture pollForUpdateInternal( // the delay is the polling interval minus the time already elapsed in the current attempt var delay = pollingStrategy.computeDelay(attempt).minus(Duration.between(startTime, Instant.now())); - return checkpointApiRequestDelayedBatcher.submit(null, delay).thenCompose(v -> { - if (future.isDone()) { - return CompletableFuture.completedFuture(null); - } - var now = Instant.now(); - if (Duration.between(startTime, now).compareTo(pollingStrategy.computeDelay(attempt)) > 0) { - // It has exceeded the previous attempt duration, starting a new attempt - return pollForUpdateInternal(future, attempt + 1, now, pollingStrategy); - } else { - // continue the previous attempt. The future was completed just because - // it was batched with other checkpoint API calls. - return pollForUpdateInternal(future, attempt, startTime, pollingStrategy); - } - }); + return checkpointApiRequestDelayedBatcher + .submit(null, delay) + .whenComplete((ignored, failure) -> { + var fatal = checkpointApiRequestDelayedBatcher.getFatalFailure(); + if (failure != null && fatal != null) future.completeExceptionally(fatal); + }) + .thenCompose(v -> { + if (future.isDone()) { + return CompletableFuture.completedFuture(null); + } + var now = Instant.now(); + if (Duration.between(startTime, now).compareTo(pollingStrategy.computeDelay(attempt)) > 0) { + // It has exceeded the previous attempt duration, starting a new attempt + return pollForUpdateInternal(future, attempt + 1, now, pollingStrategy); + } else { + // continue the previous attempt. The future was completed just because + // it was batched with other checkpoint API calls. + return pollForUpdateInternal(future, attempt, startTime, pollingStrategy); + } + }); + } + + /** Fails queued checkpoint and polling requests without taking the backend/poller lock. */ + void abortPending(Error fatal) { + checkpointApiRequestDelayedBatcher.abortPending(fatal); } /** Cancels all polling futures and waits for all pending checkpoint requests to complete */ @@ -209,7 +243,7 @@ private void checkpointBatch(List updates) { // Starting the backend request is coordinated with the last-thread suspension decision. Once suspension // wins that race, no later poll/checkpoint may advance backend state behind the PENDING response. if (!tryStartCheckpointProcessing.getAsBoolean()) { - if (!request.isEmpty()) { + if (!request.isEmpty() && checkpointApiRequestDelayedBatcher.getFatalFailure() == null) { logger.error( "Checkpoint invariant violation: skipping {} operation updates because execution has already" + " completed", diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index d8db91326..407d21d94 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -7,10 +7,17 @@ import java.nio.charset.StandardCharsets; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationAction; @@ -26,6 +33,7 @@ import software.amazon.lambda.durable.logging.DurableLogger; import software.amazon.lambda.durable.model.DurableExecutionInput; import software.amazon.lambda.durable.model.DurableExecutionOutput; +import software.amazon.lambda.durable.model.SafeCloseable; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.InvocationStatus; @@ -47,6 +55,11 @@ public class DurableExecutor { // Lambda response size limit is 6MB minus small epsilon for envelope private static final int LAMBDA_RESPONSE_SIZE_LIMIT = 6 * 1024 * 1024 - 50; + // Best-effort allowance for each configured plugin's finalization (including the bundled OTel 5s join), + // plus shutdown/response headroom. Existing arbitrary callbacks and checkpoint draining can exceed it. + private static final long PLUGIN_FINALIZATION_RESERVE_MILLIS = 5_000; + private static final long SHUTDOWN_RESPONSE_RESERVE_MILLIS = 1_000; + private DurableExecutor() {} public static DurableExecutionOutput execute( @@ -55,16 +68,22 @@ public static DurableExecutionOutput execute( TypeToken inputType, BiFunction handler, DurableConfig config) { - var pluginRunner = config.getPluginRunner(); - try (var executionManager = new ExecutionManager(input, config, lambdaContext)) { + var pluginFatal = new AtomicReference(); + try (var executionManager = new ExecutionManager(input, config, lambdaContext, pluginFatal)) { + // Scoped to this invocation: the runner creates this invocation's plugin instances from the configured + // factories when onInvocationStart fires below, and releases them when the manager closes. + var pluginRunner = executionManager.getPluginRunner(); var isFirstInvocation = !executionManager.isReplaying(); var requestId = lambdaContext != null ? lambdaContext.getAwsRequestId() : null; var executionArn = input.durableExecutionArn(); + // Capture on the runtime thread before dispatch: LMI trace carriers can be thread-local. + var xRayTraceId = pluginRunner.isEmpty() ? null : RuntimeTraceHeader.capture(lambdaContext); executionManager.registerActiveThread(null); // Captured for onInvocationEnd, which runs outside the handler thread below. var pluginExecutionInput = new AtomicReference<>(); - var handlerFuture = CompletableFuture.supplyAsync( + var hasHandlerScope = new AtomicBoolean(); + var handlerFuture = supplyAsync( () -> { executionManager.setCurrentThreadContext(new ThreadContext(null, ThreadType.CONTEXT)); @@ -99,7 +118,8 @@ public static DurableExecutionOutput execute( executionManager.getInitialOperationIds()), PluginInfoConverter.toOperationItemMap( executionManager.getUpdatedOperationsSnapshot(), - executionManager.getInitialOperationIds()))); + executionManager.getInitialOperationIds()), + xRayTraceId)); } if (inputFailure != null) { ExceptionHelper.sneakyThrow(inputFailure); @@ -109,21 +129,48 @@ public static DurableExecutionOutput execute( DurableContextImpl.setCurrentContext(context); // use a try-with-resources to clear logger properties try (var ignored = DurableLogger.attachContext()) { - return handler.apply(userInput, context); + var handlerInput = userInput; + return pluginRunner.runHandler( + () -> handler.apply(handlerInput, context), + () -> hasHandlerScope.set(true), + executionManager::recordHandlerScopeFatal); } }, - config.getExecutorService()); // Get executor from config for running user code + config.getExecutorService(), + !pluginRunner.isEmpty()); // Get executor from config for running user code // Execute the handlerFuture in ExecutionManager. If it completes successfully, the output of user function // will be returned. Otherwise, it will complete exceptionally with a SuspendExecutionException or a // failure. try { - return executionManager - .runUntilCompleteOrSuspend(handlerFuture) - .handle((result, ex) -> { + return finalizeAfterHandlerScopes( + executionManager.runUntilCompleteOrSuspend(handlerFuture), + handlerFuture, + hasHandlerScope, + lambdaContext, + config.getPluginFactories().size(), + pluginFatal, + (result, ex) -> { if (ex != null) { // an exception thrown from handlerFuture or suspension/termination occurred - Throwable cause = ExceptionHelper.unwrapCompletableFuture(ex); + Throwable cause = normalizeInvocationFailure(ex); + + if (isFatal(cause)) { + try { + fireOnInvocationEnd( + pluginRunner, + executionManager, + requestId, + executionArn, + isFirstInvocation, + InvocationStatus.RETRYING, + cause, + pluginExecutionInput.get(), + null); + } finally { + rethrowFatal(cause); + } + } // return PENDING if it's SuspendExecutionException if (cause instanceof SuspendExecutionException) { @@ -175,9 +222,45 @@ public static DurableExecutionOutput execute( } // user handler complete successfully logger.debug("Execution completed"); - var outputPayload = config.getSerDes().serialize(result); - var output = - DurableExecutionOutput.success(handleLargePayload(executionManager, outputPayload)); + // Serializing the result and checkpointing an oversized one can both fail, and this + // invocation ends either way. The end hook is the only point at which a plugin can finish: + // releasePlugins() calls nothing on the instances it drops and the contract has no close(), + // so an exit that skips the hook discards everything the plugin holds -- Insight's record + // for the execution and every exporter's flush, and both OTel plugins' invocation and + // Workflow spans. It also leaves a record queued for a pump that will export it after this + // invocation has returned, which is the out-of-order delivery drainUntilSettled exists to + // prevent. The status is RETRYING rather than FAILED because the throw below leaves the + // invocation the way a retryable failure does: the execution is not finished, and the + // backend decides whether a new invocation follows. + DurableExecutionOutput output = null; + Throwable resultDeliveryFailure = null; + try { + var outputPayload = config.getSerDes().serialize(result); + output = DurableExecutionOutput.success( + handleLargePayload(executionManager, outputPayload)); + } catch (Throwable failure) { + // handleLargePayload waits with join(), so a failed checkpoint arrives wrapped in a + // CompletionException. The plugins are told what failed, not how it was delivered, and + // the failure branches above already unwrap before they report -- so unwrap here too, + // or Insight's record and the OTel span status would name the wrapper. + resultDeliveryFailure = normalizeInvocationFailure(failure); + if (resultDeliveryFailure == null) { + resultDeliveryFailure = failure; + } + } + if (resultDeliveryFailure != null) { + fireOnInvocationEnd( + pluginRunner, + executionManager, + requestId, + executionArn, + isFirstInvocation, + InvocationStatus.RETRYING, + resultDeliveryFailure, + pluginExecutionInput.get(), + null); + ExceptionHelper.sneakyThrow(resultDeliveryFailure); + } fireOnInvocationEnd( pluginRunner, executionManager, @@ -189,16 +272,154 @@ public static DurableExecutionOutput execute( pluginExecutionInput.get(), result); return output; - }) - .join(); + }); } catch (CompletionException e) { // unwrap the CompletionException and rethrow the wrapped exception - ExceptionHelper.sneakyThrow(ExceptionHelper.unwrapCompletableFuture(e)); + ExceptionHelper.sneakyThrow(normalizeInvocationFailure(e)); return null; } + } catch (CompletionException failure) { + // Keep resource-close wrappers intact until after try-with-resources has combined exceptions; throwing + // the same fatal from both the body and close would trigger illegal self-suppression. + var fatal = pluginFatal.get(); + if (fatal != null && normalizeInvocationFailure(failure) == fatal) throw fatal; + throw failure; } } + /** Completes the observation future without absorbing fatal failures on an executor worker. */ + private static CompletableFuture supplyAsync( + Supplier task, Executor executor, boolean preservePluginMdc) { + var result = new CompletableFuture(); + var caller = Thread.currentThread(); + Runnable work = (Runnable & CompletableFuture.AsynchronousCompletionTask) () -> { + try { + T value; + // Preserve the instrumented owner's context even when startup fails. Without plugins, retain + // the existing worker cleanup without adding MDC snapshot reads. + try (SafeCloseable ignored = preservePluginMdc ? restoreMdcOnClose() : MDC::clear) { + value = task.get(); + } + result.complete(value); + } catch (Throwable failure) { + result.completeExceptionally(failure); + // A direct executor is already on the invocation caller; its fatal result is rethrown below after + // finalization. On an asynchronous executor it must also escape the runnable so the worker terminates. + if (Thread.currentThread() != caller) { + rethrowFatal(ExceptionHelper.unwrapAsyncFailure(failure)); + } + } + }; + executor.execute(work); + return result; + } + + private static Throwable normalizeInvocationFailure(Throwable failure) { + var unwrapped = ExceptionHelper.unwrapAsyncFailure(failure); + if (isFatal(unwrapped)) return unwrapped; + // ExecutionException can be an application failure itself. Only CompletionException is the SDK's + // ordinary transport here; retain the application's error type, message, and serialized cause chain. + while (failure instanceof CompletionException && failure.getCause() != null) { + failure = failure.getCause(); + } + return failure; + } + + @SuppressWarnings("removal") + private static boolean isFatal(Throwable failure) { + return failure instanceof VirtualMachineError || failure instanceof ThreadDeath; + } + + @SuppressWarnings("removal") + private static void rethrowFatal(Throwable failure) { + if (failure instanceof VirtualMachineError fatal) throw fatal; + if (failure instanceof ThreadDeath fatal) throw fatal; + } + + private static R finalizeAfterHandlerScopes( + CompletableFuture executionFuture, + CompletableFuture handlerFuture, + AtomicBoolean hasHandlerScope, + Context lambdaContext, + int pluginCount, + AtomicReference pluginFatal, + BiFunction finalizer) { + var started = new AtomicBoolean(); + var finalized = new CompletableFuture(); + // Attach before waiting so legacy hooks retain normal CompletableFuture completion-thread dispatch. + var ready = executionFuture.handle((value, failure) -> { + Runnable finish = () -> completeFinalization(started, finalized, () -> { + // Match the existing major lifecycle: a positively reported instrumentation fatal is finalized + // as RETRYING, then rethrown. No factory or end-snapshot policy changes here. + var fatal = pluginFatal.get(); + return finalizer.apply(value, fatal == null ? failure : fatal); + }); + if (!hasHandlerScope.get() || handlerFuture.isDone()) finish.run(); + else handlerFuture.whenComplete((ignored, ignoredFailure) -> finish.run()); + return finish; + }); + var finish = ready.join(); + if (!started.get()) { + // Only the caller waits: a signaling handler must be free to unwind and close its scopes. + awaitHandlerScopes( + executionFuture, handlerFuture, hasHandlerScope, lambdaContext, pluginCount, pluginFatal); + finish.run(); + } + return finalized.join(); + } + + private static void completeFinalization( + AtomicBoolean started, CompletableFuture result, Supplier finalizer) { + if (!started.compareAndSet(false, true)) return; + try { + result.complete(finalizer.get()); + } catch (Throwable failure) { + // Match CompletableFuture.handle: transfer failures unchanged for the invocation caller to rethrow. + result.completeExceptionally(failure); + } + } + + static CompletableFuture awaitHandlerScopes( + CompletableFuture executionFuture, + CompletableFuture handlerFuture, + AtomicBoolean hasHandlerScope, + Context lambdaContext, + int pluginCount, + AtomicReference pluginFatal) { + // This method runs on the invocation caller, never as a callback on the signaling handler worker. + // Preserve the winning outcome except for an observed fatal error positively reported by plugin + // instrumentation. + var failure = executionFuture.handle((result, error) -> error).join(); + var fatal = pluginFatal.get(); + if (fatal != null) return CompletableFuture.failedFuture(fatal); + if (failure == null || !hasHandlerScope.get()) return executionFuture; + var reserve = SHUTDOWN_RESPONSE_RESERVE_MILLIS + PLUGIN_FINALIZATION_RESERVE_MILLIS * pluginCount; + var budgetMillis = lambdaContext == null + ? 500L + : Math.min(500L, Math.max(0L, (long) lambdaContext.getRemainingTimeInMillis() - reserve)); + try { + handlerFuture.handle((result, error) -> null).get(budgetMillis, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + logger.warn("Handler scope cleanup exceeded its handoff budget; cleanup continues on the handler thread"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("Interrupted while awaiting handler scope cleanup; preserving the execution outcome", e); + } catch (ExecutionException e) { + logger.warn("Could not observe handler scope cleanup; preserving the execution outcome", e); + } + // The major lifecycle finalizes created plugins once before propagating an observed fatal. + fatal = pluginFatal.get(); + return fatal == null ? executionFuture : CompletableFuture.failedFuture(fatal); + } + + private static SafeCloseable restoreMdcOnClose() { + var previous = MDC.getCopyOfContextMap(); + return () -> { + if (previous == null) MDC.clear(); + else MDC.setContextMap(previous); + }; + } + private static void fireOnInvocationEnd( PluginRunner pluginRunner, ExecutionManager executionManager, @@ -212,17 +433,32 @@ private static void fireOnInvocationEnd( if (pluginRunner.isEmpty()) { return; } - pluginRunner.onInvocationEnd(new InvocationEndInfo( - requestId, - executionArn, - isFirstInvocation, - executionManager.getExecutionOperation().startTimestamp(), - PluginInfoConverter.toOperationItemMap( - executionManager.getOperationsSnapshot(), executionManager.getInitialOperationIds()), - status, - error, - executionInput, - executionResult)); + // Freeze the caller outcome immediately before finalization. Observe instrumentation fatals already + // reported at this boundary, retaining an earlier invocation fatal. Later scope failures still escape their + // owner thread, but cannot rewrite the outcome whose one end snapshot has already been dispatched. + var reported = executionManager.beginInvocationFinalization(); + var original = normalizeInvocationFailure(error); + var fatal = isFatal(original) ? (Error) original : reported; + if (fatal != null) { + status = InvocationStatus.RETRYING; + error = fatal; + executionResult = null; + } + try (var ignored = restoreMdcOnClose()) { + pluginRunner.onInvocationEnd(new InvocationEndInfo( + requestId, + executionArn, + isFirstInvocation, + executionManager.getExecutionOperation().startTimestamp(), + PluginInfoConverter.toOperationItemMap( + executionManager.getOperationsSnapshot(), executionManager.getInitialOperationIds()), + status, + error, + executionInput, + executionResult)); + } finally { + if (fatal != null) throw fatal; + } } private static String handleLargePayload(ExecutionManager executionManager, String outputPayload) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index 4c7feeda4..4ca9a9d58 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -14,6 +15,7 @@ import java.util.Set; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicReference; @@ -32,6 +34,7 @@ import software.amazon.lambda.durable.model.SafeCloseable; import software.amazon.lambda.durable.operation.BaseDurableOperation; import software.amazon.lambda.durable.plugin.PluginInfoConverter; +import software.amazon.lambda.durable.plugin.PluginRunner; import software.amazon.lambda.durable.util.ExceptionHelper; /** @@ -69,6 +72,15 @@ public class ExecutionManager implements SafeCloseable { private final Set updatedOperationIdsSinceLastInvocation; private final Set initialOperationIds; + // ===== Plugins ===== + // Created per invocation, alongside this manager: the runner materializes one plugin instance per configured + // factory when the invocation starts, and releases them in close(), so instances never outlive the invocation. + private final PluginRunner pluginRunner; + private final AtomicReference pluginFatal; + private final Object pluginFinalizationLock = new Object(); + private boolean invocationFinalized; + private Error lateScopeFatal; + // ===== Thread Coordination ===== private final Map registeredOperations = new ConcurrentHashMap<>(); private final Map operationCompletionLocks = new ConcurrentHashMap<>(); @@ -94,7 +106,17 @@ private static final class CompletionWait { private final CheckpointManager checkpointManager; public ExecutionManager(DurableExecutionInput input, DurableConfig config, Context lambdaContext) { + this(input, config, lambdaContext, new AtomicReference<>()); + } + + ExecutionManager( + DurableExecutionInput input, + DurableConfig config, + Context lambdaContext, + AtomicReference pluginFatal) { durableConfig = config; + this.pluginFatal = pluginFatal; + this.pluginRunner = new PluginRunner(config.getPluginFactories(), this::failFromPlugin); this.durableExecutionArn = input.durableExecutionArn(); this.lambdaContext = lambdaContext; @@ -109,7 +131,8 @@ public ExecutionManager(DurableExecutionInput input, DurableConfig config, Conte input.checkpointToken(), this::onCheckpointComplete, this::tryStartCheckpointProcessing, - this::finishCheckpointProcessing); + this::finishCheckpointProcessing, + pluginFatal::get); this.operationStorage = checkpointManager.fetchAllPages(input.initialExecutionState()).stream() .collect(Collectors.toConcurrentMap(Operation::id, op -> op)); @@ -140,6 +163,15 @@ public ExecutionManager(DurableExecutionInput input, DurableConfig config, Conte // ===== State Management ===== + /** + * Returns this invocation's plugin dispatcher. Scoped to this manager, i.e. to this invocation. + * + * @return PluginRunner instance (never null) + */ + public PluginRunner getPluginRunner() { + return pluginRunner; + } + /** Returns the ARN of the durable execution being managed. */ public String getDurableExecutionArn() { return durableExecutionArn; @@ -226,15 +258,11 @@ public void registerOperation(BaseDurableOperation operation, BaseDurableOperati // ===== Checkpoint Completion Handler ===== /** Called by CheckpointManager when a checkpoint completes. Updates operationStorage and notify operations . */ void onCheckpointComplete(List newOperations) { - var updatedOperations = new ArrayList(); + if (!pluginRunner.isEmpty()) notifyCheckpointChangesBeforeCompletion(newOperations); + newOperations.forEach(op -> { - // Detect a status change against the previously stored operation - var previous = operationStorage.get(op.id()); - if (previous == null || previous.status() != op.status()) { - updatedOperations.add(op); - } - // Publish the updated state and notify its waiter atomically. Otherwise, a waiter can observe the terminal - // state before its completion future is completed and attempt to suspend with no pending operations. + // Keep storage publication and waiter notification atomic: publishing terminal state ahead of completion + // can make a waiter attempt to suspend despite there being no pending operation. registeredOperations.compute(op.id(), (id, registeredOperation) -> { if (registeredOperation == null) { operationStorage.put(op.id(), op); @@ -244,18 +272,28 @@ void onCheckpointComplete(List newOperations) { return registeredOperation; }); }); + } + + private void notifyCheckpointChangesBeforeCompletion(List newOperations) { + var updatedOperations = new ArrayList(); + var checkpointSnapshot = new HashMap<>(operationStorage); + for (var operation : newOperations) { + var previous = checkpointSnapshot.put(operation.id(), operation); + if (previous == null || previous.status() != operation.status()) { + updatedOperations.add(operation); + } + } - // Fire onOperationChange when a checkpoint response changed one or more operations + // Observe the complete response before releasing any terminal waiter. A fatal observer failure must stop + // this checkpoint's continuation work. The payload sees new state without publishing terminal storage early. if (!updatedOperations.isEmpty()) { var requestId = lambdaContext != null ? lambdaContext.getAwsRequestId() : null; - durableConfig - .getPluginRunner() - .onOperationChange(PluginInfoConverter.toOperationChangeInfo( - requestId, - durableExecutionArn, - updatedOperations, - operationStorage.values(), - initialOperationIds)); + pluginRunner.onOperationChange(PluginInfoConverter.toOperationChangeInfo( + requestId, + durableExecutionArn, + updatedOperations, + checkpointSnapshot.values(), + initialOperationIds)); } } @@ -520,9 +558,16 @@ public void deregisterActiveThread(String threadId) { } } + /** Removes work that was never accepted by its executor; the dispatch caller propagates the submission error. */ + public void cancelThreadRegistration(String threadId) { + synchronized (activeThreads) { + activeThreads.remove(threadId); + } + } + boolean tryStartCheckpointProcessing() { synchronized (activeThreads) { - if (executionExceptionFuture.isDone()) { + if (executionExceptionFuture.isDone() || pluginFatal.get() != null) { return false; } checkpointRequestsInFlight++; @@ -545,7 +590,10 @@ void finishCheckpointProcessing() { } private boolean shouldSuspendExecution() { - return activeThreads.isEmpty() && checkpointRequestsInFlight == 0 && !executionExceptionFuture.isDone(); + return activeThreads.isEmpty() + && checkpointRequestsInFlight == 0 + && !executionExceptionFuture.isDone() + && pluginFatal.get() == null; } private void preSuspendCheck() { @@ -595,9 +643,24 @@ public CompletableFuture pollForOperationUpdates(String operationId, /** Shutdown the checkpoint batcher. */ @Override public void close() { - validateRunningThreads(); - - checkpointManager.shutdown(); + try { + validateRunningThreads(); + try { + checkpointManager.shutdown(); + } catch (CompletionException failure) { + if (!isLateScopeFatal(failure)) throw failure; + // The original fatal still escapes its owner. Do not deliver its batcher copy a second time + // on the invocation caller after the caller's one end snapshot has already been finalized. + logger.warn( + "Handler scope failed after invocation finalization; preserving finalized outcome", failure); + } + } finally { + // The invocation is over: drop this invocation's plugin instances so they cannot be reached again. + // In a finally, because validateRunningThreads throws on a stuck user handler: leaving the instances + // in place then carries them into the next invocation the environment hosts, which is the + // cross-execution sharing the per-invocation lifetime exists to prevent. + pluginRunner.releasePlugins(); + } } private void validateRunningThreads() { @@ -670,6 +733,49 @@ public boolean isExecutionCompletedExceptionally() { return executionExceptionFuture.isCompletedExceptionally(); } + private void failFromPlugin(Error fatal) { + var original = recordPluginFatal(fatal, false); + executionExceptionFuture.completeExceptionally(original); + stopAllOperations(original); + checkpointManager.abortPending(original); + } + + /** Stops pending work while leaving the root worker responsible for completing its scope cleanup. */ + void recordHandlerScopeFatal(Error fatal) { + var original = recordPluginFatal(fatal, true); + stopAllOperations(original); + checkpointManager.abortPending(original); + } + + /** Freezes the caller's end snapshot without waiting longer for the handler owner. */ + Error beginInvocationFinalization() { + synchronized (pluginFinalizationLock) { + invocationFinalized = true; + return pluginFatal.get(); + } + } + + private Error recordPluginFatal(Error fatal, boolean fromScope) { + synchronized (pluginFinalizationLock) { + if (pluginFatal.compareAndSet(null, fatal) && fromScope && invocationFinalized) { + lateScopeFatal = fatal; + } + return pluginFatal.get(); + } + } + + private boolean isLateScopeFatal(Throwable failure) { + synchronized (pluginFinalizationLock) { + return lateScopeFatal != null && ExceptionHelper.unwrapAsyncFailure(failure) == lateScopeFatal; + } + } + + /** Once plugin instrumentation has failed fatally, do not retry or persist unrelated operation outcomes. */ + public void rethrowPluginFatalIfPresent() { + var fatal = pluginFatal.get(); + if (fatal != null) throw fatal; + } + private void stopAllOperations(Throwable cause) { registeredOperations.values().forEach(op -> op.getCompletionFuture().completeExceptionally(cause)); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/RuntimeTraceHeader.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/RuntimeTraceHeader.java new file mode 100644 index 000000000..fa5c4086d --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/RuntimeTraceHeader.java @@ -0,0 +1,68 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.execution; + +import com.amazonaws.services.lambda.runtime.Context; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Modifier; +import java.lang.reflect.UndeclaredThrowableException; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Captures optional invocation data while retaining older visible Lambda Context API compatibility. */ +final class RuntimeTraceHeader { + private static final Logger logger = LoggerFactory.getLogger(RuntimeTraceHeader.class); + + private RuntimeTraceHeader() {} + + static String capture(Context context) { + if (context == null) return null; + try { + // Check the actual visible interface first: a legacy class may independently declare the same helper. + if (!hasAccessorApi()) return null; + var accessor = context.getClass().getMethod("getXrayTraceId"); + // Lambda Core 1.4's neutral default is not an invocation-local carrier. + if (accessor.getDeclaringClass() == Context.class + || accessor.getReturnType() != String.class + || Modifier.isStatic(accessor.getModifiers())) return null; + var header = context.getXrayTraceId(); + return header == null ? "" : header; + } catch (Throwable failure) { + rethrowFatal(failure); + // A failed available override must not borrow another invocation's global trace or sampling. + logger.debug("Lambda Context X-Ray capture failed; treating invocation header as absent"); + return ""; + } + } + + private static boolean hasAccessorApi() { + try { + Context.class.getMethod("getXrayTraceId"); + return true; + } catch (NoSuchMethodException unavailable) { + return false; + } + } + + @SuppressWarnings("removal") + private static void rethrowFatal(Throwable failure) { + if (failure instanceof VirtualMachineError fatal) throw fatal; + if (failure instanceof ThreadDeath fatal) throw fatal; + var seen = Collections.newSetFromMap(new IdentityHashMap()); + var cause = failure; + while (seen.add(cause)) { + if (cause instanceof VirtualMachineError fatal) throw fatal; + if (cause instanceof ThreadDeath fatal) throw fatal; + if (!(cause instanceof CompletionException + || cause instanceof ExecutionException + || cause instanceof InvocationTargetException + || cause instanceof UndeclaredThrowableException) + || cause.getCause() == null) return; + cause = cause.getCause(); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java index a1514e6d5..c5a87d259 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java @@ -129,6 +129,7 @@ public OperationType getType() { * otherwise starts fresh execution. */ public void execute() { + executionManager.rethrowPluginFatalIfPresent(); if (isVirtual) { // Virtual operations are not checkpointed, but we still fire plugin hooks // so the OTel plugin can emit spans for map/parallel iterations. @@ -271,14 +272,18 @@ protected Operation waitForOperationCompletion() { * @param threadType the thread type (STEP or CONTEXT) */ protected void runUserHandler(Runnable runnable, ThreadType threadType) { + executionManager.rethrowPluginFatalIfPresent(); String operationId = getOperationId(); logger.debug("Starting user handler for operation {} ({})", operationId, threadType); Runnable wrapped = () -> { executionManager.setCurrentThreadContext(new ThreadContext(operationId, threadType)); try { + // A task accepted before another hook failed may only now be starting on its worker. + executionManager.rethrowPluginFatalIfPresent(); runnable.run(); } catch (Throwable throwable) { + executionManager.rethrowPluginFatalIfPresent(); // Operations wrap the user function and handle all outcomes except for SuspendExecutionException. // Anything else reaching here is unexpected and terminates the execution. if (!executionManager.isExecutionCompletedExceptionally() @@ -321,8 +326,29 @@ protected void runUserHandler(Runnable runnable, ThreadType threadType) { // registerActiveThread is idempotent (no-op if already registered). registerActiveThread(operationId); - runningUserHandler.set(CompletableFuture.runAsync( - wrapped, getContext().getDurableConfig().getExecutorService())); + var completion = new CompletableFuture(); + runningUserHandler.set(completion); + Runnable observed = (Runnable & CompletableFuture.AsynchronousCompletionTask) () -> { + try { + wrapped.run(); + completion.complete(null); + } catch (Throwable failure) { + completion.completeExceptionally(failure); + // Settle accepted work before propagating on either an async worker or the direct caller. The + // invocation boundary still finalizes plugins, but the operation call must not return normally. + executionManager.rethrowPluginFatalIfPresent(); + } + }; + try { + getContext().getDurableConfig().getExecutorService().execute(observed); + } catch (Throwable failure) { + // No accepted task will settle a rejected submission. Complete the published future so shutdown cannot + // wait forever, and remove its reservation without turning submission failure into suspension. + if (completion.completeExceptionally(failure) && operationId != null) { + executionManager.cancelThreadRegistration(operationId); + } + ExceptionHelper.sneakyThrow(failure); + } } /** @@ -354,6 +380,7 @@ protected T runUserFunction(Integer attempt, Supplier userFunction) { PluginInfoConverter.toUserFunctionEndInfo(startInfo, UserFunctionOutcome.SUCCEEDED, null)); return result; } catch (Throwable e) { + executionManager.rethrowPluginFatalIfPresent(); var error = ExceptionHelper.unwrapCompletableFuture(e); if (error == null) { error = e; @@ -537,10 +564,13 @@ public CompletableFuture getRunningUserHandler() { // ─── Plugin hook helpers ───────────────────────────────────────────── - /** Returns the plugin runner from config, or no-op if config is unavailable. */ + /** + * Returns this invocation's plugin runner, scoped to the ExecutionManager of this invocation. Falls back to a no-op + * runner when the manager does not provide one (mocked managers in unit tests). + */ private PluginRunner getPluginRunner() { - var config = getContext().getDurableConfig(); - return config != null ? config.getPluginRunner() : PluginRunner.noOp(); + var pluginRunner = executionManager.getPluginRunner(); + return pluginRunner != null ? pluginRunner : PluginRunner.noOp(); } /** Fires onOperationStart plugin hook. */ diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java index 8c299cfa4..5065572a9 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java @@ -187,6 +187,7 @@ private void checkpointSuccess(T result, String serialized) { } private void handleChildContextFailure(Throwable exception) { + executionManager.rethrowPluginFatalIfPresent(); exception = ExceptionHelper.unwrapCompletableFuture(exception); if (exception instanceof SuspendExecutionException suspendExecutionException) { // Rethrow Error immediately — do not checkpoint diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java index 467a87b94..15d9dda05 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java @@ -157,6 +157,7 @@ private void handleStepSucceeded(T result) { } private void handleStepFailure(Throwable exception, int attempt) { + executionManager.rethrowPluginFatalIfPresent(); exception = ExceptionHelper.unwrapCompletableFuture(exception); if (exception instanceof SuspendExecutionException suspendExecutionException) { throw suspendExecutionException; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java index 72eb65b8e..b7ce5edfd 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java @@ -173,6 +173,7 @@ private void executeCheckLogic(T currentState, int attempt) { } private void handleCheckFailure(Throwable exception) { + executionManager.rethrowPluginFatalIfPresent(); exception = ExceptionHelper.unwrapCompletableFuture(exception); if (exception instanceof SuspendExecutionException suspendExecutionException) { throw suspendExecutionException; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPlugin.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPlugin.java index e2f8a46df..ac320f42b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPlugin.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPlugin.java @@ -10,8 +10,8 @@ * *

    All methods have default no-op implementations, allowing plugins to override only the hooks they need. * - *

    Plugin errors are isolated — exceptions thrown by plugin methods are caught and logged but never disrupt SDK - * execution. + *

    Non-fatal plugin failures are caught and logged without disrupting SDK execution. {@link VirtualMachineError} and + * {@link ThreadDeath} propagate, including when wrapped by asynchronous completion/future exceptions. */ public interface DurableExecutionPlugin { @@ -21,7 +21,8 @@ public interface DurableExecutionPlugin { * Called at the start of each Lambda invocation. Use to set up per-invocation state (trace ID, invocation span). * *

    Check {@link InvocationInfo#isFirstInvocation()} to detect the first invocation of an execution (useful for - * sampling decisions or execution-level span creation). + * sampling decisions or execution-level span creation). The runtime-thread snapshot is available through + * {@link InvocationInfo#xRayTraceId()}. */ default void onInvocationStart(InvocationInfo info) {} @@ -79,6 +80,10 @@ default void onUserFunctionStart(UserFunctionStartInfo info) {} * *

    This hook fires on the same thread as user code, so plugins can close OTel scopes here. * + *

    If a later plugin's start hook fails fatally, plugins whose start hooks completed successfully receive this + * hook in reverse order on the same thread, with {@link UserFunctionOutcome#FAILED} and the original fatal error. + * The user function does not run in that case. Cleanup failures do not replace the original start-hook error. + * *

    It fires for every outcome of the user function: normal return, a thrown failure, and suspension. Check * {@link UserFunctionEndInfo#outcome()} to distinguish them. A suspended function reports * {@link UserFunctionOutcome#INCOMPLETE}; its {@link UserFunctionEndInfo#error()} is the SDK's internal diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPluginFactory.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPluginFactory.java new file mode 100644 index 000000000..0f361bc5e --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPluginFactory.java @@ -0,0 +1,41 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.plugin; + +/** + * Creates one {@link DurableExecutionPlugin} instance per Lambda invocation. + * + *

    The SDK builds the {@link InvocationInfo} for an invocation, calls this factory with it, dispatches that + * invocation's hooks to the returned instance, and drops the instance when the invocation returns. A plugin instance + * therefore serves exactly one invocation and can hold per-invocation state in plain fields — no keying by execution + * ARN is needed, even when the execution environment runs several executions concurrently. + * + *

    The {@link InvocationInfo} handed to the factory is the same instance the plugin's + * {@link DurableExecutionPlugin#onInvocationStart(InvocationInfo)} hook then receives. + * + *

    Non-fatal factory failures are contained like non-fatal hook failures: the factory is logged and skipped for that + * invocation. A {@code null} result is also logged and skipped. {@link VirtualMachineError} and {@link ThreadDeath} + * propagate, including when wrapped by asynchronous completion/future exceptions. + * + * 

    {@code
+ * DurableConfig.builder()
+ *     .withPlugins(info -> new MyPlugin(info.durableExecutionArn()))
+ *     .build();
+ * }
    + */ +@FunctionalInterface +public interface DurableExecutionPluginFactory { + + /** + * Creates the plugin instance that serves the described invocation. + * + * @param invocationInfo the invocation the plugin instance will observe, including its immutable + * {@link InvocationInfo#xRayTraceId() runtime header snapshot} before any plugin spans are created + * @return the plugin instance for this invocation + */ + DurableExecutionPlugin createPlugin(InvocationInfo invocationInfo); + /** Optional exclusive instrumentation group, validated before any invocation instance is created. */ + default String getExclusiveGroup() { + return null; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPluginProvider.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPluginProvider.java index c527c2b38..e1d3d51e0 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPluginProvider.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/DurableExecutionPluginProvider.java @@ -3,16 +3,17 @@ package software.amazon.lambda.durable.plugin; /** - * Service provider interface for dynamically loading {@link DurableExecutionPlugin} implementations. + * A {@link DurableExecutionPluginFactory} that can be discovered through {@link java.util.ServiceLoader} and selected + * by name. * *

    Provider JARs register implementations in - * {@code META-INF/services/software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider}. The SDK only creates - * plugins from providers explicitly selected through {@code DURABLE_EXECUTION_PLUGINS}. + * {@code META-INF/services/software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider}. The SDK only uses + * providers explicitly selected through {@code DURABLE_EXECUTION_PLUGINS}; selection is by {@link #getName()}. + * + *

    A provider is itself the per-invocation factory: {@link #createPlugin(InvocationInfo)} is called once per + * invocation, and the returned instance serves only that invocation. */ -public interface DurableExecutionPluginProvider { - - /** Current version of the dynamic plugin provider contract. */ - int API_VERSION = 1; +public interface DurableExecutionPluginProvider extends DurableExecutionPluginFactory { /** * Returns the stable name used to select this provider. @@ -20,25 +21,4 @@ public interface DurableExecutionPluginProvider { * @return non-empty provider name */ String getName(); - - /** - * Returns the provider API version this implementation supports. - * - * @return provider API version - */ - int getApiVersion(); - - /** - * Returns the concrete plugin type created by this provider. - * - * @return plugin implementation class - */ - Class getPluginType(); - - /** - * Creates the plugin instance. - * - * @return plugin instance - */ - DurableExecutionPlugin createPlugin(); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/HandlerScoped.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/HandlerScoped.java new file mode 100644 index 000000000..e4ba959ca --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/HandlerScoped.java @@ -0,0 +1,30 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.plugin; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.function.Function; + +/** + * Explicitly opts a plugin into a root-handler scope using a JDK {@link Function} opener with a public no-argument + * constructor. The core passes the plugin instance to that opener on the handler thread; null means no scope. No method + * name is discovered on the plugin or added to the existing lifecycle interface. Inherited metadata uses the declaring + * plugin's explicit opener, so unrelated methods on old subclasses retain their behavior. Older cores ignore this + * optional metadata; plugin layer signatures require no new shared SDK type. + * + *

    Scopes open and close on the handler thread, in reverse order on exit. Suspension/termination gives cleanup a + * bounded opportunity to unwind. The handler retains ownership even after a timeout. Ordinary cleanup preserves the + * winning outcome; observed VirtualMachineError/ThreadDeath from scope callbacks escapes the invocation caller. + * Optional waiting reserves five seconds per configured plugin plus one second for shutdown/response. This is best + * effort: existing finalizers and checkpoint draining can exceed it. + */ +@Inherited +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface HandlerScoped { + Class> value(); +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/InvocationInfo.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/InvocationInfo.java index d6c67195d..c72e95109 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/plugin/InvocationInfo.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/InvocationInfo.java @@ -11,6 +11,9 @@ /** * Invocation-level information available to plugin hooks. * + *

    The nullable runtime header is the eighth record component. Legacy constructors remain available, but source + * record patterns must include the new component; reflection and record value semantics observe the new field. + * * @param requestId the Lambda request ID for this invocation * @param durableExecutionArn the durable execution ARN * @param isFirstInvocation true if this is the first invocation of the execution (not a replay invocation) @@ -21,6 +24,8 @@ * experimental * @param updatedOperations operations changed externally since the previous invocation, keyed by operation ID; this * component is experimental + * @param xRayTraceId immutable invocation-local X-Ray header; null means unavailable and permits legacy carrier + * fallback, while an empty string means an available runtime supplied no header */ public record InvocationInfo( String requestId, @@ -29,7 +34,8 @@ public record InvocationInfo( Instant executionStartTime, @Experimental Object executionInput, @Experimental Map operations, - @Experimental Map updatedOperations) { + @Experimental Map updatedOperations, + String xRayTraceId) { public InvocationInfo { requireNonNull(executionStartTime, "executionStartTime"); @@ -37,6 +43,26 @@ public record InvocationInfo( requireNonNull(updatedOperations, "updatedOperations"); } + /** Retains the original seven-argument constructor for callers without a runtime header snapshot. */ + public InvocationInfo( + String requestId, + String durableExecutionArn, + boolean isFirstInvocation, + Instant executionStartTime, + Object executionInput, + Map operations, + Map updatedOperations) { + this( + requestId, + durableExecutionArn, + isFirstInvocation, + executionStartTime, + executionInput, + operations, + updatedOperations, + null); + } + /** Creates invocation information without payload or operation snapshots. */ public InvocationInfo( String requestId, String durableExecutionArn, boolean isFirstInvocation, Instant executionStartTime) { @@ -71,7 +97,7 @@ public InvocationInfo( updatedOperations); } - /** Returns a representation that omits execution payloads and operation snapshots. */ + /** Returns a representation that omits execution payloads, operation snapshots, and the runtime header. */ @Override public String toString() { return "InvocationInfo[requestId=" + requestId + ", durableExecutionArn=" + durableExecutionArn diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java index e3a5707c4..94b42e528 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java @@ -2,16 +2,34 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.plugin; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayDeque; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.lambda.durable.util.ExceptionHelper; /** - * Composes multiple {@link DurableExecutionPlugin} instances into a single dispatcher. + * Dispatches the lifecycle hooks of a single Lambda invocation to that invocation's plugin instances. * - *

    Event hooks are fire-and-forget: each plugin is called in order, errors are swallowed. + *

    A runner is created per invocation from the configured {@link DurableExecutionPluginFactory factories} and holds + * no plugin instances until {@link #onInvocationStart(InvocationInfo)} materializes and starts each plugin in + * configured order, from the very {@link InvocationInfo} the first hook then receives. {@link #releasePlugins()} drops + * them when the invocation returns, so a plugin instance is never shared between invocations and never needs to key its + * state by execution ARN. + * + *

    Event hooks are fire-and-forget: each plugin is called in order, non-fatal failures are contained. A factory with + * a non-fatal failure or returns {@code null} is contained the same way — the plugin is skipped for the invocation. + * Containment covers every non-fatal throwable, not only {@link Exception}, because a plugin built against a different + * SDK version, one missing an optional dependency, and one running with assertions enabled all fail with an + * {@code Error}. It stops short of two cases, which keep propagating: the errors that report the JVM itself failing, + * and the {@code ThreadDeath} that reports the thread running the plugin has already been terminated. * *

    {@code onInvocationEnd} is awaited (the SDK blocks until it returns) to allow plugins to flush data before Lambda * freezes. @@ -19,71 +37,351 @@ public class PluginRunner { private static final Logger logger = LoggerFactory.getLogger(PluginRunner.class); - private static final PluginRunner NO_OP = new PluginRunner(Collections.emptyList()); - private final List plugins; + private final List pluginFactories; + private final Consumer operationFatalObserver; - public PluginRunner(List plugins) { - this.plugins = plugins != null ? List.copyOf(plugins) : Collections.emptyList(); + /** + * This invocation's plugin instances. Written once on the thread that fires {@code onInvocationStart}, read from + * the user, checkpoint, and operation threads that fire the later hooks — volatile for that publication. + */ + private volatile List plugins = List.of(); + + public PluginRunner(List pluginFactories) { + this(pluginFactories, fatal -> {}); + } + + /** Reports fatal operation and invocation-end hooks to their invocation before rethrowing. */ + public PluginRunner(List pluginFactories, Consumer operationFatalObserver) { + this.operationFatalObserver = operationFatalObserver; + this.pluginFactories = pluginFactories != null ? List.copyOf(pluginFactories) : Collections.emptyList(); + validateExclusiveGroups(this.pluginFactories); + } + + /** Validates configured factory metadata without creating invocation-owned plugins or spans. */ + public static void validateExclusiveGroups(List factories) { + var groups = new HashMap(); + for (var factory : factories) { + var group = factory.getExclusiveGroup(); + if (group == null) continue; + var previous = groups.putIfAbsent(group, factory); + if (previous != null) + throw new IllegalArgumentException("Conflicting plugin factories " + previous + " and " + factory + + " in exclusive group '" + group + "'. Configure only one plugin from this group."); + } } - /** Returns a no-op runner that does nothing. */ + /** Returns a runner with no plugin factories, which does nothing. */ public static PluginRunner noOp() { - return NO_OP; + return new PluginRunner(Collections.emptyList()); } - /** Returns true if no plugins are registered. */ + /** Returns true if no plugin factories are registered. */ public boolean isEmpty() { - return plugins.isEmpty(); + return pluginFactories.isEmpty(); } - /** Returns the list of registered plugins. */ - public List getPlugins() { - return plugins; + // ─── Per-invocation lifetime ───────────────────────────────────────── + + /** + * Creates this invocation's plugin instances, one per registered factory. + * + *

    Called from {@link #onInvocationStart(InvocationInfo)}. Each start hook runs before constructing the next + * plugin, preserving startup context installed by earlier registrations. Factories that fail non-fatally or return + * null are logged and skipped. Fatal causes propagate, including through completion/future wrappers. + * + *

    Every non-fatal throwable is contained, not just {@link Exception}. The contract says a non-fatal factory + * failure is skipped and never disrupts the execution, and a throwable that escapes here fails an execution the + * plugin was only observing. Narrowing the catch to a list of types would leave that promise conditional on the + * list being complete, and it was not: a provider JAR compiled against an earlier version of + * {@link DurableExecutionPluginFactory} throws {@link AbstractMethodError}, a provider whose optional dependency is + * missing from the deployment package throws {@link NoClassDefFoundError}, a provider running with assertions + * enabled throws {@link AssertionError}, and a provider that loads its own exporter back ends through + * {@link java.util.ServiceLoader} throws {@link java.util.ServiceConfigurationError}. Only the first two are + * {@link LinkageError} and none is an {@link Exception}. Catching {@code Throwable} and rethrowing only the fatal + * cases makes the promise unconditional. See {@link #contain} for which cases stay fatal. + */ + private void createPlugins(InvocationInfo info) { + var created = new ArrayList(pluginFactories.size()); + try { + for (var factory : pluginFactories) { + var plugin = createPlugin(factory, info); + if (plugin == null) continue; + created.add(plugin); + runPlugin(plugin, p -> p.onInvocationStart(info)); + } + } finally { + // Even a fatal constructor/start failure must leave already-created instances available for finalization. + // Publishing after startup also makes state assigned by start hooks visible to later SDK threads. + this.plugins = List.copyOf(created); + } + } + + private static DurableExecutionPlugin createPlugin(DurableExecutionPluginFactory factory, InvocationInfo info) { + try { + var plugin = factory.createPlugin(info); + if (plugin == null) + logger.warn("Plugin factory {} returned null; skipping it for this invocation", factory); + return plugin; + } catch (Throwable failure) { + contain(failure, "Plugin factory failed; skipping it for this invocation"); + return null; + } + } + + /** + * Drops this invocation's plugin instances. Called when the invocation returns so the instances are unreachable + * from the SDK and cannot leak into the next invocation the environment hosts. + * + *

    No containment here: this only replaces the field, and calls nothing on the plugins it drops. There is no + * {@code close()} in the plugin contract, so releasing cannot run plugin code and cannot fail. + */ + public void releasePlugins() { + this.plugins = List.of(); } // ─── Event hooks ───────────────────────────────────────────────────── - /** Calls a void hook on all plugins, swallowing any errors. */ + /** + * Calls a void hook on all of this invocation's plugins, swallowing any non-fatal throwable. + * + *

    Containment here follows the same rule as {@link #createPlugins}, because the fire-and-forget contract makes + * no distinction between the two boundaries. A plugin fails a hook with the same shapes a factory fails with, and + * one plugin's failure must not stop the remaining plugins from receiving the hook or fail the execution. See + * {@link #contain} for which cases stay fatal. + */ private void run(Consumer hook) { for (var plugin : plugins) { + runPlugin(plugin, hook); + } + } + + private static void runPlugin(DurableExecutionPlugin plugin, Consumer hook) { + try { + hook.accept(plugin); + } catch (Throwable failure) { + contain(failure, "Plugin hook failed"); + } + } + + /** + * Logs a throwable that plugin code produced, or rethrows it if it is fatal. + * + *

    A {@link VirtualMachineError} is the JVM reporting that it can no longer run correctly, which covers + * {@link OutOfMemoryError}, {@link StackOverflowError}, {@link InternalError} and {@link UnknownError}. That is not + * a plugin defect, and the process cannot be assumed able to continue past it. Logging it as a contained plugin + * failure would therefore hide a condition the caller has to see, so it is rethrown unchanged. The rule names the + * supertype rather than the four subclasses so that a subclass added later is fatal without an edit here. + * + *

    {@code ThreadDeath} is fatal for a different reason. It is not a report of a failure but a thread termination + * that has already begun: {@code Thread.stop()} delivers it by throwing it into the target thread, which unwinds + * that thread's stack from wherever it stood and releases the monitors it held over state it had only half updated. + * The threads that create plugins and fire hooks are SDK threads that carry SDK and user work after the plugin + * returns. Containing the {@code ThreadDeath} would therefore return one of those threads to that work with its + * invariants already broken and the termination it was sent silently dropped. It is rethrown unchanged so the + * termination completes. + * + *

    {@code Thread.stop()} throws {@link UnsupportedOperationException} on JDK 20 and later, so the JVM cannot + * deliver a {@code ThreadDeath} on those runtimes. It can deliver one on JDK 17, and {@code maven.compiler.source} + * is 17, so the rethrow is reachable on a runtime this SDK supports. A {@code ThreadDeath} that plugin code + * constructs and throws itself is rethrown on every runtime; the boundary cannot distinguish it from a delivered + * one, and treating the ambiguous case as fatal is the safe direction. + * + *

    {@code ThreadDeath} is deprecated for removal since JDK 20, so naming it emits a removal warning when this + * class is compiled on a JDK 20 or later compiler. The {@code @SuppressWarnings("removal")} below is scoped to this + * method rather than the class so it cannot mask a removal warning that appears elsewhere in {@code PluginRunner}. + * + *

    An {@link InterruptedException} is contained like any other non-fatal throwable, and the interrupt status is + * not restored. Three facts decide it. The thread that creates plugins and fires {@code onInvocationStart} is the + * handler thread — the hook runs there on purpose, so a plugin can set a {@code ThreadLocal} or an MDC key the + * handler's own logging then reads — so setting the flag there leaves the handler's next blocking call to fail with + * an {@code InterruptedException} that no user code asked for, which is the containment contract broken by the + * boundary meant to enforce it. A thrown {@code InterruptedException} is also no proof that the thread was + * interrupted: no hook and no factory method declares a checked exception, so the only way one arrives is plugin + * code rethrowing it undeclared, and plugin code can construct one and throw it with the interrupt status clear. + * And no SDK code interrupts these threads or reads their interrupt status, so restoring the flag serves no waiting + * reader. The interrupt is reported the way every other contained plugin failure is, as a logged warning naming the + * plugin boundary that produced it. + */ + @SuppressWarnings("removal") // ThreadDeath is deprecated for removal since JDK 20; see the javadoc above. + private static void contain(Throwable t, String message) { + var cause = ExceptionHelper.unwrapAsyncFailure(t); + if (cause instanceof VirtualMachineError fatal) { + throw fatal; + } + if (cause instanceof ThreadDeath fatal) { + throw fatal; + } + logger.warn(message, t); + } + + /** + * Called at the start of each invocation. Materializes this invocation's plugin instances from the registered + * factories, then dispatches the hook to them with the same {@link InvocationInfo} the factories received. + */ + /** Runs the root handler with optional plugin scopes, closing them on the same thread in reverse order. */ + public T runHandler(Supplier handler) { + return runHandler(handler, () -> {}); + } + + /** Runs the handler and notifies the invocation when a scope requires same-thread finalization. */ + public T runHandler(Supplier handler, Runnable onScopeOpened) { + return runHandler(handler, onScopeOpened, fatal -> {}); + } + + /** Reports fatal errors originating only in scope callbacks, before rethrowing on the owner thread. */ + @SuppressWarnings("removal") + public T runHandler(Supplier handler, Runnable onScopeOpened, Consumer onScopeFatal) { + var scopes = new ArrayDeque(); + try { + for (var plugin : plugins) { + try { + var scope = openHandlerScope(plugin); + if (scope != null) { + scopes.push(scope); + onScopeOpened.run(); + } + } catch (Throwable e) { + reportHandlerScopeFailure("Plugin handler scope threw exception", e, onScopeFatal); + } + } + return handler.get(); + } finally { + closeHandlerScopes(scopes, onScopeFatal); + } + } + + @SuppressWarnings("unchecked") + private static AutoCloseable openHandlerScope(DurableExecutionPlugin plugin) throws ReflectiveOperationException { + var metadata = plugin.getClass().getAnnotation(HandlerScoped.class); + if (metadata == null) return null; + var constructor = metadata.value().getConstructor(); + if (!constructor.canAccess(null) && !constructor.trySetAccessible()) { + throw new IllegalAccessException( + "Cannot access @HandlerScoped opener " + metadata.value().getName()); + } + try { + var opener = (Function) constructor.newInstance(); + return opener.apply(plugin); + } catch (InvocationTargetException failure) { + ExceptionHelper.sneakyThrow(failure.getCause()); + return null; + } + } + + @SuppressWarnings("removal") + private static void closeHandlerScopes(ArrayDeque scopes, Consumer onScopeFatal) { + Error firstFatal = null; + while (!scopes.isEmpty()) { try { - hook.accept(plugin); - } catch (Exception e) { - logger.warn("Plugin hook threw exception", e); + scopes.pop().close(); + } catch (Throwable e) { + try { + reportHandlerScopeFailure("Plugin handler scope cleanup threw exception", e, onScopeFatal); + } catch (VirtualMachineError | ThreadDeath fatal) { + if (firstFatal == null) firstFatal = fatal; + } } } + if (firstFatal != null) throw firstFatal; + } + + @SuppressWarnings("removal") + private static void reportHandlerScopeFailure(String message, Throwable failure, Consumer onScopeFatal) { + var cause = ExceptionHelper.unwrapAsyncFailure(failure); + if (cause instanceof VirtualMachineError || cause instanceof ThreadDeath) { + var fatal = (Error) cause; + onScopeFatal.accept(fatal); + throw fatal; + } + logger.warn(message, failure); } public void onInvocationStart(InvocationInfo info) { - run(p -> p.onInvocationStart(info)); + createPlugins(info); } /** * Called at the end of each invocation. Awaited — the SDK blocks until all plugins return, allowing plugins to - * flush spans/metrics before Lambda freezes. + * flush spans/metrics before Lambda freezes. The first fatal immediately stops outstanding operation work; an + * earlier invocation fatal retains precedence. Remaining plugins still receive their one finalization opportunity + * with the shared snapshot before the first end-hook fatal is rethrown. */ + @SuppressWarnings("removal") public void onInvocationEnd(InvocationEndInfo info) { - run(p -> p.onInvocationEnd(info)); + Error firstFatal = null; + for (var plugin : plugins) { + try { + runPlugin(plugin, p -> p.onInvocationEnd(info)); + } catch (VirtualMachineError | ThreadDeath fatal) { + if (firstFatal == null) { + firstFatal = fatal; + // Stop queued work before a later exporter can block, but still finalize every plugin below. + var primary = ExceptionHelper.unwrapAsyncFailure(info.executionError()); + operationFatalObserver.accept( + primary instanceof VirtualMachineError || primary instanceof ThreadDeath + ? (Error) primary + : firstFatal); + } + } + } + if (firstFatal != null) throw firstFatal; + } + + @SuppressWarnings("removal") + private void runOperationHook(Consumer hook) { + try { + run(hook); + } catch (VirtualMachineError | ThreadDeath fatal) { + operationFatalObserver.accept(fatal); + throw fatal; + } } public void onOperationStart(OperationInfo info) { - run(p -> p.onOperationStart(info)); + runOperationHook(p -> p.onOperationStart(info)); } public void onOperationEnd(OperationEndInfo info) { - run(p -> p.onOperationEnd(info)); + runOperationHook(p -> p.onOperationEnd(info)); } public void onOperationChange(OperationChangeInfo info) { - run(p -> p.onOperationChange(info)); + runOperationHook(p -> p.onOperationChange(info)); } + @SuppressWarnings("removal") public void onUserFunctionStart(UserFunctionStartInfo info) { - run(p -> p.onUserFunctionStart(info)); + var started = new ArrayDeque(); + try { + run(plugin -> { + plugin.onUserFunctionStart(info); + started.push(plugin); + }); + } catch (VirtualMachineError | ThreadDeath fatal) { + // Unwind attempt scopes on their owner before publishing the fatal: publication can start invocation + // finalization on another thread, where it is too late to restore these thread-local scopes safely. + closeStartedUserFunctions(started, info, fatal); + operationFatalObserver.accept(fatal); + throw fatal; + } + } + + @SuppressWarnings("removal") + private static void closeStartedUserFunctions( + ArrayDeque started, UserFunctionStartInfo info, Error fatal) { + var endInfo = PluginInfoConverter.toUserFunctionEndInfo(info, UserFunctionOutcome.FAILED, fatal); + while (!started.isEmpty()) { + try { + runPlugin(started.pop(), plugin -> plugin.onUserFunctionEnd(endInfo)); + } catch (VirtualMachineError | ThreadDeath cleanupFailure) { + // Each earlier start gets its cleanup opportunity; preserve the original start-hook failure. + if (cleanupFailure != fatal) fatal.addSuppressed(cleanupFailure); + } + } } public void onUserFunctionEnd(UserFunctionEndInfo info) { - run(p -> p.onUserFunctionEnd(info)); + runOperationHook(p -> p.onUserFunctionEnd(info)); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/util/ExceptionHelper.java b/sdk/src/main/java/software/amazon/lambda/durable/util/ExceptionHelper.java index b6d5a0e26..87d7dc4ca 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/util/ExceptionHelper.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/util/ExceptionHelper.java @@ -5,6 +5,7 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; @@ -38,6 +39,22 @@ public static Throwable unwrapCompletableFuture(Throwable throwable) { return throwable; } + /** + * Unwraps completion/future wrappers to inspect potentially fatal causes. Callers must retain ordinary application + * {@link ExecutionException} values when reporting or serializing failures. Non-wrapper failures and cause-less + * wrappers keep their identity. + * + * @param failure the failure to inspect, possibly null + * @return the underlying asynchronous failure, or the original non-wrapper failure + */ + public static Throwable unwrapAsyncFailure(Throwable failure) { + while ((failure instanceof CompletionException || failure instanceof ExecutionException) + && failure.getCause() != null) { + failure = failure.getCause(); + } + return failure; + } + /** * build an ErrorObject from a Throwable * diff --git a/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java b/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java index 266e43a6e..a79b8c515 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java @@ -13,8 +13,10 @@ import static org.mockito.Mockito.mock; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutorService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -23,6 +25,9 @@ import software.amazon.lambda.durable.client.DurableExecutionClient; import software.amazon.lambda.durable.client.LambdaDurableFunctionsClient; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.PluginRunner; import software.amazon.lambda.durable.retry.JitterStrategy; import software.amazon.lambda.durable.retry.PollingStrategies; import software.amazon.lambda.durable.serde.JacksonSerDes; @@ -507,47 +512,41 @@ private static void setField(Object target, String fieldName, Object value) thro // --- Plugin registration tests --- @Test - void testDefaultConfig_PluginRunnerIsNoOp() { + void testDefaultConfig_NoPluginFactories() { var config = DurableConfig.defaultConfig(); - assertNotNull(config.getPluginRunner()); - assertTrue(config.getPluginRunner().isEmpty()); + assertNotNull(config.getPluginFactories()); + assertTrue(config.getPluginFactories().isEmpty()); } @Test - void testBuilder_NoPlugins_PluginRunnerIsNoOp() { + void testBuilder_NoPlugins_NoPluginFactories() { var config = DurableConfig.builder().withDurableExecutionClient(mockClient).build(); - assertNotNull(config.getPluginRunner()); - assertTrue(config.getPluginRunner().isEmpty()); + assertNotNull(config.getPluginFactories()); + assertTrue(config.getPluginFactories().isEmpty()); } @Test - void testBuilder_WithPlugin_CreatesActivePluginRunner() { - var plugin = new DurableExecutionPlugin() {}; + void testBuilder_WithPlugin_RegistersFactory() { var config = DurableConfig.builder() .withDurableExecutionClient(mockClient) - .withPlugins(plugin) + .withPlugins(info -> new DurableExecutionPlugin() {}) .build(); - assertNotNull(config.getPluginRunner()); - assertFalse(config.getPluginRunner().isEmpty()); + assertEquals(1, config.getPluginFactories().size()); } @Test - void testBuilder_WithMultiplePlugins_AllRegistered() { + void testBuilder_WithMultiplePlugins_AllRegisteredInOrder() { var calls = new ArrayList(); - var plugin1 = new TestPlugin("p1", calls); - var plugin2 = new TestPlugin("p2", calls); var config = DurableConfig.builder() .withDurableExecutionClient(mockClient) - .withPlugins(plugin1, plugin2) + .withPlugins(info -> new TestPlugin("p1", calls), info -> new TestPlugin("p2", calls)) .build(); - config.getPluginRunner() - .onInvocationStart(new software.amazon.lambda.durable.plugin.InvocationInfo( - "req-1", "arn:test", true, java.time.Instant.now(), java.util.Map.of(), java.util.Map.of())); + new PluginRunner(config.getPluginFactories()).onInvocationStart(invocationInfo()); assertEquals(List.of("p1:onInvocationStart", "p2:onInvocationStart"), calls); } @@ -555,19 +554,14 @@ void testBuilder_WithMultiplePlugins_AllRegistered() { @Test void testBuilder_WithPlugins_CalledMultipleTimes_Replaces() { var calls = new ArrayList(); - var plugin1 = new TestPlugin("p1", calls); - var plugin2 = new TestPlugin("p2", calls); - var plugin3 = new TestPlugin("p3", calls); var config = DurableConfig.builder() .withDurableExecutionClient(mockClient) - .withPlugins(plugin1) - .withPlugins(plugin2, plugin3) + .withPlugins(info -> new TestPlugin("p1", calls)) + .withPlugins(info -> new TestPlugin("p2", calls), info -> new TestPlugin("p3", calls)) .build(); - config.getPluginRunner() - .onInvocationStart(new software.amazon.lambda.durable.plugin.InvocationInfo( - "req-1", "arn:test", true, java.time.Instant.now(), java.util.Map.of(), java.util.Map.of())); + new PluginRunner(config.getPluginFactories()).onInvocationStart(invocationInfo()); assertEquals(List.of("p2:onInvocationStart", "p3:onInvocationStart"), calls); } @@ -576,7 +570,8 @@ void testBuilder_WithPlugins_CalledMultipleTimes_Replaces() { void testBuilder_WithPlugins_NullArrayThrows() { var builder = DurableConfig.builder(); - var ex = assertThrows(NullPointerException.class, () -> builder.withPlugins((DurableExecutionPlugin[]) null)); + var ex = assertThrows( + NullPointerException.class, () -> builder.withPlugins((DurableExecutionPluginFactory[]) null)); assertEquals("Plugins array cannot be null", ex.getMessage()); } @@ -585,16 +580,19 @@ void testBuilder_WithPlugins_NullElementThrows() { var builder = DurableConfig.builder(); var ex = assertThrows( - NullPointerException.class, () -> builder.withPlugins(new DurableExecutionPlugin[] {null})); + NullPointerException.class, () -> builder.withPlugins(new DurableExecutionPluginFactory[] {null})); assertEquals("Plugin cannot be null", ex.getMessage()); } @Test void testBuilder_WithPlugins_FluentAPI() { var builder = DurableConfig.builder(); - var plugin = new DurableExecutionPlugin() {}; - assertSame(builder, builder.withPlugins(plugin)); + assertSame(builder, builder.withPlugins(info -> new DurableExecutionPlugin() {})); + } + + private static InvocationInfo invocationInfo() { + return new InvocationInfo("req-1", "arn:test", true, Instant.now(), Map.of(), Map.of()); } /** Simple test plugin that records hook calls. */ @@ -608,7 +606,7 @@ private static class TestPlugin implements DurableExecutionPlugin { } @Override - public void onInvocationStart(software.amazon.lambda.durable.plugin.InvocationInfo info) { + public void onInvocationStart(InvocationInfo info) { calls.add(name + ":onInvocationStart"); } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/DynamicPluginLoaderStaleProviderTest.java b/sdk/src/test/java/software/amazon/lambda/durable/DynamicPluginLoaderStaleProviderTest.java new file mode 100644 index 000000000..87f70a798 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/DynamicPluginLoaderStaleProviderTest.java @@ -0,0 +1,561 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.lang.reflect.Modifier; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.CodeSigner; +import java.security.CodeSource; +import java.security.ProtectionDomain; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider; +import software.amazon.lambda.durable.plugin.InvocationInfo; + +/** + * Covers the startup check that rejects a plugin provider compiled against the older provider interface. + * + *

    The condition being checked is a property of a class file, not of source: the provider's class file declares that + * it implements {@code DurableExecutionPluginProvider} but contains no {@code createPlugin(InvocationInfo)} method. + * That class file cannot be produced from this source tree, because this source tree contains only the current + * interface and a class that fails to implement one of its abstract methods does not compile. Each test here therefore + * compiles a provider against a stub of the older interface and then loads the result against the current interface, + * which is the deployment it stands in for: a provider JAR built against an earlier SDK and left in place while the + * function's SDK dependency was raised. + * + *

    The fixture reproduces the condition rather than approximating it, so these tests establish that the check fires + * on a class file with the shape a stale provider JAR has. A cheaper fixture would not: a + * {@link java.lang.reflect.Proxy} over the provider interface generates a concrete + * {@code createPlugin(InvocationInfo)}, so it does not reproduce the condition at all. + * + *

    What these tests do not establish is that a provider JAR built by some other toolchain against some other 2.x + * point release produces exactly this class file shape. They cover one stale shape, the one the migration guide + * describes. + */ +class DynamicPluginLoaderStaleProviderTest { + + private static final String PROVIDER_CLASS = "com.example.audit.StaleAuditProvider"; + private static final String PROVIDER_NAME = "com.example.audit"; + + /** The plugin interface, which is unchanged, so the stale provider's references to it still resolve. */ + private static final String PLUGIN_SOURCE = """ + package software.amazon.lambda.durable.plugin; + + public interface DurableExecutionPlugin {} + """; + + /** The provider interface as an earlier SDK declared it, against which the fixture provider is compiled. */ + private static final String OLD_PROVIDER_INTERFACE_SOURCE = """ + package software.amazon.lambda.durable.plugin; + + public interface DurableExecutionPluginProvider { + + int API_VERSION = 1; + + String getName(); + + int getApiVersion(); + + Class getPluginType(); + + DurableExecutionPlugin createPlugin(); + } + """; + + /** + * The invocation info as a stub, so a fixture can name it in a signature. + * + *

    Not handed to the loader, so a compiled reference to it resolves to this SDK's class at load time and the + * fixture's method descriptor matches the one {@code getMethod} is asked for. + */ + private static final String INVOCATION_INFO_SOURCE = """ + package software.amazon.lambda.durable.plugin; + + public final class InvocationInfo {} + """; + + /** A provider whose only createPlugin(InvocationInfo) is static, which cannot implement an instance method. */ + private static final String STATIC_PROVIDER_SOURCE = """ + package com.example.audit; + + import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; + import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider; + import software.amazon.lambda.durable.plugin.InvocationInfo; + + public final class StaleAuditProvider implements DurableExecutionPluginProvider { + + @Override + public String getName() { + return "com.example.audit"; + } + + @Override + public int getApiVersion() { + return API_VERSION; + } + + @Override + public Class getPluginType() { + return StaleAuditPlugin.class; + } + + @Override + public DurableExecutionPlugin createPlugin() { + return new StaleAuditPlugin(); + } + + public static DurableExecutionPlugin createPlugin(InvocationInfo info) { + return new StaleAuditPlugin(); + } + + public static final class StaleAuditPlugin implements DurableExecutionPlugin {} + } + """; + + /** + * A provider whose createPlugin(InvocationInfo) returns a plugin subtype and overrides nothing. + * + *

    Compiled against the older interface, so the method overrides no abstract declaration and javac emits no + * bridge returning {@code DurableExecutionPlugin}. The return type is still a plugin, so a check that asked only + * whether the return type were assignable to {@code DurableExecutionPlugin} would accept it, while + * {@code invokeinterface} looks for the interface's erased descriptor and finds none. + */ + private static final String SUBTYPE_RETURN_PROVIDER_SOURCE = """ + package com.example.audit; + + import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; + import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider; + import software.amazon.lambda.durable.plugin.InvocationInfo; + + public final class StaleAuditProvider implements DurableExecutionPluginProvider { + + @Override + public String getName() { + return "com.example.audit"; + } + + @Override + public int getApiVersion() { + return API_VERSION; + } + + @Override + public Class getPluginType() { + return StaleAuditPlugin.class; + } + + @Override + public DurableExecutionPlugin createPlugin() { + return new StaleAuditPlugin(); + } + + public StaleAuditPlugin createPlugin(InvocationInfo info) { + return new StaleAuditPlugin(); + } + + public static final class StaleAuditPlugin implements DurableExecutionPlugin {} + } + """; + + /** A provider whose createPlugin(InvocationInfo) returns something that is not a plugin. */ + private static final String WRONG_RETURN_PROVIDER_SOURCE = """ + package com.example.audit; + + import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; + import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider; + import software.amazon.lambda.durable.plugin.InvocationInfo; + + public final class StaleAuditProvider implements DurableExecutionPluginProvider { + + @Override + public String getName() { + return "com.example.audit"; + } + + @Override + public int getApiVersion() { + return API_VERSION; + } + + @Override + public Class getPluginType() { + return StaleAuditPlugin.class; + } + + @Override + public DurableExecutionPlugin createPlugin() { + return new StaleAuditPlugin(); + } + + public String createPlugin(InvocationInfo info) { + return "not a plugin"; + } + + public static final class StaleAuditPlugin implements DurableExecutionPlugin {} + } + """; + + /** A provider written against the interface above, exactly as the migration guide's "before" example is. */ + private static final String PROVIDER_SOURCE = """ + package com.example.audit; + + import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; + import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider; + + public final class StaleAuditProvider implements DurableExecutionPluginProvider { + + @Override + public String getName() { + return "com.example.audit"; + } + + @Override + public int getApiVersion() { + return API_VERSION; + } + + @Override + public Class getPluginType() { + return StaleAuditPlugin.class; + } + + @Override + public DurableExecutionPlugin createPlugin() { + return new StaleAuditPlugin(); + } + + public static final class StaleAuditPlugin implements DurableExecutionPlugin {} + } + """; + + @Test + void staleProviderIsIndistinguishableFromACurrentOneUntilCreatePluginIsResolved(@TempDir Path workDir) + throws Exception { + var provider = staleProvider(workDir); + + // Nothing the provider's class file references was removed, so it loads, instantiates, and reports its name. + assertEquals(PROVIDER_NAME, provider.getName()); + + // Its createPlugin(InvocationInfo) resolves to the abstract declaration on the interface it inherits. + var createPlugin = provider.getClass().getMethod("createPlugin", InvocationInfo.class); + assertTrue(Modifier.isAbstract(createPlugin.getModifiers())); + assertTrue(createPlugin.getDeclaringClass().isInterface()); + + // Calling it fails, which is the outcome the startup check exists to reach first. + assertThrows(AbstractMethodError.class, () -> provider.createPlugin(invocationInfo())); + } + + @Test + void rejectsSelectedStaleProviderAtConfigurationTime(@TempDir Path workDir) throws Exception { + var provider = staleProvider(workDir); + + var error = assertThrows( + IllegalStateException.class, + () -> DynamicPluginLoader.loadConfiguredPluginFactories(PROVIDER_NAME, List.of(provider), List.of())); + + var message = error.getMessage(); + assertTrue(message.contains("Dynamic plugin configuration failed"), message); + assertTrue(message.contains("Plugin provider '" + PROVIDER_NAME + "'"), message); + assertTrue(message.contains(PROVIDER_CLASS), message); + assertTrue(message.contains("does not implement createPlugin(InvocationInfo)"), message); + assertTrue(message.contains("compiled against an older Durable Execution SDK"), message); + assertTrue(message.contains("Rebuild the provider against this SDK version and redeploy it"), message); + assertTrue(message.contains("Lambda layer is versioned and deployed separately"), message); + + // The artifact to rebuild is named, because an operator with several provider layers deployed needs to know + // which one is stale. + var artifactLocation = + provider.getClass().getProtectionDomain().getCodeSource().getLocation(); + assertTrue(message.contains(artifactLocation.toString()), message); + } + + @Test + void rejectsAProviderWhoseCreatePluginIsStatic(@TempDir Path workDir) throws Exception { + // getMethod searches the class before the interfaces it implements and returns static methods, so a stale class + // carrying a static createPlugin(InvocationInfo) helper resolves to that helper. The instance method the + // interface call dispatches to is still missing, so the absence of the abstract modifier proves nothing here. + var provider = staleProviderOfShape(workDir, STATIC_PROVIDER_SOURCE); + + var createPlugin = provider.getClass().getMethod("createPlugin", InvocationInfo.class); + assertTrue(Modifier.isStatic(createPlugin.getModifiers())); + assertTrue(!Modifier.isAbstract(createPlugin.getModifiers())); + assertThrows(AbstractMethodError.class, () -> provider.createPlugin(invocationInfo())); + + var error = assertThrows( + IllegalStateException.class, + () -> DynamicPluginLoader.loadConfiguredPluginFactories(PROVIDER_NAME, List.of(provider), List.of())); + + var message = error.getMessage(); + assertTrue(message.contains("does not implement createPlugin(InvocationInfo)"), message); + assertTrue(message.contains("is static"), message); + assertTrue(message.contains("Rebuild the provider against this SDK version and redeploy it"), message); + } + + @Test + void rejectsAProviderWhoseCreatePluginReturnsSomethingElse(@TempDir Path workDir) throws Exception { + // A createPlugin(InvocationInfo) whose return type is unrelated to DurableExecutionPlugin does not override the + // interface method, so it is concrete and still leaves the interface call unimplemented. + var provider = staleProviderOfShape(workDir, WRONG_RETURN_PROVIDER_SOURCE); + + var createPlugin = provider.getClass().getMethod("createPlugin", InvocationInfo.class); + assertEquals(String.class, createPlugin.getReturnType()); + assertTrue(!Modifier.isAbstract(createPlugin.getModifiers())); + assertThrows(AbstractMethodError.class, () -> provider.createPlugin(invocationInfo())); + + var error = assertThrows( + IllegalStateException.class, + () -> DynamicPluginLoader.loadConfiguredPluginFactories(PROVIDER_NAME, List.of(provider), List.of())); + + var message = error.getMessage(); + assertTrue(message.contains("does not implement createPlugin(InvocationInfo)"), message); + assertTrue(message.contains("returns java.lang.String"), message); + } + + @Test + void rejectsAProviderWhoseCreatePluginHasNoBridge(@TempDir Path workDir) throws Exception { + // The shape an assignability test accepts and the JVM does not. The method is concrete, takes this SDK's + // InvocationInfo, and returns a DurableExecutionPlugin subtype, but it overrides nothing, so there is no bridge + // carrying the interface's erased descriptor and invokeinterface finds nothing to dispatch to. + var provider = staleProviderOfShape(workDir, SUBTYPE_RETURN_PROVIDER_SOURCE); + + var resolved = provider.getClass().getMethod("createPlugin", InvocationInfo.class); + assertTrue(!Modifier.isAbstract(resolved.getModifiers())); + assertTrue(!Modifier.isStatic(resolved.getModifiers())); + assertTrue( + DurableExecutionPlugin.class.isAssignableFrom(resolved.getReturnType()), + "the fixture is only interesting while an assignability test would accept it"); + assertTrue( + Stream.of(provider.getClass().getMethods()) + .noneMatch(method -> "createPlugin".equals(method.getName()) + && method.getParameterCount() == 1 + && method.getParameterTypes()[0] == InvocationInfo.class + && method.getReturnType() == DurableExecutionPlugin.class + && !Modifier.isAbstract(method.getModifiers())), + "the fixture must carry no bridge method, which is what makes the call fail"); + assertThrows(AbstractMethodError.class, () -> provider.createPlugin(invocationInfo())); + + var error = assertThrows( + IllegalStateException.class, + () -> DynamicPluginLoader.loadConfiguredPluginFactories(PROVIDER_NAME, List.of(provider), List.of())); + + var message = error.getMessage(); + assertTrue(message.contains("does not implement createPlugin(InvocationInfo)"), message); + assertTrue(message.contains("carries no method returning"), message); + } + + @Test + void acceptsAProviderWhoseCreatePluginReturnsASubtype() { + // The same covariant return, compiled against the current interface: javac emits the bridge, the interface call + // dispatches, and the check must accept it. This is what keeps the exact-descriptor rule from rejecting a + // provider that works. + var provider = new CovariantProvider(); + + var factories = DynamicPluginLoader.loadConfiguredPluginFactories("covariant", List.of(provider), List.of()); + + assertEquals(List.of(provider), factories); + assertTrue(provider.createPlugin(invocationInfo()) instanceof CurrentPlugin); + } + + @Test + void doesNotRejectAStaleProviderThatWasNotSelected(@TempDir Path workDir) throws Exception { + var staleProvider = staleProvider(workDir); + var selectedProvider = new CurrentProvider(); + + // A stale provider JAR on the class path that no name in the environment variable selects is never called, so + // rejecting it would fail startup for a deployment that works. Only selected providers are checked. + var factories = DynamicPluginLoader.loadConfiguredPluginFactories( + "current", List.of(staleProvider, selectedProvider), List.of()); + + assertEquals(List.of(selectedProvider), factories); + } + + @Test + void namesTheProviderClassWhenNoArtifactLocationIsReported(@TempDir Path workDir) throws Exception { + // A class whose loader reports no code source has no artifact to name. That says nothing about whether the + // provider is usable, so the failure is still reported and only the location is left out of the message. + var provider = staleProvider(workDir, null); + + var error = assertThrows( + IllegalStateException.class, + () -> DynamicPluginLoader.loadConfiguredPluginFactories(PROVIDER_NAME, List.of(provider), List.of())); + + var message = error.getMessage(); + assertTrue(message.contains("(" + PROVIDER_CLASS + ")"), message); + assertTrue(message.contains("does not implement createPlugin(InvocationInfo)"), message); + } + + private static InvocationInfo invocationInfo() { + return new InvocationInfo("req-123", "arn:test", true, Instant.now()); + } + + /** + * Compiles the fixture provider against the older interface and returns an instance of it loaded against the + * current interface. + * + *

    The stub interfaces are compiled only so the provider source has something to compile against, and they are + * not handed to the loader. Class loading for every {@code software.amazon.lambda.durable} name therefore reaches + * the parent loader and resolves to this SDK's classes, which is the same resolution a deployed provider JAR gets + * from the function class path. + */ + private static DurableExecutionPluginProvider staleProvider(Path workDir) throws Exception { + return staleProviderOfShape(workDir, PROVIDER_SOURCE); + } + + /** @param providerSource the stale shape to compile, one of the provider sources above */ + private static DurableExecutionPluginProvider staleProviderOfShape(Path workDir, String providerSource) + throws Exception { + return staleProvider(workDir, workDir.resolve("classes").toUri().toURL(), providerSource); + } + + /** @param artifactLocation reported as the fixture classes' code source, or null to report none */ + private static DurableExecutionPluginProvider staleProvider(Path workDir, URL artifactLocation) throws Exception { + return staleProvider(workDir, artifactLocation, PROVIDER_SOURCE); + } + + /** @param artifactLocation reported as the fixture classes' code source, or null to report none */ + private static DurableExecutionPluginProvider staleProvider( + Path workDir, URL artifactLocation, String providerSource) throws Exception { + var compiler = ToolProvider.getSystemJavaCompiler(); + assumeTrue(compiler != null, "This test compiles a fixture and needs a JDK rather than a JRE"); + + var classDir = compileFixture(compiler, workDir, providerSource); + var loader = new FixtureClassLoader( + DynamicPluginLoaderStaleProviderTest.class.getClassLoader(), + fixtureClasses(classDir), + artifactLocation); + var type = loader.loadClass(PROVIDER_CLASS); + return (DurableExecutionPluginProvider) type.getDeclaredConstructor().newInstance(); + } + + private static Path compileFixture(JavaCompiler compiler, Path workDir, String providerSource) throws Exception { + var sourceDir = Files.createDirectories(workDir.resolve("source")); + var classDir = Files.createDirectories(workDir.resolve("classes")); + var sources = new String[] { + write(sourceDir, "DurableExecutionPlugin.java", PLUGIN_SOURCE), + write(sourceDir, "DurableExecutionPluginProvider.java", OLD_PROVIDER_INTERFACE_SOURCE), + write(sourceDir, "InvocationInfo.java", INVOCATION_INFO_SOURCE), + write(sourceDir, "StaleAuditProvider.java", providerSource), + }; + + // The class path holds only the output directory, which is empty when the compile starts. This SDK's current + // interfaces are therefore not visible to the compile, and the provider is compiled against the stub above + // rather than against the interface it is meant to predate. + var arguments = Stream.concat( + Stream.of("--release", "17", "-classpath", classDir.toString(), "-d", classDir.toString()), + Stream.of(sources)) + .toArray(String[]::new); + var diagnostics = new ByteArrayOutputStream(); + var exitCode = compiler.run(null, null, diagnostics, arguments); + if (exitCode != 0) { + fail("Failed to compile the stale provider fixture: " + diagnostics.toString(StandardCharsets.UTF_8)); + } + return classDir; + } + + /** + * Returns the compiled fixture classes outside the {@code software.amazon.lambda.durable} packages, keyed by binary + * name. + * + *

    Excluding those packages is what leaves the stub interfaces behind. A stub that reached the loader would + * shadow this SDK's interface of the same name, and the provider would then implement the stub rather than the + * current interface, which is not the condition under test. + */ + private static Map fixtureClasses(Path classDir) throws Exception { + var classes = new HashMap(); + try (var files = Files.walk(classDir)) { + for (var file : files.filter(f -> f.toString().endsWith(".class")).toList()) { + var relativePath = classDir.relativize(file).toString(); + var binaryName = relativePath + .substring(0, relativePath.length() - ".class".length()) + .replace(File.separatorChar, '.'); + if (!binaryName.startsWith("software.amazon.lambda.durable.")) { + classes.put(binaryName, Files.readAllBytes(file)); + } + } + } + return classes; + } + + private static String write(Path sourceDir, String fileName, String source) throws Exception { + var file = sourceDir.resolve(fileName); + Files.writeString(file, source); + return file.toString(); + } + + /** Defines the fixture classes and delegates every other name to the parent loader. */ + private static final class FixtureClassLoader extends ClassLoader { + + private final Map fixtureClasses; + private final ProtectionDomain protectionDomain; + + /** + * @param artifactLocation where the fixture classes were loaded from, reported as their code source so the + * failure message can name it as it names a deployed provider's JAR, or null to report no code source + */ + FixtureClassLoader(ClassLoader parent, Map fixtureClasses, URL artifactLocation) { + super(parent); + this.fixtureClasses = Map.copyOf(fixtureClasses); + this.protectionDomain = artifactLocation == null + ? null + : new ProtectionDomain(new CodeSource(artifactLocation, (CodeSigner[]) null), null); + } + + @Override + protected Class findClass(String name) throws ClassNotFoundException { + var bytes = fixtureClasses.get(name); + if (bytes == null) { + return super.findClass(name); + } + return defineClass(name, bytes, 0, bytes.length, protectionDomain); + } + } + + /** A provider written against the current interface, used to show that only selected providers are checked. */ + private static final class CurrentProvider implements DurableExecutionPluginProvider { + + @Override + public String getName() { + return "current"; + } + + @Override + public DurableExecutionPlugin createPlugin(InvocationInfo invocationInfo) { + return new CurrentPlugin(); + } + } + + /** A provider written against the current interface with a covariant return, so javac emits a bridge. */ + private static final class CovariantProvider implements DurableExecutionPluginProvider { + + @Override + public String getName() { + return "covariant"; + } + + @Override + public CurrentPlugin createPlugin(InvocationInfo invocationInfo) { + return new CurrentPlugin(); + } + } + + private static final class CurrentPlugin implements DurableExecutionPlugin {} +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/DynamicPluginLoaderTest.java b/sdk/src/test/java/software/amazon/lambda/durable/DynamicPluginLoaderTest.java index 17fe9f5b3..1aa7a572b 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/DynamicPluginLoaderTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/DynamicPluginLoaderTest.java @@ -8,92 +8,93 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.util.ArrayList; +import java.time.Instant; import java.util.Iterator; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import org.junit.jupiter.api.Test; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider; +import software.amazon.lambda.durable.plugin.InvocationInfo; class DynamicPluginLoaderTest { @Test - void unsetConfigurationPreservesExplicitPluginsWithoutDiscoveringProviders() { - var explicitPlugin = new FirstPlugin(); + void unsetConfigurationPreservesExplicitFactoriesWithoutDiscoveringProviders() { + DurableExecutionPluginFactory explicitFactory = info -> new FirstPlugin(); Iterable providers = () -> { throw new AssertionError("Providers should not be discovered"); }; - var plugins = DynamicPluginLoader.loadConfiguredPlugins(null, providers, List.of(explicitPlugin)); + var factories = DynamicPluginLoader.loadConfiguredPluginFactories(null, providers, List.of(explicitFactory)); - assertEquals(1, plugins.size()); - assertSame(explicitPlugin, plugins.get(0)); + assertEquals(1, factories.size()); + assertSame(explicitFactory, factories.get(0)); } @Test - void loadsRequestedProvidersBeforeExplicitPluginsInConfiguredOrder() { - var creationOrder = new ArrayList(); - var explicitPlugin = new ExplicitPlugin(); - var firstProvider = provider("first", FirstPlugin.class, () -> { - creationOrder.add("first"); + void loadsRequestedProvidersBeforeExplicitFactoriesInConfiguredOrder() { + DurableExecutionPluginFactory explicitFactory = info -> new ExplicitPlugin(); + var firstProvider = provider("first", FirstPlugin::new); + var secondProvider = provider("second", SecondPlugin::new); + + var factories = DynamicPluginLoader.loadConfiguredPluginFactories( + " second, first ", List.of(firstProvider, secondProvider), List.of(explicitFactory)); + + assertSame(secondProvider, factories.get(0)); + assertSame(firstProvider, factories.get(1)); + assertSame(explicitFactory, factories.get(2)); + } + + @Test + void doesNotCreatePluginsAtConfigurationTime() { + var creations = new AtomicInteger(); + var requestedProvider = provider("requested", () -> { + creations.incrementAndGet(); return new FirstPlugin(); }); - var secondProvider = provider("second", SecondPlugin.class, () -> { - creationOrder.add("second"); - return new SecondPlugin(); - }); - var plugins = DynamicPluginLoader.loadConfiguredPlugins( - " second, first ", List.of(firstProvider, secondProvider), List.of(explicitPlugin)); + var factories = + DynamicPluginLoader.loadConfiguredPluginFactories("requested", List.of(requestedProvider), List.of()); - assertInstanceOf(SecondPlugin.class, plugins.get(0)); - assertInstanceOf(FirstPlugin.class, plugins.get(1)); - assertSame(explicitPlugin, plugins.get(2)); - assertEquals(List.of("second", "first"), creationOrder); + // Plugins are created per invocation, not while configuration is resolved. + assertEquals(1, factories.size()); + assertEquals(0, creations.get()); + assertInstanceOf(FirstPlugin.class, factories.get(0).createPlugin(invocationInfo())); + assertEquals(1, creations.get()); } @Test - void doesNotCreateProvidersOutsideTheAllowList() { - var unrequestedCreations = new AtomicInteger(); - var requestedProvider = provider("requested", FirstPlugin.class, FirstPlugin::new); - var unrequestedProvider = provider("unrequested", SecondPlugin.class, () -> { - unrequestedCreations.incrementAndGet(); - return new SecondPlugin(); - }); + void doesNotSelectProvidersOutsideTheAllowList() { + var requestedProvider = provider("requested", FirstPlugin::new); + var unrequestedProvider = provider("unrequested", SecondPlugin::new); - var plugins = DynamicPluginLoader.loadConfiguredPlugins( + var factories = DynamicPluginLoader.loadConfiguredPluginFactories( "requested", List.of(requestedProvider, unrequestedProvider), List.of()); - assertEquals(1, plugins.size()); - assertEquals(0, unrequestedCreations.get()); + assertEquals(List.of(requestedProvider), factories); } @Test - void loadsExplicitAndDynamicPluginsOfTheSameType() { - var creations = new AtomicInteger(); - var explicitPlugin = new FirstPlugin(); - var dynamicPlugin = new FirstPlugin(); - var duplicateProvider = provider("first", FirstPlugin.class, () -> { - creations.incrementAndGet(); - return dynamicPlugin; - }); + void loadsExplicitAndDynamicFactoriesOfTheSameType() { + DurableExecutionPluginFactory explicitFactory = info -> new FirstPlugin(); + var duplicateProvider = provider("first", FirstPlugin::new); - var plugins = - DynamicPluginLoader.loadConfiguredPlugins("first", List.of(duplicateProvider), List.of(explicitPlugin)); + var factories = DynamicPluginLoader.loadConfiguredPluginFactories( + "first", List.of(duplicateProvider), List.of(explicitFactory)); - assertEquals(2, plugins.size()); - assertSame(dynamicPlugin, plugins.get(0)); - assertSame(explicitPlugin, plugins.get(1)); - assertEquals(1, creations.get()); + assertEquals(2, factories.size()); + assertSame(duplicateProvider, factories.get(0)); + assertSame(explicitFactory, factories.get(1)); } @Test void rejectsEmptyConfiguredProviderName() { var error = assertThrows( IllegalStateException.class, - () -> DynamicPluginLoader.loadConfiguredPlugins("first,,second", List.of(), List.of())); + () -> DynamicPluginLoader.loadConfiguredPluginFactories("first,,second", List.of(), List.of())); assertTrue(error.getMessage().contains("must be non-empty")); assertTrue(error.getMessage().contains(DynamicPluginLoader.PLUGINS_ENVIRONMENT_VARIABLE)); @@ -103,18 +104,19 @@ void rejectsEmptyConfiguredProviderName() { void rejectsDuplicateConfiguredProviderName() { var error = assertThrows( IllegalStateException.class, - () -> DynamicPluginLoader.loadConfiguredPlugins("first,first", List.of(), List.of())); + () -> DynamicPluginLoader.loadConfiguredPluginFactories("first,first", List.of(), List.of())); assertTrue(error.getMessage().contains("listed more than once")); } @Test void rejectsUnknownProviderAndListsAvailableNames() { - var availableProvider = provider("available", FirstPlugin.class, FirstPlugin::new); + var availableProvider = provider("available", FirstPlugin::new); var error = assertThrows( IllegalStateException.class, - () -> DynamicPluginLoader.loadConfiguredPlugins("missing", List.of(availableProvider), List.of())); + () -> DynamicPluginLoader.loadConfiguredPluginFactories( + "missing", List.of(availableProvider), List.of())); assertTrue(error.getMessage().contains("No DurableExecutionPluginProvider named 'missing'")); assertTrue(error.getMessage().contains("available")); @@ -122,12 +124,12 @@ void rejectsUnknownProviderAndListsAvailableNames() { @Test void rejectsDuplicateDiscoveredProviderNames() { - var firstProvider = provider("duplicate", FirstPlugin.class, FirstPlugin::new); - var secondProvider = provider("duplicate", SecondPlugin.class, SecondPlugin::new); + var firstProvider = provider("duplicate", FirstPlugin::new); + var secondProvider = provider("duplicate", SecondPlugin::new); var error = assertThrows( IllegalStateException.class, - () -> DynamicPluginLoader.loadConfiguredPlugins( + () -> DynamicPluginLoader.loadConfiguredPluginFactories( "duplicate", List.of(firstProvider, secondProvider), List.of())); assertTrue(error.getMessage().contains("Multiple DurableExecutionPluginProvider implementations")); @@ -135,38 +137,15 @@ void rejectsDuplicateDiscoveredProviderNames() { } @Test - void rejectsIncompatibleProviderApiVersion() { - var provider = new TestProvider("first", 2, FirstPlugin.class, FirstPlugin::new); + void rejectsProviderWithInvalidName() { + var blankNameProvider = provider(" ", FirstPlugin::new); var error = assertThrows( IllegalStateException.class, - () -> DynamicPluginLoader.loadConfiguredPlugins("first", List.of(provider), List.of())); + () -> DynamicPluginLoader.loadConfiguredPluginFactories( + "first", List.of(blankNameProvider), List.of())); - assertTrue(error.getMessage().contains("uses provider API version 2")); - assertTrue(error.getMessage().contains("requires version " + DurableExecutionPluginProvider.API_VERSION)); - } - - @Test - void rejectsInvalidDeclaredPluginType() { - var provider = provider("invalid", DurableExecutionPlugin.class, FirstPlugin::new); - - var error = assertThrows( - IllegalStateException.class, - () -> DynamicPluginLoader.loadConfiguredPlugins("invalid", List.of(provider), List.of())); - - assertTrue(error.getMessage().contains("must declare a concrete DurableExecutionPlugin type")); - } - - @Test - void rejectsPluginThatDoesNotMatchDeclaredType() { - var provider = provider("first", FirstPlugin.class, SecondPlugin::new); - - var error = assertThrows( - IllegalStateException.class, - () -> DynamicPluginLoader.loadConfiguredPlugins("first", List.of(provider), List.of())); - - assertTrue(error.getMessage().contains("declared type")); - assertTrue(error.getMessage().contains(SecondPlugin.class.getName())); + assertTrue(error.getMessage().contains("returned an invalid name")); } @Test @@ -185,38 +164,70 @@ public DurableExecutionPluginProvider next() { var error = assertThrows( IllegalStateException.class, - () -> DynamicPluginLoader.loadConfiguredPlugins("first", providers, List.of())); + () -> DynamicPluginLoader.loadConfiguredPluginFactories("first", providers, List.of())); assertTrue(error.getMessage().contains("Failed to discover")); assertInstanceOf(LinkageError.class, error.getCause()); } + // ─── Providers that do implement createPlugin(InvocationInfo) ──────── + // + // The startup check that rejects a provider compiled against the older provider interface reads whether + // createPlugin(InvocationInfo) resolves to an abstract method on the runtime class. These cases cover the shapes + // in which a provider written against this SDK supplies that method without declaring it on its own class, so the + // check must accept all of them. DynamicPluginLoaderStaleProviderTest covers the case the check rejects. + @Test - void wrapsPluginCreationFailure() { - var provider = provider("first", FirstPlugin.class, () -> { - throw new IllegalArgumentException("bad settings"); - }); + void acceptsProviderThatDeclaresCreatePluginItself() { + var declaringProvider = provider("declaring", FirstPlugin::new); - var error = assertThrows( - IllegalStateException.class, - () -> DynamicPluginLoader.loadConfiguredPlugins("first", List.of(provider), List.of())); + var factories = + DynamicPluginLoader.loadConfiguredPluginFactories("declaring", List.of(declaringProvider), List.of()); - assertTrue(error.getMessage().contains("failed to create its plugin")); - assertInstanceOf(IllegalArgumentException.class, error.getCause()); + assertInstanceOf(FirstPlugin.class, factories.get(0).createPlugin(invocationInfo())); } - private static TestProvider provider( - String name, - Class pluginType, - Supplier pluginSupplier) { - return new TestProvider(name, DurableExecutionPluginProvider.API_VERSION, pluginType, pluginSupplier); + @Test + void acceptsProviderThatInheritsCreatePluginFromAbstractBaseClass() { + var inheritingProvider = new InheritsFromBaseProvider(); + + var factories = DynamicPluginLoader.loadConfiguredPluginFactories( + "inherits-from-base", List.of(inheritingProvider), List.of()); + + assertInstanceOf(FirstPlugin.class, factories.get(0).createPlugin(invocationInfo())); + } + + @Test + void acceptsProviderThatInheritsCreatePluginAsDefaultMethod() { + var inheritingProvider = new InheritsDefaultMethodProvider(); + + var factories = DynamicPluginLoader.loadConfiguredPluginFactories( + "inherits-default-method", List.of(inheritingProvider), List.of()); + + assertInstanceOf(FirstPlugin.class, factories.get(0).createPlugin(invocationInfo())); } - private record TestProvider( - String name, - int apiVersion, - Class pluginType, - Supplier pluginSupplier) + @Test + void acceptsProviderThatNarrowsTheCreatePluginReturnType() { + // A narrowed return type makes the compiler emit a bridge method, so createPlugin(InvocationInfo) resolves to + // one of two declarations on the provider class. Neither is abstract. + var covariantProvider = new NarrowedReturnTypeProvider(); + + var factories = DynamicPluginLoader.loadConfiguredPluginFactories( + "narrowed-return-type", List.of(covariantProvider), List.of()); + + assertInstanceOf(FirstPlugin.class, factories.get(0).createPlugin(invocationInfo())); + } + + private static InvocationInfo invocationInfo() { + return new InvocationInfo("req-123", "arn:test", true, Instant.now()); + } + + private static TestProvider provider(String name, Supplier pluginSupplier) { + return new TestProvider(name, pluginSupplier); + } + + private record TestProvider(String name, Supplier pluginSupplier) implements DurableExecutionPluginProvider { @Override @@ -225,24 +236,62 @@ public String getName() { } @Override - public int getApiVersion() { - return apiVersion; + public DurableExecutionPlugin createPlugin(InvocationInfo invocationInfo) { + return pluginSupplier.get(); } + } + + private static final class ExplicitPlugin implements DurableExecutionPlugin {} + + private static final class FirstPlugin implements DurableExecutionPlugin {} + + private static final class SecondPlugin implements DurableExecutionPlugin {} + + /** A provider whose {@code createPlugin} implementation is inherited from a superclass. */ + private abstract static class BaseProvider implements DurableExecutionPluginProvider { @Override - public Class getPluginType() { - return pluginType; + public DurableExecutionPlugin createPlugin(InvocationInfo invocationInfo) { + return new FirstPlugin(); } + } + + private static final class InheritsFromBaseProvider extends BaseProvider { @Override - public DurableExecutionPlugin createPlugin() { - return pluginSupplier.get(); + public String getName() { + return "inherits-from-base"; } } - private static final class ExplicitPlugin implements DurableExecutionPlugin {} + /** A provider whose {@code createPlugin} implementation is inherited as a default method. */ + private interface DefaultMethodProvider extends DurableExecutionPluginProvider { - private static final class FirstPlugin implements DurableExecutionPlugin {} + @Override + default DurableExecutionPlugin createPlugin(InvocationInfo invocationInfo) { + return new FirstPlugin(); + } + } - private static final class SecondPlugin implements DurableExecutionPlugin {} + private static final class InheritsDefaultMethodProvider implements DefaultMethodProvider { + + @Override + public String getName() { + return "inherits-default-method"; + } + } + + /** A provider that declares {@code createPlugin} with a narrowed return type. */ + private static final class NarrowedReturnTypeProvider implements DurableExecutionPluginProvider { + + @Override + public String getName() { + return "narrowed-return-type"; + } + + @Override + public FirstPlugin createPlugin(InvocationInfo invocationInfo) { + return new FirstPlugin(); + } + } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/ApiRequestDelayedBatcherTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/ApiRequestDelayedBatcherTest.java index 6aec8f7c1..136833bb1 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/ApiRequestDelayedBatcherTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/ApiRequestDelayedBatcherTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -18,11 +19,17 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.util.ExceptionHelper; class ApiRequestDelayedBatcherTest { private static final Duration SHORT_DELAY = Duration.ofMillis(5); @@ -226,4 +233,117 @@ void whenMultipleBatchesFlushedViaShutdown_allFuturesComplete() { cut.shutdown(); assertTrue(future2.isDone()); } + + @Test + void pluginFatalAbortsQueuedDelayedAndFutureRequestsWithoutHangingShutdown() throws Exception { + var fatal = new InternalError("plugin callback fatal"); + var signal = new AtomicReference(); + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + var calls = new AtomicInteger(); + var batcher = new ApiRequestDelayedBatcher( + 1, + 100, + value -> 1, + batch -> { + calls.incrementAndGet(); + entered.countDown(); + try { + assertTrue(release.await(2, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + signal.set(fatal); + throw fatal; + }, + signal::get); + var first = batcher.submit(input, Duration.ofDays(1)); + var queued = batcher.submit(input, Duration.ofDays(1)); + var shutdown = CompletableFuture.runAsync(batcher::shutdown); + try { + assertTrue(entered.await(2, TimeUnit.SECONDS)); + var delayed = batcher.submit(input, Duration.ofDays(1)); + release.countDown(); + for (var request : List.of(first, queued, delayed)) + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> request.get(2, TimeUnit.SECONDS)) + .getCause()); + assertSame( + fatal, + ExceptionHelper.unwrapAsyncFailure( + assertThrows(ExecutionException.class, () -> shutdown.get(2, TimeUnit.SECONDS)))); + var later = batcher.submit(input, Duration.ZERO); + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> later.get(2, TimeUnit.SECONDS)) + .getCause()); + assertEquals(1, calls.get()); + } finally { + release.countDown(); + } + } + + @Test + void unreportedFatalKeepsTheLegacyBatchFailureBehavior() throws Exception { + var error = new InternalError("not reported by a plugin"); + var calls = new AtomicInteger(); + var batcher = new ApiRequestDelayedBatcher(1, 100, value -> 1, batch -> { + if (calls.getAndIncrement() == 0) throw error; + }); + var first = batcher.submit(input, Duration.ZERO); + assertSame( + error, + assertThrows(ExecutionException.class, () -> first.get(2, TimeUnit.SECONDS)) + .getCause()); + batcher.submit(input, Duration.ZERO).get(2, TimeUnit.SECONDS); + batcher.shutdown(); + assertEquals(2, calls.get()); + } + + @Test + @SuppressWarnings("removal") + void reportedFatalEscapesItsCheckpointWorker() throws Exception { + for (Error fatal : List.of(new InternalError("checkpoint hook"), new ThreadDeath())) { + var signal = new AtomicReference(); + var uncaught = new AtomicReference(); + var escaped = new CountDownLatch(1); + var batcher = new ApiRequestDelayedBatcher( + 1, + 100, + value -> 1, + batch -> { + Thread.currentThread().setUncaughtExceptionHandler((owner, failure) -> { + uncaught.set(failure); + escaped.countDown(); + }); + signal.set(fatal); + throw fatal; + }, + signal::get); + var request = batcher.submit(input, Duration.ZERO); + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> request.get(2, TimeUnit.SECONDS)) + .getCause()); + assertTrue(escaped.await(2, TimeUnit.SECONDS), "fatal must leave CompletableFuture containment"); + assertSame(fatal, uncaught.get()); + } + } + + @Test + void alreadyReportedPluginFatalPreventsBatchExecution() throws Exception { + var fatal = new InternalError("already reported by another hook"); + var calls = new AtomicInteger(); + var batcher = + new ApiRequestDelayedBatcher(1, 100, value -> 1, batch -> calls.incrementAndGet(), () -> fatal); + var request = batcher.submit(input, Duration.ofMinutes(1)); + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> request.get(2, TimeUnit.SECONDS)) + .getCause()); + assertEquals(0, calls.get(), "do not start a backend request after a reported plugin fatal"); + assertSame( + fatal, ExceptionHelper.unwrapAsyncFailure(assertThrows(CompletionException.class, batcher::shutdown))); + } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/DurableExecutionTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/DurableExecutionTest.java index e0773aaeb..e042e182a 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/DurableExecutionTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/DurableExecutionTest.java @@ -8,8 +8,11 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static software.amazon.lambda.durable.TypeToken.get; +import com.amazonaws.services.lambda.runtime.Context; import java.time.Instant; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -34,6 +37,8 @@ import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.operation.BaseDurableOperation; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationInfo; class DurableExecutionTest { @@ -50,6 +55,47 @@ private DurableConfig configWithMockClient() { .build(); } + @Test + void capturesTraceHeaderOnRuntimeThreadBeforeDispatch() { + var runtimeThread = Thread.currentThread(); + var header = "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=0"; + var lambdaContext = mock(RuntimeContext.class); + when(lambdaContext.getXrayTraceId()).thenAnswer(ignored -> { + assertEquals(runtimeThread, Thread.currentThread()); + return header; + }); + when(lambdaContext.getRemainingTimeInMillis()).thenReturn(30000); + var seen = new AtomicReference(); + var plugin = new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo info) { + assertFalse(runtimeThread == Thread.currentThread()); + seen.set(info.xRayTraceId()); + } + }; + var executionOp = Operation.builder() + .id(EXECUTION_OP_ID) + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(EXECUTION_START_TIME) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + var input = new DurableExecutionInput( + EXECUTION_ARN, + "token", + CheckpointUpdatedExecutionState.builder() + .operations(executionOp) + .build()); + var config = DurableConfig.builder() + .withDurableExecutionClient(TestUtils.createMockClient()) + .withPlugins(ignored -> plugin) + .build(); + var output = DurableExecutor.execute(input, lambdaContext, get(String.class), (value, ctx) -> value, config); + assertEquals(ExecutionStatus.SUCCEEDED, output.status()); + assertEquals(header, seen.get()); + } + @Test void testExecuteSuccess() { var executionOp = Operation.builder() @@ -411,7 +457,7 @@ void testExecutorNotShutdownAfterMultipleHandlerInvocations() { (userInput, ctx) -> ctx.step("test1", String.class, stepCtx -> "Result 1: " + userInput), config); - assertEquals(ExecutionStatus.SUCCEEDED, output1.status()); + assertEquals(ExecutionStatus.SUCCEEDED, output1.status(), () -> "First invocation error: " + output1.error()); assertFalse(sharedExecutor.isShutdown(), "Executor should not be shutdown after first execution"); // Create second input with different execution operation @@ -440,7 +486,7 @@ void testExecutorNotShutdownAfterMultipleHandlerInvocations() { (userInput, ctx) -> ctx.step("test2", String.class, stepCtx -> "Result 2: " + userInput), config); - assertEquals(ExecutionStatus.SUCCEEDED, output2.status()); + assertEquals(ExecutionStatus.SUCCEEDED, output2.status(), () -> "Second invocation error: " + output2.error()); assertFalse(sharedExecutor.isShutdown(), "Executor should not be shutdown after second execution"); // Verify both executions completed successfully and used the same executor @@ -448,6 +494,13 @@ void testExecutorNotShutdownAfterMultipleHandlerInvocations() { assertTrue(output2.result().contains("Result 2: test-input-2")); } + private abstract static class RuntimeContext implements Context { + @Override + public String getXrayTraceId() { + return null; + } + } + private Operation executionOp() { return Operation.builder() .id(EXECUTION_OP_ID) diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java index 1552d2b21..ea7f95230 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/ExecutionManagerTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -12,6 +13,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -372,6 +374,35 @@ void deferredSuspensionOccursWhenCheckpointFinishesWithoutReactivatingThread() { assertTrue(manager.isExecutionCompletedExceptionally()); } + @Test + void lateScopeFatalDoesNotHideAnUnrelatedShutdownFailure() throws Exception { + var manager = createManager(List.of(executionOp())); + var checkpoint = mock(CheckpointManager.class); + var checkpointField = ExecutionManager.class.getDeclaredField("checkpointManager"); + checkpointField.setAccessible(true); + checkpointField.set(manager, checkpoint); + var unrelated = new CompletionException(new IllegalStateException("unrelated shutdown failure")); + doThrow(unrelated).when(checkpoint).shutdown(); + assertNull(manager.beginInvocationFinalization()); + manager.recordHandlerScopeFatal(new InternalError("late owner failure")); + assertSame(unrelated, assertThrows(CompletionException.class, manager::close)); + } + + @Test + void scopeFatalBeforeFinalizationRetainsItsShutdownCause() throws Exception { + var manager = createManager(List.of(executionOp())); + var checkpoint = mock(CheckpointManager.class); + var checkpointField = ExecutionManager.class.getDeclaredField("checkpointManager"); + checkpointField.setAccessible(true); + checkpointField.set(manager, checkpoint); + var fatal = new InternalError("early owner failure"); + doThrow(new CompletionException(fatal)).when(checkpoint).shutdown(); + manager.recordHandlerScopeFatal(fatal); + assertSame(fatal, manager.beginInvocationFinalization()); + assertSame( + fatal, assertThrows(CompletionException.class, manager::close).getCause()); + } + @Test void checkpointDeliveryIsAtomicWithOperationRegistration() throws Exception { var manager = createManager(List.of(executionOp(), stepOp("step", OperationStatus.PENDING))); @@ -379,19 +410,30 @@ void checkpointDeliveryIsAtomicWithOperationRegistration() throws Exception { when(durableContext.getExecutionManager()).thenReturn(manager); var publicationReached = new CountDownLatch(1); var allowPublication = new CountDownLatch(1); - var idCalls = new AtomicInteger(); - var terminalOperation = mock(Operation.class); - when(terminalOperation.id()).thenAnswer(invocation -> { - if (idCalls.incrementAndGet() == 3) { - publicationReached.countDown(); - assertTrue(allowPublication.await(5, TimeUnit.SECONDS)); + var terminalOperation = stepOp("step", OperationStatus.SUCCEEDED).toBuilder() + .name("step") + .subType(OperationSubType.STEP.getValue()) + .build(); + var storageField = ExecutionManager.class.getDeclaredField("operationStorage"); + storageField.setAccessible(true); + @SuppressWarnings("unchecked") + var storage = (Map) storageField.get(manager); + // Block the actual publication, not an incidental number of Operation.id() reads. + storageField.set(manager, new ConcurrentHashMap(storage) { + @Override + public Operation put(String id, Operation value) { + if (value == terminalOperation) { + publicationReached.countDown(); + try { + assertTrue(allowPublication.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError(failure); + } + } + return super.put(id, value); } - return "step"; }); - when(terminalOperation.name()).thenReturn("step"); - when(terminalOperation.type()).thenReturn(OperationType.STEP); - when(terminalOperation.subType()).thenReturn(OperationSubType.STEP.getValue()); - when(terminalOperation.status()).thenReturn(OperationStatus.SUCCEEDED); class TestOperation extends BaseDurableOperation { TestOperation() { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/FatalInvocationBoundaryTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/FatalInvocationBoundaryTest.java new file mode 100644 index 000000000..fdec2f596 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/FatalInvocationBoundaryTest.java @@ -0,0 +1,294 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.execution; + +import static org.junit.jupiter.api.Assertions.*; +import static software.amazon.lambda.durable.TypeToken.get; + +import java.time.Instant; +import java.util.List; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.ExecutionDetails; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TestUtils; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.serde.JacksonSerDes; + +class FatalInvocationBoundaryTest { + @SuppressWarnings("removal") + static Stream fatalCases() { + return Stream.of( + "factory", + "hook", + "handler", + "wrapped-handler", + "wrapped-factory", + "wrapped-hook", + "future-factory", + "future-hook", + "future-handler") + .flatMap(stage -> Stream.of(new OutOfMemoryError("simulated VM failure"), new ThreadDeath()) + .map(error -> Arguments.of(stage, error))); + } + + @ParameterizedTest + @MethodSource("fatalCases") + void fatalCauseEscapesCallerAndWorkerAfterFinalization(String stage, Error fatal) throws Exception { + var uncaught = new AtomicReference(); + var workerFailure = new CountDownLatch(1); + var executor = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "fatal-invocation-test"); + thread.setDaemon(true); + thread.setUncaughtExceptionHandler((failed, error) -> { + uncaught.set(error); + workerFailure.countDown(); + }); + return thread; + }); + try { + assertFatalInvocation(stage, fatal, executor); + assertTrue(workerFailure.await(5, TimeUnit.SECONDS), "Fatal error must escape the async runnable"); + assertSame(fatal, uncaught.get()); + } finally { + executor.shutdownNow(); + } + } + + @Test + @SuppressWarnings("removal") + void directExecutorFinalizesBeforeRethrowingFatalOnCaller() { + assertFatalInvocation("factory", new ThreadDeath(), new DirectExecutor()); + } + + @ParameterizedTest + @ValueSource(strings = {"factory", "handler"}) + void endHookFatalDoesNotReplaceAnEarlierInvocationFatal(String stage) { + var primary = new InternalError("original invocation fatal"); + var cleanup = new InternalError("end hook fatal"); + var end = new AtomicReference(); + DurableExecutionPluginFactory failingEnd = ignored -> new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + throw cleanup; + } + }; + DurableExecutionPluginFactory recorder = ignored -> new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo info) { + assertNull(end.getAndSet(info), "finalization must remain paired exactly once"); + } + }; + var config = DurableConfig.builder() + .withDurableExecutionClient(TestUtils.createMockClient()) + .withExecutorService(new DirectExecutor()) + .withPlugins(failingEnd, recorder, failingPlugin(stage, primary)) + .build(); + assertSame( + primary, + assertThrows( + InternalError.class, + () -> DurableExecutor.execute( + input(), + null, + get(String.class), + (value, context) -> { + throw primary; + }, + config))); + assertEquals(InvocationStatus.RETRYING, end.get().invocationStatus()); + assertSame(primary, end.get().executionError()); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void incompatibleOptionalApiDoesNotChangeHandlerResult(boolean wrapped) { + var end = new AtomicReference(); + DurableExecutionPluginFactory observer = info -> new DurableExecutionPlugin() { + @Override + public void onInvocationEnd(InvocationEndInfo info) { + end.set(info); + } + }; + DurableExecutionPluginFactory incompatible = info -> { + var failure = new NoSuchMethodError("boolean io.opentelemetry.api.GlobalOpenTelemetry.isSet()"); + if (wrapped) + CompletableFuture.failedFuture(new ExecutionException(failure)).join(); + throw failure; + }; + var config = DurableConfig.builder() + .withDurableExecutionClient(TestUtils.createMockClient()) + .withPlugins(observer, incompatible) + .build(); + var output = DurableExecutor.execute(input(), null, get(String.class), (value, context) -> value, config); + assertEquals(ExecutionStatus.SUCCEEDED, output.status()); + assertEquals("\"input\"", output.result()); + assertEquals(InvocationStatus.SUCCEEDED, end.get().invocationStatus()); + } + + @Test + void nestedFatalResultDeliveryEscapesWithItsOriginalCause() { + var fatal = new OutOfMemoryError("simulated result-delivery failure"); + var end = new AtomicReference(); + DurableExecutionPluginFactory observer = info -> new DurableExecutionPlugin() { + @Override + public void onInvocationEnd(InvocationEndInfo info) { + assertNull(end.getAndSet(info)); + } + }; + var serDes = new JacksonSerDes() { + @Override + public String serialize(Object value) { + return CompletableFuture.failedFuture(new ExecutionException(fatal)) + .join(); + } + }; + var config = DurableConfig.builder() + .withDurableExecutionClient(TestUtils.createMockClient()) + .withSerDes(serDes) + .withExecutorService(new DirectExecutor()) + .withPlugins(observer) + .build(); + assertSame( + fatal, + assertThrows( + Error.class, + () -> DurableExecutor.execute( + input(), null, get(String.class), (value, context) -> value, config))); + assertNotNull(end.get()); + assertEquals(InvocationStatus.RETRYING, end.get().invocationStatus()); + assertSame(fatal, end.get().executionError()); + } + + private static void assertFatalInvocation(String stage, Error fatal, ExecutorService executor) { + var end = new AtomicReference(); + var called = new AtomicBoolean(); + DurableExecutionPluginFactory recorder = info -> new DurableExecutionPlugin() { + @Override + public void onInvocationEnd(InvocationEndInfo info) { + assertNull(end.getAndSet(info), "End hooks must fire once"); + } + }; + var config = DurableConfig.builder() + .withDurableExecutionClient(TestUtils.createMockClient()) + .withExecutorService(executor) + .withPlugins(recorder, failingPlugin(stage, fatal)) + .build(); + assertSame( + fatal, + assertThrows( + Error.class, + () -> DurableExecutor.execute( + input(), + null, + get(String.class), + (value, context) -> { + called.set(true); + if (stage.equals("wrapped-handler")) throw new CompletionException(fatal); + if (stage.equals("future-handler")) + CompletableFuture.failedFuture(new ExecutionException(fatal)) + .join(); + if (stage.equals("handler")) throw fatal; + return value; + }, + config))); + assertEquals(stage.endsWith("handler"), called.get()); + assertNotNull(end.get()); + assertEquals(InvocationStatus.RETRYING, end.get().invocationStatus()); + assertSame(fatal, end.get().executionError()); + } + + private static DurableExecutionPluginFactory failingPlugin(String stage, Error fatal) { + return info -> { + switch (stage) { + case "factory" -> throw fatal; + case "wrapped-factory" -> CompletableFuture.failedFuture(fatal).join(); + case "future-factory" -> + CompletableFuture.failedFuture(new ExecutionException(fatal)) + .join(); + default -> {} + } + return new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo info) { + switch (stage) { + case "hook" -> throw fatal; + case "wrapped-hook" -> + CompletableFuture.failedFuture(fatal).join(); + case "future-hook" -> + CompletableFuture.failedFuture(new ExecutionException(fatal)) + .join(); + default -> {} + } + } + }; + }; + } + + private static DurableExecutionInput input() { + var operation = Operation.builder() + .id("id") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.parse("2026-10-02T00:00:00Z")) + .executionDetails( + ExecutionDetails.builder().inputPayload("\"input\"").build()) + .build(); + return new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/id", + "token", + CheckpointUpdatedExecutionState.builder().operations(operation).build()); + } + + private static final class DirectExecutor extends AbstractExecutorService { + @Override + public void execute(Runnable task) { + task.run(); + } + + @Override + public void shutdown() {} + + @Override + public List shutdownNow() { + return List.of(); + } + + @Override + public boolean isShutdown() { + return false; + } + + @Override + public boolean isTerminated() { + return false; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return true; + } + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/HandlerScopeHandoffTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/HandlerScopeHandoffTest.java new file mode 100644 index 000000000..0f107f043 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/HandlerScopeHandoffTest.java @@ -0,0 +1,100 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.execution; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import com.amazonaws.services.lambda.runtime.Context; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; + +class HandlerScopeHandoffTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void throwingCleanupCannotReplaceTheWinningControlFlow(boolean retry) throws Exception { + var original = control(retry); + var execution = CompletableFuture.failedFuture(original); + var handler = CompletableFuture.failedFuture(new IllegalStateException("finally failed")); + var result = handoff(execution, handler); + var thrown = assertThrows(CompletionException.class, result::join); + assertSame(original, thrown.getCause()); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void nonUnwindingCleanupDoesNotWaitForever(boolean retry) throws Exception { + var original = control(retry); + var execution = CompletableFuture.failedFuture(original); + var handler = new CompletableFuture(); + var caller = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "handoff-caller"); + thread.setDaemon(true); + return thread; + }); + try { + var response = caller.submit(() -> { + var result = handoff(execution, handler); + var thrown = assertThrows(CompletionException.class, result::join); + assertSame(original, thrown.getCause()); + return null; + }); + response.get(2, TimeUnit.SECONDS); + assertFalse(handler.isDone(), "timeout must not complete or cancel the owner task"); + } finally { + handler.complete("released"); + caller.shutdownNow(); + } + } + + @Test + void reservesFinalizationForEveryConfiguredPluginWithoutCancellingTheOwner() { + var context = mock(Context.class); + when(context.getRemainingTimeInMillis()).thenReturn(10_000); + var execution = CompletableFuture.failedFuture(control(false)); + var handler = new CompletableFuture(); + assertTimeoutPreemptively( + Duration.ofMillis(200), + () -> assertSame( + execution, + DurableExecutor.awaitHandlerScopes( + execution, handler, new AtomicBoolean(true), context, 2, new AtomicReference<>()))); + assertFalse(handler.isDone()); + } + + @Test + void noScopeDoesNotConsultTheRemainingTimeOrWaitForTheHandler() { + var context = mock(Context.class); + var execution = CompletableFuture.failedFuture(control(false)); + var handler = new CompletableFuture(); + assertSame( + execution, + DurableExecutor.awaitHandlerScopes( + execution, handler, new AtomicBoolean(false), context, 1, new AtomicReference<>())); + verifyNoInteractions(context); + assertFalse(handler.isDone()); + } + + private static CompletableFuture handoff( + CompletableFuture execution, CompletableFuture handler) { + return DurableExecutor.awaitHandlerScopes( + execution, handler, new AtomicBoolean(true), null, 1, new AtomicReference<>()); + } + + private static Throwable control(boolean retry) { + return retry + ? new UnrecoverableDurableExecutionException( + ErrorObject.builder().errorMessage("retry").build(), true) + : new SuspendExecutionException(); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/PluginFatalCheckpointAbortTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/PluginFatalCheckpointAbortTest.java new file mode 100644 index 000000000..e52d3704e --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/PluginFatalCheckpointAbortTest.java @@ -0,0 +1,158 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.execution; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.verifyNoInteractions; + +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import software.amazon.awssdk.services.lambda.model.*; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TestUtils; +import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.OperationIdentifier; +import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.operation.WaitOperation; +import software.amazon.lambda.durable.util.ExceptionHelper; + +@Timeout(10) +class PluginFatalCheckpointAbortTest { + @Test + void externalAbortSettlesDelayedRequestsAndRejectsLaterWork() throws Exception { + var fatal = new InternalError("external fatal"); + var calls = new AtomicInteger(); + var batcher = new ApiRequestDelayedBatcher(10, 100, String::length, items -> calls.incrementAndGet()); + var pending = batcher.submit("pending", Duration.ofMinutes(1)); + batcher.abortPending(fatal); + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> pending.get(1, TimeUnit.SECONDS)) + .getCause()); + var later = batcher.submit("later", Duration.ofMinutes(1)); + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> later.get(1, TimeUnit.SECONDS)) + .getCause()); + assertSame( + fatal, ExceptionHelper.unwrapAsyncFailure(assertThrows(CompletionException.class, batcher::shutdown))); + assertEquals(0, calls.get()); + } + + @Test + void abortWhileTheWorkerAssemblesABatchPreservesTheFatal() throws Exception { + var fatal = new InternalError("abort during assembly"); + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + var calls = new AtomicInteger(); + var batcher = new ApiRequestDelayedBatcher( + 10, + 100, + value -> { + entered.countDown(); + try { + assertTrue(release.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + throw new AssertionError(failure); + } + return value.length(); + }, + values -> calls.incrementAndGet()); + var pending = batcher.submit("work", Duration.ZERO); + try { + assertTrue(entered.await(3, TimeUnit.SECONDS)); + batcher.abortPending(fatal); + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> pending.get(1, TimeUnit.SECONDS)) + .getCause()); + } finally { + release.countDown(); + } + assertSame( + fatal, ExceptionHelper.unwrapAsyncFailure(assertThrows(CompletionException.class, batcher::shutdown))); + assertEquals(0, calls.get(), "a drained item must never reach the backend"); + } + + @Test + void scopeFatalAbortsCheckpointsAndPollersWithoutCompletingRootCleanupEarly() throws Exception { + var fatal = new InternalError("scope fatal"); + var client = TestUtils.createMockClient(); + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ofMinutes(1)) + .build(); + var execution = Operation.builder() + .id("execution") + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .build(); + var resumedWait = Operation.builder() + .id("wait") + .name("wait") + .type(OperationType.WAIT) + .subType(OperationSubType.WAIT.getValue()) + .status(OperationStatus.STARTED) + .waitDetails(WaitDetails.builder() + .scheduledEndTimestamp(Instant.now().plusSeconds(60)) + .build()) + .build(); + var input = new DurableExecutionInput( + "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/test/execution", + "token", + CheckpointUpdatedExecutionState.builder() + .operations(execution, resumedWait) + .build()); + var manager = new ExecutionManager(input, config, null); + var owner = new CompletableFuture(); + var outcome = manager.runUntilCompleteOrSuspend(owner); + try { + manager.registerActiveThread(null); + manager.setCurrentThreadContext(new ThreadContext(null, ThreadType.CONTEXT)); + var context = DurableContextImpl.createRootContext(manager, config, null); + var wait = new WaitOperation( + OperationIdentifier.of("wait", "wait", OperationSubType.WAIT), Duration.ofMinutes(1), context); + wait.execute(); + var checkpoint = manager.sendOperationUpdate(OperationUpdate.builder() + .id("step") + .name("step") + .type(OperationType.STEP) + .action(OperationAction.START) + .build()); + var poll = manager.pollForOperationUpdates("remote", Instant.now().plusSeconds(60)); + manager.recordHandlerScopeFatal(fatal); + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> checkpoint.get(1, TimeUnit.SECONDS)) + .getCause()); + assertSame( + fatal, + assertThrows(ExecutionException.class, () -> poll.get(1, TimeUnit.SECONDS)) + .getCause()); + assertSame( + fatal, + ExceptionHelper.unwrapAsyncFailure(assertThrows( + CompletionException.class, + () -> wait.getCompletionFuture().join()))); + var admitted = manager.tryStartCheckpointProcessing(); + if (admitted) manager.finishCheckpointProcessing(); + assertAll( + () -> assertFalse(admitted, "a published fatal must reject backend admission during scope cleanup"), + () -> assertDoesNotThrow(() -> manager.deregisterActiveThread(null)), + () -> assertFalse(outcome.isDone(), "a fatal must not become a suspension before owner cleanup")); + assertFalse(outcome.isDone(), "the root must retain ownership of its remaining scope cleanup"); + owner.completeExceptionally(fatal); + assertSame( + fatal, ExceptionHelper.unwrapAsyncFailure(assertThrows(CompletionException.class, outcome::join))); + verifyNoInteractions(client); + } finally { + assertSame( + fatal, ExceptionHelper.unwrapAsyncFailure(assertThrows(CompletionException.class, manager::close))); + } + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/RuntimeTraceHeaderTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/RuntimeTraceHeaderTest.java new file mode 100644 index 000000000..cf396a111 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/RuntimeTraceHeaderTest.java @@ -0,0 +1,256 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.execution; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import com.amazonaws.services.lambda.runtime.Context; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; +import javax.tools.ToolProvider; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class RuntimeTraceHeaderTest { + @Test + void missingAccessorOnOlderVisibleLambdaApiRetainsFallback(@TempDir Path directory) throws Exception { + var source = directory.resolve("Context.java"); + Files.writeString(source, "package com.amazonaws.services.lambda.runtime; public interface Context {} "); + var errors = new ByteArrayOutputStream(); + assertEquals( + 0, + ToolProvider.getSystemJavaCompiler() + .run(null, null, errors, "--release", "17", "-d", directory.toString(), source.toString()), + errors.toString(StandardCharsets.UTF_8)); + try (var loader = oldRuntimeLoader(directory)) { + var contextType = loader.loadClass(Context.class.getName()); + var context = Proxy.newProxyInstance(loader, new Class[] {contextType}, (proxy, method, args) -> { + throw new AssertionError("No accessor should be invoked"); + }); + var helper = loader.loadClass(RuntimeTraceHeader.class.getName()); + var capture = helper.getDeclaredMethod("capture", contextType); + capture.setAccessible(true); + assertNull(capture.invoke(null, context)); + } + } + + @Test + void preAccessorContextBinaryRetainsFallbackWithNewDefaultInterface(@TempDir Path directory) throws Exception { + var oldApi = Files.createDirectories(directory.resolve("old-api")); + var classes = Files.createDirectories(directory.resolve("classes")); + var apiSource = directory.resolve("Context.java"); + Files.writeString(apiSource, """ + package com.amazonaws.services.lambda.runtime; + public interface Context { + String getAwsRequestId(); String getLogGroupName(); String getLogStreamName(); + String getFunctionName(); String getFunctionVersion(); String getInvokedFunctionArn(); + CognitoIdentity getIdentity(); ClientContext getClientContext(); + int getRemainingTimeInMillis(); int getMemoryLimitInMB(); LambdaLogger getLogger(); + } + """); + var compiler = ToolProvider.getSystemJavaCompiler(); + var classpath = System.getProperty("java.class.path"); + var errors = new ByteArrayOutputStream(); + assertEquals( + 0, + compiler.run( + null, + null, + errors, + "--release", + "17", + "-cp", + classpath, + "-d", + oldApi.toString(), + apiSource.toString()), + errors.toString(StandardCharsets.UTF_8)); + var legacySource = directory.resolve("LegacyContext.java"); + Files.writeString(legacySource, """ + import com.amazonaws.services.lambda.runtime.*; + public class LegacyContext implements Context { + public String getAwsRequestId() { return "legacy"; } + public String getLogGroupName() { return null; } + public String getLogStreamName() { return null; } + public String getFunctionName() { return null; } + public String getFunctionVersion() { return null; } + public String getInvokedFunctionArn() { return null; } + public CognitoIdentity getIdentity() { return null; } + public ClientContext getClientContext() { return null; } + public int getRemainingTimeInMillis() { return 30000; } + public int getMemoryLimitInMB() { return 128; } + public LambdaLogger getLogger() { return null; } + } + """); + assertEquals( + 0, + compiler.run( + null, + null, + errors, + "--release", + "17", + "-cp", + oldApi + File.pathSeparator + classpath, + "-d", + classes.toString(), + legacySource.toString()), + errors.toString(StandardCharsets.UTF_8)); + var overrideSource = directory.resolve("ExplicitNullContext.java"); + Files.writeString(overrideSource, """ + public class ExplicitNullContext extends LegacyContext { + @Override public String getXrayTraceId() { return null; } + } + """); + var inheritedSource = directory.resolve("InheritedNullContext.java"); + Files.writeString(inheritedSource, "public class InheritedNullContext extends ExplicitNullContext {}"); + assertEquals( + 0, + compiler.run( + null, + null, + errors, + "--release", + "17", + "-cp", + classes + File.pathSeparator + classpath, + "-d", + classes.toString(), + overrideSource.toString(), + inheritedSource.toString()), + errors.toString(StandardCharsets.UTF_8)); + try (var loader = new URLClassLoader(new URL[] {classes.toUri().toURL()}, Context.class.getClassLoader())) { + var legacy = + (Context) loader.loadClass("LegacyContext").getConstructor().newInstance(); + assertEquals( + Context.class, legacy.getClass().getMethod("getXrayTraceId").getDeclaringClass()); + assertNull(legacy.getXrayTraceId(), "the new interface default returns null"); + assertNull(RuntimeTraceHeader.capture(legacy), "inherited interface default is not a runtime carrier"); + var explicit = (Context) + loader.loadClass("ExplicitNullContext").getConstructor().newInstance(); + assertEquals( + "", + RuntimeTraceHeader.capture(explicit), + "actual runtime override is authoritative even when null"); + var inherited = (Context) + loader.loadClass("InheritedNullContext").getConstructor().newInstance(); + assertEquals("", RuntimeTraceHeader.capture(inherited), "an inherited runtime override remains available"); + } + } + + @Test + void availableAccessorWithNoHeaderIsDistinctFromUnavailableApi() { + var context = mock(RuntimeContext.class); + when(context.getXrayTraceId()).thenReturn(null, ""); + assertEquals("", RuntimeTraceHeader.capture(context)); + assertEquals("", RuntimeTraceHeader.capture(context)); + assertNull(RuntimeTraceHeader.capture(null), "no runtime snapshot retains legacy fallback"); + } + + @Test + void runtimeAccessorFailuresAreAuthoritativeAbsence() { + var context = mock(RuntimeContext.class); + when(context.getXrayTraceId()) + .thenThrow(new SecurityException("access denied"), new IllegalStateException("carrier unavailable")); + assertEquals("", RuntimeTraceHeader.capture(context)); + assertEquals("", RuntimeTraceHeader.capture(context)); + } + + @Test + void fatalRuntimeFailuresStillPropagate() { + var context = mock(RuntimeContext.class); + var fatal = new OutOfMemoryError("simulated"); + when(context.getXrayTraceId()).thenThrow(fatal); + assertSame(fatal, assertThrows(OutOfMemoryError.class, () -> RuntimeTraceHeader.capture(context))); + } + + @ParameterizedTest + @MethodSource("nonfatalAccessorFailures") + void nonfatalErrorsFromAvailableOverrideAreAuthoritativeAbsence(Throwable failure) { + var context = mock(RuntimeContext.class); + when(context.getXrayTraceId()).thenAnswer(invocation -> { + throw failure; + }); + assertEquals("", RuntimeTraceHeader.capture(context)); + verify(context, times(1)).getXrayTraceId(); + } + + private static Stream nonfatalAccessorFailures() { + return Stream.of( + new AssertionError("optional accessor assertion"), + new NoClassDefFoundError("optional carrier dependency"), + new NoSuchMethodError("method inside the runtime override"), + new AbstractMethodError("implementation inside the runtime override"), + new ExceptionInInitializerError(new IllegalStateException("optional dependency init")), + new CompletionException(new AssertionError("wrapped optional assertion"))); + } + + @ParameterizedTest + @MethodSource("fatalAccessorFailures") + void directAndWrappedFatalAccessorFailuresRetainIdentity(Throwable failure, Error fatal) { + var context = mock(RuntimeContext.class); + when(context.getXrayTraceId()).thenAnswer(invocation -> { + throw failure; + }); + assertSame(fatal, assertThrows(Error.class, () -> RuntimeTraceHeader.capture(context))); + verify(context, times(1)).getXrayTraceId(); + } + + @SuppressWarnings("removal") + private static Stream fatalAccessorFailures() { + return Stream.of(new InternalError("simulated VM fatal"), new ThreadDeath()) + .flatMap(fatal -> Stream.of( + Arguments.of(fatal, fatal), + Arguments.of(new CompletionException(fatal), fatal), + Arguments.of(new ExecutionException(fatal), fatal), + Arguments.of(new InvocationTargetException(fatal), fatal), + Arguments.of(new UndeclaredThrowableException(fatal), fatal), + Arguments.of(new CompletionException(new InvocationTargetException(fatal)), fatal))); + } + + private abstract static class RuntimeContext implements Context { + @Override + public String getXrayTraceId() { + return null; + } + } + + private static URLClassLoader oldRuntimeLoader(Path directory) throws Exception { + byte[] helper; + try (var stream = RuntimeTraceHeader.class.getResourceAsStream("RuntimeTraceHeader.class")) { + helper = stream.readAllBytes(); + } + return new URLClassLoader(new URL[] {directory.toUri().toURL()}, RuntimeTraceHeader.class.getClassLoader()) { + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + if (!name.equals(Context.class.getName()) && !name.equals(RuntimeTraceHeader.class.getName())) { + return super.loadClass(name, resolve); + } + synchronized (getClassLoadingLock(name)) { + var type = findLoadedClass(name); + if (type == null) + type = name.equals(Context.class.getName()) + ? findClass(name) + : defineClass(name, helper, 0, helper.length); + if (resolve) resolveClass(type); + return type; + } + } + }; + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/BaseDurableOperationPluginTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/BaseDurableOperationPluginTest.java index 463a824ea..acae60678 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/BaseDurableOperationPluginTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/BaseDurableOperationPluginTest.java @@ -7,6 +7,7 @@ import static org.mockito.Mockito.when; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -27,6 +28,7 @@ import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.plugin.OperationInfo; /** @@ -55,7 +57,7 @@ void execute_firesOnOperationStart_withIsReplayTrue_forNonTerminalWait() { .build(); var executionManager = createExecutionManager(List.of(waitOp), plugin); - var durableContext = mockDurableContext(executionManager, plugin); + var durableContext = mockDurableContext(executionManager); var operation = new WaitOperation( OperationIdentifier.of(OPERATION_ID, OPERATION_NAME, OperationSubType.WAIT), @@ -87,7 +89,7 @@ void execute_doesNotFireOnOperationStart_forTerminalOperation(OperationStatus te .build(); var executionManager = createExecutionManager(List.of(waitOp), plugin); - var durableContext = mockDurableContext(executionManager, plugin); + var durableContext = mockDurableContext(executionManager); var operation = new WaitOperation( OperationIdentifier.of(OPERATION_ID, OPERATION_NAME, OperationSubType.WAIT), @@ -107,7 +109,7 @@ void execute_firesOnOperationStart_withIsReplayFalse_forFirstExecution() { var plugin = new RecordingPlugin(); // No existing operations — first execution var executionManager = createExecutionManager(List.of(), plugin); - var durableContext = mockDurableContext(executionManager, plugin); + var durableContext = mockDurableContext(executionManager); var operation = new WaitOperation( OperationIdentifier.of(OPERATION_ID, OPERATION_NAME, OperationSubType.WAIT), @@ -125,6 +127,10 @@ void execute_firesOnOperationStart_withIsReplayFalse_forFirstExecution() { // ─── Helpers ───────────────────────────────────────────────────────── + /** + * Builds the per-invocation ExecutionManager and starts its invocation, which is what materializes the plugin + * instance the operation hooks are then dispatched to. + */ private ExecutionManager createExecutionManager(List additionalOps, RecordingPlugin plugin) { var client = TestUtils.createMockClient(); var operations = new ArrayList(); @@ -138,19 +144,20 @@ private ExecutionManager createExecutionManager(List additionalOps, R CheckpointUpdatedExecutionState.builder().operations(operations).build(); var config = DurableConfig.builder() .withDurableExecutionClient(client) - .withPlugins(plugin) + .withPlugins(info -> plugin) .build(); var executionManager = new ExecutionManager( new DurableExecutionInput(EXECUTION_ARN, "test-token", initialState), config, null); executionManager.setCurrentThreadContext(new ThreadContext("Root", ThreadType.CONTEXT)); + executionManager + .getPluginRunner() + .onInvocationStart(new InvocationInfo("req-1", EXECUTION_ARN, true, Instant.now())); return executionManager; } - private DurableContextImpl mockDurableContext(ExecutionManager executionManager, RecordingPlugin plugin) { + private DurableContextImpl mockDurableContext(ExecutionManager executionManager) { var durableContext = mock(DurableContextImpl.class); when(durableContext.getExecutionManager()).thenReturn(executionManager); - when(durableContext.getDurableConfig()) - .thenReturn(DurableConfig.builder().withPlugins(plugin).build()); return durableContext; } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/plugin/HandlerScopeTest.java b/sdk/src/test/java/software/amazon/lambda/durable/plugin/HandlerScopeTest.java new file mode 100644 index 000000000..4e12b22b4 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/plugin/HandlerScopeTest.java @@ -0,0 +1,253 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.plugin; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.function.Supplier; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.execution.SuspendExecutionException; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class HandlerScopeTest { + @ParameterizedTest + @ValueSource(strings = {"success", "failure", "suspension"}) + void scopesCloseInReverseOrderOnTheHandlerThread(String outcome) { + var calls = new ArrayList(); + var owner = Thread.currentThread(); + var runner = startedRunner(List.of(scope("a", calls, owner), scope("b", calls, owner))); + Throwable error = + outcome.equals("suspension") ? new SuspendExecutionException() : new IllegalStateException("user"); + if (outcome.equals("success")) { + assertEquals("ok", runner.runHandler(() -> { + calls.add("handler"); + return "ok"; + })); + } else { + assertSame( + error, + assertThrows( + Throwable.class, + () -> runner.runHandler(() -> { + calls.add("handler"); + ExceptionHelper.sneakyThrow(error); + return null; + }))); + } + assertEquals(List.of("open-a", "open-b", "handler", "close-b", "close-a"), calls); + } + + @Test + void ordinarySetupAndCleanupFailuresDoNotReplaceTheHandlerResult() { + var calls = new ArrayList(); + var brokenSetup = new ScopedPlugin() { + @Override + public AutoCloseable openHandlerScope() { + throw new IllegalStateException("setup"); + } + }; + var brokenClose = new ScopedPlugin() { + @Override + public AutoCloseable openHandlerScope() { + return () -> { + throw new IllegalStateException("cleanup"); + }; + } + }; + var runner = startedRunner(List.of(scope("first", calls, Thread.currentThread()), brokenSetup, brokenClose)); + assertEquals("ok", runner.runHandler(() -> "ok")); + assertEquals(List.of("open-first", "close-first"), calls); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void linkageFailuresPreserveTheBodyOutcomeAndAllEarlierScopes(boolean bodyFails) { + var active = new ThreadLocal(); + var calls = new ArrayList(); + var healthy = new ScopedPlugin() { + @Override + public AutoCloseable openHandlerScope() { + calls.add("open-healthy"); + active.set("healthy"); + return () -> { + calls.add("close-healthy"); + active.remove(); + }; + } + }; + var brokenOpen = new ScopedPlugin() { + @Override + public AutoCloseable openHandlerScope() { + calls.add("open-broken"); + throw new NoSuchMethodError("optional API missing"); + } + }; + var brokenClose = new ScopedPlugin() { + @Override + public AutoCloseable openHandlerScope() { + calls.add("open-last"); + return () -> { + calls.add("close-last"); + throw new NoClassDefFoundError("optional class missing"); + }; + } + }; + var runner = startedRunner(List.of(healthy, brokenOpen, brokenClose)); + var bodyFailure = new IllegalStateException("original body error"); + Supplier body = () -> { + calls.add("body"); + assertEquals("healthy", active.get()); + if (bodyFails) throw bodyFailure; + return "ok"; + }; + try { + if (bodyFails) + assertSame(bodyFailure, assertThrows(IllegalStateException.class, () -> runner.runHandler(body))); + else assertEquals("ok", runner.runHandler(body)); + assertNull(active.get(), "a later linkage failure must not prevent earlier context cleanup"); + assertEquals( + List.of("open-healthy", "open-broken", "open-last", "body", "close-last", "close-healthy"), calls); + } finally { + active.remove(); + } + } + + @SuppressWarnings("removal") + @ParameterizedTest + @CsvSource({"true,false", "false,false", "true,true", "false,true"}) + void fatalScopeFailuresStillPropagate(boolean duringOpen, boolean wrapped) { + for (Error fatal : List.of(new InternalError("fatal VM failure"), new ThreadDeath())) { + var plugin = new ScopedPlugin() { + @Override + public AutoCloseable openHandlerScope() { + if (duringOpen) { + if (wrapped) + CompletableFuture.failedFuture(new ExecutionException(fatal)) + .join(); + throw fatal; + } + return () -> { + if (wrapped) + CompletableFuture.failedFuture(new ExecutionException(fatal)) + .join(); + throw fatal; + }; + } + }; + var reported = new AtomicReference(); + var calls = new ArrayList(); + var active = new ThreadLocal(); + var owner = Thread.currentThread(); + var healthy = new ScopedPlugin() { + @Override + public AutoCloseable openHandlerScope() { + active.set("healthy"); + calls.add("open-healthy"); + return () -> { + assertSame(owner, Thread.currentThread()); + active.remove(); + calls.add("close-healthy"); + }; + } + }; + var runner = startedRunner(List.of(healthy, plugin)); + assertSame(fatal, assertThrows(Error.class, () -> runner.runHandler(() -> "ok", () -> {}, reported::set))); + assertSame(fatal, reported.get(), "only a scope callback reports this fatal"); + assertNull(active.get(), "earlier context must not leak into reuse of the owner thread"); + assertEquals(List.of("open-healthy", "close-healthy"), calls); + } + } + + @Test + void openerConstructionLinkageFailurePreservesBodyAndEarlierCleanup() { + var calls = new ArrayList(); + var runner = startedRunner(List.of(scope("healthy", calls, Thread.currentThread()), new BrokenOpenerPlugin())); + assertEquals("body", runner.runHandler(() -> "body")); + assertEquals(List.of("open-healthy", "close-healthy"), calls); + } + + @Test + void openerConstructionFatalRetainsIdentityAndEarlierCleanup() { + var calls = new ArrayList(); + var reported = new AtomicReference(); + var runner = startedRunner(List.of(scope("healthy", calls, Thread.currentThread()), new FatalOpenerPlugin())); + assertSame( + FatalOpener.FAILURE, + assertThrows( + InternalError.class, + () -> runner.runHandler(() -> fail("handler must not run"), () -> {}, reported::set))); + assertSame(FatalOpener.FAILURE, reported.get()); + assertEquals(List.of("open-healthy", "close-healthy"), calls); + } + + @HandlerScoped(BrokenOpener.class) + private static class BrokenOpenerPlugin implements DurableExecutionPlugin {} + + public static class BrokenOpener implements Function { + public BrokenOpener() { + throw new NoSuchMethodError("opener dependency"); + } + + public AutoCloseable apply(BrokenOpenerPlugin plugin) { + return null; + } + } + + @HandlerScoped(FatalOpener.class) + private static class FatalOpenerPlugin implements DurableExecutionPlugin {} + + public static class FatalOpener implements Function { + static final InternalError FAILURE = new InternalError("opener fatal"); + + public FatalOpener() { + throw FAILURE; + } + + public AutoCloseable apply(FatalOpenerPlugin plugin) { + return null; + } + } + + private static DurableExecutionPlugin scope(String name, List calls, Thread owner) { + return new ScopedPlugin() { + @Override + public AutoCloseable openHandlerScope() { + assertSame(owner, Thread.currentThread()); + calls.add("open-" + name); + return () -> { + assertSame(owner, Thread.currentThread()); + calls.add("close-" + name); + }; + } + }; + } + + private static PluginRunner startedRunner(List plugins) { + var runner = new PluginRunner(plugins.stream() + .map(plugin -> (DurableExecutionPluginFactory) info -> plugin) + .toList()); + runner.onInvocationStart(new InvocationInfo("request", "arn", true, Instant.EPOCH)); + return runner; + } + + @HandlerScoped(ScopedPlugin.Opener.class) + private abstract static class ScopedPlugin implements DurableExecutionPlugin { + public abstract AutoCloseable openHandlerScope(); + + public static class Opener implements Function { + public AutoCloseable apply(ScopedPlugin plugin) { + return plugin.openHandlerScope(); + } + } + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/plugin/InvocationInfoCompatibilityTest.java b/sdk/src/test/java/software/amazon/lambda/durable/plugin/InvocationInfoCompatibilityTest.java new file mode 100644 index 000000000..bec6c2dfc --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/plugin/InvocationInfoCompatibilityTest.java @@ -0,0 +1,189 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.plugin; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import javax.tools.ToolProvider; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; +import org.junit.jupiter.api.io.TempDir; + +class InvocationInfoCompatibilityTest { + // The released public constructor/accessor/record shape, compiled separately from the current SDK. + private static final String LEGACY_API = """ + package software.amazon.lambda.durable.plugin; + import java.time.Instant; + import java.util.Map; + public record InvocationInfo(String requestId, String durableExecutionArn, boolean isFirstInvocation, + Instant executionStartTime, Object executionInput, Map operations, + Map updatedOperations) { + public InvocationInfo(String request, String arn, boolean first, Instant start) { + this(request,arn,first,start,null,Map.of(),Map.of()); + } + public InvocationInfo(String request, String arn, boolean first, Instant start, Object input) { + this(request,arn,first,start,input,Map.of(),Map.of()); + } + public InvocationInfo(String request, String arn, boolean first, Instant start, + Map operations, Map updated) { + this(request,arn,first,start,null,operations,updated); + } + } + """; + private static final String CONSTRUCTOR_CALLER = """ + import software.amazon.lambda.durable.plugin.InvocationInfo; + import java.time.Instant; + import java.util.Map; + import java.util.List; + public class LegacyConstructorProbe { + public static List construct() { + var four = new InvocationInfo("four","arn",true,Instant.EPOCH); + var five = new InvocationInfo("five","arn",true,Instant.EPOCH,"input"); + var six = new InvocationInfo("six","arn",true,Instant.EPOCH,Map.of(),Map.of()); + var seven = new InvocationInfo("seven","arn",true,Instant.EPOCH,"input",Map.of(),Map.of()); + Record record = seven; + if (!"arn".equals(seven.durableExecutionArn()) || !seven.isFirstInvocation() + || !Instant.EPOCH.equals(seven.executionStartTime()) || !"input".equals(seven.executionInput()) + || !six.operations().isEmpty() || !seven.updatedOperations().isEmpty() + || !record.getClass().isRecord()) throw new AssertionError(); + return List.of(four,five,six,seven); + } + } + """; + private static final String LEGACY_PATTERN = """ + import software.amazon.lambda.durable.plugin.InvocationInfo; + public class RecordPatternProbe { + public static String read(Object event) { + if (event instanceof InvocationInfo(var request, var arn, var first, var start, + var input, var operations, var updated)) return request; + return null; + } + } + """; + + @Test + void oldConstructorCallsRetainSourceAndBinaryCompatibility(@TempDir Path directory) throws Exception { + var oldApi = legacyApi(directory); + var oldCaller = directory.resolve("old-caller"); + assertCompiled(compile( + oldCaller, + "LegacyConstructorProbe", + CONSTRUCTOR_CALLER, + oldApi + File.pathSeparator + classpath(), + "17")); + var newCaller = directory.resolve("new-caller"); + assertCompiled(compile(newCaller, "LegacyConstructorProbe", CONSTRUCTOR_CALLER, classpath(), "17")); + for (var caller : List.of(oldCaller, newCaller)) { + try (var loader = loader(caller)) { + var result = (List) loader.loadClass("LegacyConstructorProbe") + .getMethod("construct") + .invoke(null); + assertEquals(4, result.size()); + for (var value : result) { + var info = assertInstanceOf(InvocationInfo.class, value); + assertNull( + info.xRayTraceId(), "All retained constructors default the optional field to unavailable"); + } + } + } + } + + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void eightComponentPatternReadsTheActualHeaderField(@TempDir Path directory) throws Exception { + var source = LEGACY_PATTERN.replace("var updated)) return request", "var updated, var header)) return header"); + assertCompiled(compile(directory, "RecordPatternProbe", source, classpath(), "21")); + try (var loader = loader(directory)) { + assertEquals( + "runtime-header", + loader.loadClass("RecordPatternProbe") + .getMethod("read", Object.class) + .invoke(null, invocation())); + } + var components = InvocationInfo.class.getRecordComponents(); + assertEquals(8, components.length); + assertEquals("xRayTraceId", components[7].getName()); + assertEquals(String.class, components[7].getType()); + assertFalse(invocation().toString().contains("runtime-header")); + } + + @Test + @EnabledForJreRange(min = JRE.JAVA_21) + void sevenComponentPatternHasAnExplicitSourceBoundaryButOldBytecodeStillRuns(@TempDir Path directory) + throws Exception { + var oldApi = legacyApi(directory); + var oldCaller = directory.resolve("old-pattern"); + assertCompiled(compile( + oldCaller, "RecordPatternProbe", LEGACY_PATTERN, oldApi + File.pathSeparator + classpath(), "21")); + var recompiled = + compile(directory.resolve("new-pattern"), "RecordPatternProbe", LEGACY_PATTERN, classpath(), "21"); + assertNotEquals(0, recompiled.exitCode(), "An eighth component cannot preserve seven-component pattern source"); + assertTrue(recompiled.errors().contains("incorrect number of nested patterns"), recompiled.errors()); + try (var loader = loader(oldCaller)) { + assertEquals( + "request", + loader.loadClass("RecordPatternProbe") + .getMethod("read", Object.class) + .invoke(null, invocation())); + } + } + + private static InvocationInfo invocation() { + return new InvocationInfo("request", "arn", true, Instant.EPOCH, "input", Map.of(), Map.of(), "runtime-header"); + } + + private static Path legacyApi(Path directory) throws Exception { + var output = directory.resolve("old-api"); + assertCompiled(compile(output, "InvocationInfo", LEGACY_API, classpath(), "17")); + return output; + } + + private static String classpath() { + return System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")); + } + + private URLClassLoader loader(Path directory) throws Exception { + // The consumer is loaded from disk; InvocationInfo resolves to the current SDK, never the old fixture. + return new URLClassLoader( + new URL[] {directory.toUri().toURL()}, getClass().getClassLoader()); + } + + private static Compilation compile(Path output, String name, String text, String classpath, String release) + throws Exception { + var source = output.resolve("src").resolve(name + ".java"); + Files.createDirectories(source.getParent()); + Files.writeString(source, text); + var errors = new ByteArrayOutputStream(); + var compiler = ToolProvider.getSystemJavaCompiler(); + assertNotNull(compiler, "Compatibility tests require a JDK"); + var status = compiler.run( + null, + null, + errors, + "--release", + release, + "-classpath", + classpath, + "-d", + output.toString(), + source.toString()); + return new Compilation(status, errors.toString(StandardCharsets.UTF_8)); + } + + private static void assertCompiled(Compilation result) { + assertEquals(0, result.exitCode(), result.errors()); + } + + private record Compilation(int exitCode, String errors) {} +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/plugin/LegacyHandlerScopeCompatibilityTest.java b/sdk/src/test/java/software/amazon/lambda/durable/plugin/LegacyHandlerScopeCompatibilityTest.java new file mode 100644 index 000000000..ea8f44533 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/plugin/LegacyHandlerScopeCompatibilityTest.java @@ -0,0 +1,87 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.plugin; + +import static org.junit.jupiter.api.Assertions.*; + +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import javax.tools.ToolProvider; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class LegacyHandlerScopeCompatibilityTest { + @TempDir + Path directory; + + @Test + void precompiledUnrelatedDefaultMethodStillLinksAndConfigures() throws Exception { + var baseline = directory.resolve("baseline"); + var classes = directory.resolve("classes"); + Files.createDirectories(baseline); + Files.createDirectories(classes); + var oldApi = directory.resolve("DurableExecutionPlugin.java"); + Files.writeString(oldApi, """ + package software.amazon.lambda.durable.plugin; + public interface DurableExecutionPlugin {} + """); + var compiler = ToolProvider.getSystemJavaCompiler(); + assertEquals( + 0, compiler.run(null, null, null, "--release", "17", "-d", baseline.toString(), oldApi.toString())); + var source = directory.resolve("LegacyPlugin.java"); + Files.writeString(source, """ + import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; + interface ExistingCapability { + default AutoCloseable openHandlerScope() { LegacyPlugin.opened++; return () -> {}; } + } + public class LegacyPlugin implements DurableExecutionPlugin, ExistingCapability { + public static int opened; + public String originalCall() throws Exception { + try (var scope = openHandlerScope()) { return "legacy application value"; } + } + } + """); + assertEquals( + 0, + compiler.run( + null, + null, + null, + "--release", + "17", + "-cp", + baseline.toString(), + "-d", + classes.toString(), + source.toString())); + try (var loader = new URLClassLoader( + new URL[] {classes.toUri().toURL()}, getClass().getClassLoader())) { + var type = loader.loadClass("LegacyPlugin"); + var plugin = (DurableExecutionPlugin) type.getConstructor().newInstance(); + var runner = new PluginRunner(List.of(info -> plugin)); + runner.onInvocationStart(new InvocationInfo("test", "arn", true, Instant.EPOCH)); + assertEquals("body", runner.runHandler(() -> "body")); + assertEquals(0, type.getField("opened").get(null), "unannotated application capability is not SDK opt-in"); + assertEquals( + "legacy application value", type.getMethod("originalCall").invoke(plugin)); + } + // Source compatibility matters too: the same downstream source compiles against this core. + assertEquals( + 0, + compiler.run( + null, + null, + null, + "--release", + "17", + "-cp", + System.getProperty("java.class.path"), + "-d", + classes.toString(), + source.toString())); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginRunnerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginRunnerTest.java index dc21c9e8b..ac7812030 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginRunnerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginRunnerTest.java @@ -8,10 +8,58 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.ServiceConfigurationError; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; class PluginRunnerTest { + @Test + void invocationSnapshotReachesExistingHookOnce() { + var calls = new ArrayList(); + var legacy = new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo info) { + calls.add(info); + } + }; + var info = invocationInfo(); + new PluginRunner(List.of(ignored -> legacy)).onInvocationStart(info); + assertEquals(List.of(info), calls); + } + + @Test + @SuppressWarnings("removal") + void invocationEndAttemptsEveryPluginAndPreservesFirstFatal() { + var first = new InternalError("first fatal"); + var calls = new ArrayList(); + var reported = new AtomicReference(); + var runner = new PluginRunner( + List.of( + endingPlugin("first", calls, first), + endingPlugin("linkage", calls, new NoSuchMethodError("optional API")), + endingPlugin("second", calls, new ThreadDeath()), + endingPlugin("healthy", calls, null)), + fatal -> { + calls.add("reported"); + reported.set(fatal); + }); + runner.onInvocationStart(invocationInfo()); + assertSame(first, assertThrows(InternalError.class, () -> runner.onInvocationEnd(invocationEndInfo()))); + assertEquals(List.of("first", "reported", "linkage", "second", "healthy"), calls); + assertSame(first, reported.get()); + } + + private static DurableExecutionPluginFactory endingPlugin(String name, List calls, Error failure) { + return info -> new DurableExecutionPlugin() { + public void onInvocationEnd(InvocationEndInfo end) { + calls.add(name); + if (failure != null) throw failure; + } + }; + } + // ─── No-op / empty behavior ────────────────────────────────────────── @Test @@ -24,7 +72,7 @@ void noOpRunner_doesNothing() { } @Test - void emptyPluginList_behavesAsNoOp() { + void emptyFactoryList_behavesAsNoOp() { var runner = new PluginRunner(List.of()); assertTrue(runner.isEmpty()); @@ -32,21 +80,485 @@ void emptyPluginList_behavesAsNoOp() { } @Test - void nullPluginList_behavesAsNoOp() { + void nullFactoryList_behavesAsNoOp() { var runner = new PluginRunner(null); assertTrue(runner.isEmpty()); assertDoesNotThrow(() -> runner.onOperationStart(operationInfo())); } + // ─── Per-invocation lifetime ───────────────────────────────────────── + + @Test + void invocationStart_createsOnePluginPerFactory_andPassesTheHookInfo() { + var calls = new ArrayList(); + var receivedByFactory = new ArrayList(); + var receivedByHook = new ArrayList(); + var runner = new PluginRunner(List.of(info -> { + receivedByFactory.add(info); + return new TestPlugin("p1", calls) { + @Override + public void onInvocationStart(InvocationInfo hookInfo) { + receivedByHook.add(hookInfo); + super.onInvocationStart(hookInfo); + } + }; + })); + var info = invocationInfo(); + + runner.onInvocationStart(info); + + assertEquals(List.of("p1:onInvocationStart"), calls); + assertEquals(1, receivedByFactory.size()); + assertSame(info, receivedByFactory.get(0), "the factory must receive this invocation's info"); + assertSame(info, receivedByHook.get(0), "the first hook must receive the same info instance"); + } + + @Test + void startupFollowsRegistrationOrder_andLaterFactorySeesEarlierHookState() { + // Factories and hooks share the handler thread; earlier startup state must be available to later factories. + var order = new ArrayList(); + var seenByLaterConstructor = new ArrayList(); + var installed = new ThreadLocal(); + + DurableExecutionPluginFactory first = info -> { + order.add("construct:first"); + return new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo hookInfo) { + order.add("start:first"); + installed.set("from-first-start-hook"); + } + }; + }; + DurableExecutionPluginFactory second = info -> { + order.add("construct:second"); + seenByLaterConstructor.add(String.valueOf(installed.get())); + return new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo hookInfo) { + order.add("start:second"); + } + }; + }; + + try { + new PluginRunner(List.of(first, second)).onInvocationStart(invocationInfo()); + } finally { + installed.remove(); + } + + assertEquals(List.of("construct:first", "start:first", "construct:second", "start:second"), order); + assertEquals(List.of("from-first-start-hook"), seenByLaterConstructor); + } + + @Test + void factoriesAreCalledOncePerInvocation_notPerHook() { + var creations = new AtomicInteger(); + var calls = new ArrayList(); + var runner = new PluginRunner(List.of(info -> { + creations.incrementAndGet(); + return new TestPlugin("p", calls); + })); + + runner.onInvocationStart(invocationInfo()); + runner.onOperationStart(operationInfo()); + runner.onOperationEnd(operationEndInfo()); + runner.onInvocationEnd(invocationEndInfo()); + + assertEquals(1, creations.get()); + assertEquals( + List.of("p:onInvocationStart", "p:onOperationStart", "p:onOperationEnd", "p:onInvocationEnd"), calls); + } + + @Test + void eachInvocationGetsItsOwnPluginInstance() { + var instances = new ArrayList(); + DurableExecutionPluginFactory factory = info -> { + var plugin = new TestPlugin("p", new ArrayList<>()); + instances.add(plugin); + return plugin; + }; + + // One runner per invocation, as the SDK creates one per ExecutionManager. + new PluginRunner(List.of(factory)).onInvocationStart(invocationInfo()); + new PluginRunner(List.of(factory)).onInvocationStart(invocationInfo()); + + assertEquals(2, instances.size()); + assertNotSame(instances.get(0), instances.get(1), "invocations must not share a plugin instance"); + } + + @Test + void hooksBeforeInvocationStart_dispatchToNothing() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of(info -> new TestPlugin("p", calls))); + + // Plugins only exist between onInvocationStart and the end of the invocation. + runner.onOperationStart(operationInfo()); + + assertTrue(calls.isEmpty()); + } + + @Test + void releasePlugins_dropsThisInvocationsInstances() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of(info -> new TestPlugin("p", calls))); + runner.onInvocationStart(invocationInfo()); + calls.clear(); + + runner.releasePlugins(); + runner.onOperationStart(operationInfo()); + runner.onInvocationEnd(invocationEndInfo()); + + assertTrue(calls.isEmpty(), "released plugin instances must not receive further hooks"); + } + + @Test + void factoryList_isCopiedAtConstruction() { + var calls = new ArrayList(); + var mutableList = new ArrayList(); + mutableList.add(info -> new TestPlugin("p1", calls)); + var runner = new PluginRunner(mutableList); + + // Modifying the original list should not affect the runner + mutableList.add(info -> new TestPlugin("p2", calls)); + + runner.onInvocationStart(invocationInfo()); + + // Only p1 should be called — p2 was added after construction + assertEquals(List.of("p1:onInvocationStart"), calls); + } + + // ─── Factory error isolation ───────────────────────────────────────── + + @Test + void throwingFactory_isContained_andRemainingPluginsStillRun() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of( + info -> { + throw new RuntimeException("boom"); + }, + info -> new TestPlugin("p2", calls))); + + assertDoesNotThrow(() -> runner.onInvocationStart(invocationInfo())); + runner.onInvocationEnd(invocationEndInfo()); + + assertEquals(List.of("p2:onInvocationStart", "p2:onInvocationEnd"), calls); + } + + @Test + void nullReturningFactory_isContained_andRemainingPluginsStillRun() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of(info -> null, info -> new TestPlugin("p2", calls))); + + assertDoesNotThrow(() -> runner.onInvocationStart(invocationInfo())); + runner.onOperationStart(operationInfo()); + + assertEquals(List.of("p2:onInvocationStart", "p2:onOperationStart"), calls); + } + + // ─── Factory and hook linkage failures ─────────────────────────────── + // + // A LinkageError is an Error, not an Exception, so a catch of Exception does not contain it. Both of the shapes + // below are reachable through the plugin contract rather than hypothetical: a provider JAR compiled against an + // earlier version of DurableExecutionPluginProvider throws AbstractMethodError the first time the SDK invokes the + // method it does not implement, and a provider whose optional dependency is absent from the deployment package + // throws NoClassDefFoundError when it first touches that class. Both must be contained, because the contract says a + // factory or hook failure is logged and skipped and never disrupts the execution. + + @Test + void factoryThrowingAbstractMethodError_isContained_andRemainingPluginsStillRun() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of( + info -> { + // What a provider compiled against the previous interface throws when the new factory method is + // invoked on it. + throw new AbstractMethodError( + "software.amazon.example.LegacyProvider.createPlugin(InvocationInfo)"); + }, + info -> new TestPlugin("p2", calls))); + + assertDoesNotThrow(() -> runner.onInvocationStart(invocationInfo())); + runner.onInvocationEnd(invocationEndInfo()); + + assertEquals(List.of("p2:onInvocationStart", "p2:onInvocationEnd"), calls); + } + + @Test + void factoryThrowingNoClassDefFoundError_isContained_andRemainingPluginsStillRun() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of( + info -> { + // What a provider with a missing optional dependency throws while building its plugin. + throw new NoClassDefFoundError("software/amazon/example/OptionalExporter"); + }, + info -> new TestPlugin("p2", calls))); + + assertDoesNotThrow(() -> runner.onInvocationStart(invocationInfo())); + runner.onInvocationEnd(invocationEndInfo()); + + assertEquals(List.of("p2:onInvocationStart", "p2:onInvocationEnd"), calls); + } + + @Test + void hookThrowingLinkageError_isContained_andRemainingPluginsStillRun() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of( + info -> new LinkageErrorPlugin(), + info -> new TestPlugin("p2", calls), + info -> new TestPlugin("p3", calls))); + + assertDoesNotThrow(() -> runner.onInvocationStart(invocationInfo())); + calls.clear(); + assertDoesNotThrow(() -> runner.onOperationStart(operationInfo())); + assertDoesNotThrow(() -> runner.onInvocationEnd(invocationEndInfo())); + + assertEquals( + List.of("p2:onOperationStart", "p3:onOperationStart", "p2:onInvocationEnd", "p3:onInvocationEnd"), + calls); + } + + // ─── Factory and hook throwables that are neither Exception nor LinkageError ── + // + // AssertionError and ServiceConfigurationError extend Error and Error respectively, and neither is a LinkageError, + // so a catch of `Exception | LinkageError` lets both escape. Escaping the plugin boundary fails the invocation the + // plugin was only observing. The contract says a factory or hook failure is logged and skipped and never disrupts + // the execution, so both must be contained. Both shapes are reachable through the plugin contract: a plugin that + // ships with assertions enabled, or that calls a library which asserts internally, throws AssertionError, and a + // plugin that runs its own ServiceLoader over its exporter back ends throws ServiceConfigurationError when one of + // them is misdeclared. + + @Test + void factoryThrowingAssertionError_isContained_andRemainingPluginsStillRun() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of( + info -> { + throw new AssertionError("plugin invariant violated"); + }, + info -> new TestPlugin("p2", calls))); + + assertDoesNotThrow(() -> runner.onInvocationStart(invocationInfo())); + runner.onInvocationEnd(invocationEndInfo()); + + assertEquals(List.of("p2:onInvocationStart", "p2:onInvocationEnd"), calls); + } + + @Test + void factoryThrowingServiceConfigurationError_isContained_andRemainingPluginsStillRun() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of( + info -> { + throw new ServiceConfigurationError("software.amazon.example.Exporter: provider not found"); + }, + info -> new TestPlugin("p2", calls))); + + assertDoesNotThrow(() -> runner.onInvocationStart(invocationInfo())); + runner.onInvocationEnd(invocationEndInfo()); + + assertEquals(List.of("p2:onInvocationStart", "p2:onInvocationEnd"), calls); + } + + @Test + void hookThrowingAssertionError_isContained_andRemainingPluginsStillRun() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of( + info -> new AssertionErrorPlugin(), + info -> new TestPlugin("p2", calls), + info -> new TestPlugin("p3", calls))); + + assertDoesNotThrow(() -> runner.onInvocationStart(invocationInfo())); + calls.clear(); + assertDoesNotThrow(() -> runner.onOperationStart(operationInfo())); + assertDoesNotThrow(() -> runner.onInvocationEnd(invocationEndInfo())); + + assertEquals( + List.of("p2:onOperationStart", "p3:onOperationStart", "p2:onInvocationEnd", "p3:onInvocationEnd"), + calls); + } + + @Test + void hookThrowingServiceConfigurationError_isContained_andRemainingPluginsStillRun() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of( + info -> new ServiceConfigurationErrorPlugin(), + info -> new TestPlugin("p2", calls), + info -> new TestPlugin("p3", calls))); + + assertDoesNotThrow(() -> runner.onInvocationStart(invocationInfo())); + calls.clear(); + assertDoesNotThrow(() -> runner.onInvocationEnd(invocationEndInfo())); + + assertEquals(List.of("p2:onInvocationEnd", "p3:onInvocationEnd"), calls); + } + + // ─── Interrupts ────────────────────────────────────────────────────── + // + // An InterruptedException from plugin code is contained like any other non-fatal throwable, and the interrupt + // status is left alone. onInvocationStart runs on the handler thread, so setting the flag there would leave the + // handler's next blocking call to fail with an interrupt no user code asked for. A thrown InterruptedException is + // also no proof of interruption: no hook and no factory method declares a checked exception, so plugin code reaches + // the boundary with one only by rethrowing it undeclared, and it can construct one with the status clear. The tests + // below use that shape deliberately. + + @Test + void factoryThrowingInterruptedException_isContained_andLeavesTheThreadUninterrupted() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of( + info -> { + sneakyThrow(new InterruptedException("flush interrupted")); + return null; + }, + info -> new TestPlugin("p2", calls))); + + try { + assertDoesNotThrow(() -> runner.onInvocationStart(invocationInfo())); + + assertFalse( + Thread.currentThread().isInterrupted(), + "containment must not interrupt the thread that runs the handler"); + assertEquals(List.of("p2:onInvocationStart"), calls); + } finally { + // Clear the status so a failure here does not leak into whatever else runs on this thread. + Thread.interrupted(); + } + } + + @Test + void hookThrowingInterruptedException_isContained_andLeavesTheThreadUninterrupted() { + var calls = new ArrayList(); + var runner = new PluginRunner(List.of(info -> new InterruptingPlugin(), info -> new TestPlugin("p2", calls))); + runner.onInvocationStart(invocationInfo()); + calls.clear(); + + try { + assertDoesNotThrow(() -> runner.onInvocationEnd(invocationEndInfo())); + + assertFalse( + Thread.currentThread().isInterrupted(), + "containment must not interrupt the thread that runs the handler"); + assertEquals(List.of("p2:onInvocationEnd"), calls, "remaining plugins must still be called"); + } finally { + Thread.interrupted(); + } + } + + @Test + void containmentPreservesAnInterruptTheThreadAlreadyCarried() { + // The boundary neither sets nor clears the flag: a thread that was already interrupted before it entered plugin + // code still carries the interrupt when containment returns. + var calls = new ArrayList(); + var runner = new PluginRunner(List.of(info -> new ThrowingPlugin(), info -> new TestPlugin("p2", calls))); + runner.onInvocationStart(invocationInfo()); + calls.clear(); + + try { + Thread.currentThread().interrupt(); + + assertDoesNotThrow(() -> runner.onInvocationEnd(invocationEndInfo())); + + assertTrue(Thread.currentThread().isInterrupted(), "an interrupt the thread already carried must survive"); + assertEquals(List.of("p2:onInvocationEnd"), calls); + } finally { + Thread.interrupted(); + } + } + + @Test + void aFatalFactoryFailure_stillPublishesTheInstancesAlreadyBuilt() { + // A plugin constructor is where both OTel plugins bind their tracer and start the Invocation span, so an + // instance built before a fatal failure already owns spans that only onInvocationEnd ends and flushes. + // Publishing after the loop meant a VirtualMachineError from a later factory left the runner looking empty, + // and the end hook the failure path fires reached nothing. + var calls = new ArrayList(); + var runner = new PluginRunner(List.of(info -> new TestPlugin("p1", calls), info -> { + throw new OutOfMemoryError("fatal factory"); + })); + + assertThrows(OutOfMemoryError.class, () -> runner.onInvocationStart(invocationInfo())); + + // The earlier start hook ran before the later fatal factory failure; its end hook + // must still reach it. + assertEquals(List.of("p1:onInvocationStart"), calls); + runner.onInvocationEnd(invocationEndInfo()); + assertEquals( + List.of("p1:onInvocationStart", "p1:onInvocationEnd"), + calls, + "an instance already built must still be finalized"); + } + + @Test + void factoryThrowingAJvmError_stillPropagates() { + // The containment is deliberately narrow: an Error that says the JVM itself is failing must not be swallowed as + // if it were a plugin defect, because the process cannot be assumed able to continue. + var runner = new PluginRunner(List.of(info -> { + throw new OutOfMemoryError("Java heap space"); + })); + + assertThrows(OutOfMemoryError.class, () -> runner.onInvocationStart(invocationInfo())); + } + + @Test + void hookThrowingAJvmError_stillPropagates() { + var runner = new PluginRunner(List.of(info -> new StackOverflowPlugin())); + + assertThrows(StackOverflowError.class, () -> runner.onInvocationStart(invocationInfo())); + } + + @Test + void factoryThrowingAnyVirtualMachineError_stillPropagates() { + // The fatal set is named by the VirtualMachineError supertype rather than by listing its subclasses, so an + // InternalError propagates for the same reason OutOfMemoryError does. This pins the supertype, not the list. + var runner = new PluginRunner(List.of(info -> { + throw new InternalError("JVM internal invariant violated"); + })); + + assertThrows(InternalError.class, () -> runner.onInvocationStart(invocationInfo())); + } + + @Test + void hookThrowingAnyVirtualMachineError_stillPropagates() { + var runner = new PluginRunner(List.of(info -> new UnknownErrorPlugin())); + + assertThrows(UnknownError.class, () -> runner.onInvocationStart(invocationInfo())); + } + + // ─── Thread termination ────────────────────────────────────────────── + // + // Thread.stop() terminates a thread by throwing ThreadDeath into it, which unwinds that thread's stack from + // wherever it stood and releases the monitors it held over state it had only half updated. maven.compiler.source is + // 17, and Thread.stop() still delivers ThreadDeath on a JDK 17 runtime, so the delivery is possible on a runtime + // this SDK supports. Containing the ThreadDeath would return the factory or hook thread to the SDK and user work it + // carries after the plugin returns, with that thread's invariants already broken and the termination dropped. The + // runner therefore rethrows it, at both the factory boundary and the hook boundary. + // + // These tests throw the ThreadDeath directly. Thread.stop() throws UnsupportedOperationException on the JDK 20 or + // later runtime the build uses, so a test cannot ask the JVM to deliver one. + + @Test + @SuppressWarnings("removal") // ThreadDeath is deprecated for removal since JDK 20. + void factoryThrowingThreadDeath_stillPropagates() { + var runner = new PluginRunner(List.of(info -> { + throw new ThreadDeath(); + })); + + assertThrows(ThreadDeath.class, () -> runner.onInvocationStart(invocationInfo())); + } + + @Test + @SuppressWarnings("removal") // ThreadDeath is deprecated for removal since JDK 20. + void hookThrowingThreadDeath_stillPropagates() { + var runner = new PluginRunner(List.of(info -> new ThreadDeathPlugin())); + + assertThrows(ThreadDeath.class, () -> runner.onInvocationStart(invocationInfo())); + } + // ─── Fire-and-forget event hooks ───────────────────────────────────── @Test void fireAndForget_callsAllPlugins() { var calls = new ArrayList(); - var plugin1 = new TestPlugin("p1", calls); - var plugin2 = new TestPlugin("p2", calls); - var runner = new PluginRunner(List.of(plugin1, plugin2)); + var runner = + new PluginRunner(List.of(info -> new TestPlugin("p1", calls), info -> new TestPlugin("p2", calls))); runner.onInvocationStart(invocationInfo()); @@ -56,9 +568,7 @@ void fireAndForget_callsAllPlugins() { @Test void fireAndForget_swallowsExceptions() { var calls = new ArrayList(); - var throwingPlugin = new ThrowingPlugin(); - var normalPlugin = new TestPlugin("p2", calls); - var runner = new PluginRunner(List.of(throwingPlugin, normalPlugin)); + var runner = new PluginRunner(List.of(info -> new ThrowingPlugin(), info -> new TestPlugin("p2", calls))); assertDoesNotThrow(() -> runner.onInvocationStart(invocationInfo())); assertEquals(List.of("p2:onInvocationStart"), calls); @@ -67,26 +577,25 @@ void fireAndForget_swallowsExceptions() { @Test void fireAndForget_callsAllHookTypes() { var calls = new ArrayList(); - var plugin = new TestPlugin("p", calls); - var runner = new PluginRunner(List.of(plugin)); + var runner = new PluginRunner(List.of(info -> new TestPlugin("p", calls))); runner.onInvocationStart(invocationInfo()); - runner.onInvocationEnd(invocationEndInfo()); runner.onOperationStart(operationInfo()); runner.onOperationEnd(operationEndInfo()); runner.onOperationChange(operationChangeInfo()); runner.onUserFunctionStart(attemptInfo()); runner.onUserFunctionEnd(attemptEndInfo()); + runner.onInvocationEnd(invocationEndInfo()); assertEquals( List.of( "p:onInvocationStart", - "p:onInvocationEnd", "p:onOperationStart", "p:onOperationEnd", "p:onOperationChange", "p:onUserFunctionStart", - "p:onUserFunctionEnd"), + "p:onUserFunctionEnd", + "p:onInvocationEnd"), calls); } @@ -95,9 +604,10 @@ void fireAndForget_callsAllHookTypes() { @Test void awaitedHooks_callAllPlugins() { var calls = new ArrayList(); - var plugin1 = new TestPlugin("p1", calls); - var plugin2 = new TestPlugin("p2", calls); - var runner = new PluginRunner(List.of(plugin1, plugin2)); + var runner = + new PluginRunner(List.of(info -> new TestPlugin("p1", calls), info -> new TestPlugin("p2", calls))); + runner.onInvocationStart(invocationInfo()); + calls.clear(); runner.onInvocationEnd(invocationEndInfo()); @@ -107,32 +617,14 @@ void awaitedHooks_callAllPlugins() { @Test void awaitedHooks_swallowExceptions_butCallRemainingPlugins() { var calls = new ArrayList(); - var throwingPlugin = new ThrowingPlugin(); - var normalPlugin = new TestPlugin("p2", calls); - var runner = new PluginRunner(List.of(throwingPlugin, normalPlugin)); + var runner = new PluginRunner(List.of(info -> new ThrowingPlugin(), info -> new TestPlugin("p2", calls))); + runner.onInvocationStart(invocationInfo()); + calls.clear(); assertDoesNotThrow(() -> runner.onInvocationEnd(invocationEndInfo())); assertEquals(List.of("p2:onInvocationEnd"), calls); } - // ─── Thread safety (basic) ─────────────────────────────────────────── - - @Test - void pluginRunner_isImmutable() { - var calls = new ArrayList(); - var mutableList = new ArrayList(); - mutableList.add(new TestPlugin("p1", calls)); - var runner = new PluginRunner(mutableList); - - // Modifying the original list should not affect the runner - mutableList.add(new TestPlugin("p2", calls)); - - runner.onInvocationStart(invocationInfo()); - - // Only p1 should be called — p2 was added after construction - assertEquals(List.of("p1:onInvocationStart"), calls); - } - // ─── Execution input / result components ───────────────────────────── @Test @@ -316,4 +808,98 @@ public void onInvocationEnd(InvocationEndInfo info) { throw new RuntimeException("boom"); } } + + /** + * Plugin whose hooks fail to link, as a plugin compiled against a different SDK version or missing an optional + * dependency does. + */ + private static class LinkageErrorPlugin implements DurableExecutionPlugin { + @Override + public void onInvocationStart(InvocationInfo info) { + throw new NoClassDefFoundError("software/amazon/example/OptionalExporter"); + } + + @Override + public void onOperationStart(OperationInfo info) { + throw new AbstractMethodError("software.amazon.example.LegacyPlugin.onOperationStart(OperationInfo)"); + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + throw new IncompatibleClassChangeError("software.amazon.example.LegacyPlugin"); + } + } + + /** Plugin whose hook reports that the JVM itself is failing, which must not be contained. */ + private static class StackOverflowPlugin implements DurableExecutionPlugin { + @Override + public void onInvocationStart(InvocationInfo info) { + throw new StackOverflowError(); + } + } + + /** Plugin whose hook throws a VirtualMachineError other than the two the older tests pin. */ + private static class UnknownErrorPlugin implements DurableExecutionPlugin { + @Override + public void onInvocationStart(InvocationInfo info) { + throw new UnknownError("unknown JVM failure"); + } + } + + /** Plugin whose hook thread has been terminated by {@code Thread.stop()}, which must not be contained. */ + @SuppressWarnings("removal") // ThreadDeath is deprecated for removal since JDK 20. + private static class ThreadDeathPlugin implements DurableExecutionPlugin { + @Override + public void onInvocationStart(InvocationInfo info) { + throw new ThreadDeath(); + } + } + + /** Plugin whose hooks fail an assertion, as a plugin running with assertions enabled does. */ + private static class AssertionErrorPlugin implements DurableExecutionPlugin { + @Override + public void onInvocationStart(InvocationInfo info) { + throw new AssertionError("plugin invariant violated"); + } + + @Override + public void onOperationStart(OperationInfo info) { + throw new AssertionError("plugin invariant violated"); + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + throw new AssertionError("plugin invariant violated"); + } + } + + /** Plugin whose hook fails its own service lookup, as a plugin loading its exporter back ends does. */ + private static class ServiceConfigurationErrorPlugin implements DurableExecutionPlugin { + @Override + public void onInvocationStart(InvocationInfo info) { + throw new ServiceConfigurationError("software.amazon.example.Exporter: provider not found"); + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + throw new ServiceConfigurationError("software.amazon.example.Exporter: provider not found"); + } + } + + /** Plugin whose awaited hook is interrupted while flushing and rethrows the InterruptedException undeclared. */ + private static class InterruptingPlugin implements DurableExecutionPlugin { + @Override + public void onInvocationEnd(InvocationEndInfo info) { + sneakyThrow(new InterruptedException("flush interrupted")); + } + } + + /** + * Throws {@code t} without declaring it, which is how plugin code can reach the runner with an + * {@link InterruptedException} even though no hook signature permits a checked exception. + */ + @SuppressWarnings("unchecked") + private static void sneakyThrow(Throwable t) throws E { + throw (E) t; + } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/plugin/UserFunctionStartFailureTest.java b/sdk/src/test/java/software/amazon/lambda/durable/plugin/UserFunctionStartFailureTest.java new file mode 100644 index 000000000..cfbdd07ca --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/plugin/UserFunctionStartFailureTest.java @@ -0,0 +1,129 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.plugin; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class UserFunctionStartFailureTest { + @SuppressWarnings("removal") + static Stream failures() { + return Stream.of(false, true) + .flatMap(wrapped -> Stream.of(new InternalError("start hook"), new ThreadDeath()) + .map(fatal -> Arguments.of(wrapped, fatal))); + } + + @ParameterizedTest + @MethodSource("failures") + void completedStartHooksUnwindOnOwnerBeforeFatalPublication(boolean wrapped, Error fatal) { + var owner = Thread.currentThread(); + var scopes = new ArrayDeque(); + var calls = new ArrayList(); + var endInfos = new ArrayList(); + var reported = new AtomicReference(); + var cleanupAtPublication = new AtomicInteger(-1); + var runner = new PluginRunner( + List.of( + scope("first", scopes, calls, endInfos, owner, null), + scope("second", scopes, calls, endInfos, owner, null), + ignored -> new DurableExecutionPlugin() { + public void onUserFunctionStart(UserFunctionStartInfo info) { + if (wrapped) throw new CompletionException(new ExecutionException(fatal)); + throw fatal; + } + + public void onUserFunctionEnd(UserFunctionEndInfo info) { + calls.add("failed:end"); + } + }, + scope("unreached", scopes, calls, endInfos, owner, null)), + error -> { + cleanupAtPublication.set(scopes.size()); + reported.set(error); + }); + runner.onInvocationStart(invocation()); + var start = start(); + assertSame(fatal, assertThrows(Error.class, () -> runner.onUserFunctionStart(start))); + assertSame(fatal, reported.get()); + assertEquals(0, cleanupAtPublication.get()); + assertEquals(List.of("first:start", "second:start", "second:end", "first:end"), calls); + assertEquals(2, endInfos.size()); + for (var end : endInfos) { + assertEquals(start.id(), end.id()); + assertEquals(start.attempt(), end.attempt()); + assertEquals(start.startTimestamp(), end.startTimestamp()); + assertEquals(UserFunctionOutcome.FAILED, end.outcome()); + assertSame(fatal, end.error()); + } + } + + @Test + void cleanupFatalDoesNotMaskOriginalOrSkipEarlierCleanup() { + var fatal = new InternalError("start"); + var cleanupFailure = new InternalError("cleanup"); + var scopes = new ArrayDeque(); + var calls = new ArrayList(); + var ends = new ArrayList(); + var reported = new AtomicReference(); + var runner = new PluginRunner( + List.of( + scope("first", scopes, calls, ends, Thread.currentThread(), null), + scope("second", scopes, calls, ends, Thread.currentThread(), cleanupFailure), + ignored -> new DurableExecutionPlugin() { + public void onUserFunctionStart(UserFunctionStartInfo info) { + throw fatal; + } + }), + reported::set); + runner.onInvocationStart(invocation()); + assertSame(fatal, assertThrows(InternalError.class, () -> runner.onUserFunctionStart(start()))); + assertSame(fatal, reported.get()); + assertTrue(scopes.isEmpty()); + assertEquals(List.of("first:start", "second:start", "second:end", "first:end"), calls); + assertArrayEquals(new Throwable[] {cleanupFailure}, fatal.getSuppressed()); + } + + private static DurableExecutionPluginFactory scope( + String name, + ArrayDeque scopes, + List calls, + List ends, + Thread owner, + Error cleanupFailure) { + return ignored -> new DurableExecutionPlugin() { + public void onUserFunctionStart(UserFunctionStartInfo info) { + scopes.push(name); + calls.add(name + ":start"); + } + + public void onUserFunctionEnd(UserFunctionEndInfo info) { + assertSame(owner, Thread.currentThread()); + assertEquals(name, scopes.pop()); + calls.add(name + ":end"); + ends.add(info); + if (cleanupFailure != null) throw cleanupFailure; + } + }; + } + + private static InvocationInfo invocation() { + return new InvocationInfo("request", "arn:test", true, Instant.EPOCH); + } + + private static UserFunctionStartInfo start() { + return new UserFunctionStartInfo("op", "step", "STEP", null, null, Instant.EPOCH, false, 1); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/util/AsyncFailureUnwrappingTest.java b/sdk/src/test/java/software/amazon/lambda/durable/util/AsyncFailureUnwrappingTest.java new file mode 100644 index 000000000..7bc614bdb --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/util/AsyncFailureUnwrappingTest.java @@ -0,0 +1,36 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.util; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import org.junit.jupiter.api.Test; + +class AsyncFailureUnwrappingTest { + @Test + void nestedFutureWrappersPreserveTheActualFatalCause() { + var fatal = new InternalError("fatal"); + var checked = new ExecutionException(fatal); + var nested = new CompletionException(new ExecutionException(new CompletionException(checked))); + assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(nested)); + // Existing operation callers retain their completion-only policy and checkpoint error meaning. + assertSame(checked, ExceptionHelper.unwrapCompletableFuture(new CompletionException(checked))); + } + + @Test + void applicationExceptionIsNotReplacedByItsCause() { + var applicationFailure = new IllegalArgumentException("business error", new InternalError("cause")); + assertSame(applicationFailure, ExceptionHelper.unwrapAsyncFailure(new CompletionException(applicationFailure))); + } + + @Test + void absentCausesRetainDiagnosticWrappers() { + var completion = new CompletionException((Throwable) null); + var execution = new ExecutionException((Throwable) null); + assertSame(completion, ExceptionHelper.unwrapAsyncFailure(completion)); + assertSame(execution, ExceptionHelper.unwrapAsyncFailure(execution)); + assertNull(ExceptionHelper.unwrapAsyncFailure(null)); + } +} From 10e8f11248dab23bac46fc7a1cece829592b6d03 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:29:49 -0700 Subject: [PATCH 03/17] test: verify installed OpenTelemetry API compatibility --- .../scripts/verify_otel_api_compatibility.py | 189 +++++++++++++++++ .github/workflows/build.yml | 4 + .../compatibility/b1/InstalledApiProbe.java | 200 ++++++++++++++++++ .../src/test/compatibility/b1/README.md | 39 ++++ otel-plugin/src/test/compatibility/b1/pom.xml | 52 +++++ 5 files changed, 484 insertions(+) create mode 100644 .github/scripts/verify_otel_api_compatibility.py create mode 100644 otel-plugin/src/test/compatibility/b1/InstalledApiProbe.java create mode 100644 otel-plugin/src/test/compatibility/b1/README.md create mode 100644 otel-plugin/src/test/compatibility/b1/pom.xml diff --git a/.github/scripts/verify_otel_api_compatibility.py b/.github/scripts/verify_otel_api_compatibility.py new file mode 100644 index 000000000..b0e88c7aa --- /dev/null +++ b/.github/scripts/verify_otel_api_compatibility.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exercise real released/candidate core and plugin artifacts with two visible OTel APIs. + +Dependency resolution and every probe are required: a network, compilation, or case +failure returns nonzero. No production dependency versions are modified. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ET + +API_VERSIONS = ("1.49.0", "1.66.0") +RELEASED_VERSION = "2.2.1" +CORE = "aws-durable-execution-sdk-java" +PLUGIN = "aws-durable-execution-sdk-java-plugin-otel" +PROBE = "software.amazon.lambda.durable.otel.InstalledApiProbe" + + +def execute(command: list[str], log: Path, *, env: dict[str, str] | None = None, timeout: int = 300) -> None: + with log.open("w") as output: + try: + result = subprocess.run(command, stdout=output, stderr=subprocess.STDOUT, env=env, + check=False, timeout=timeout) + except subprocess.TimeoutExpired as error: + raise RuntimeError(f"Command timed out after {timeout}s; log={log}") from error + if result.returncode: + tail = "\n".join(log.read_text(errors="replace").splitlines()[-35:]) + raise RuntimeError(f"Command failed ({result.returncode}); log={log}\n{tail}") + + +def artifact(entries: list[Path], name: str, version: str) -> Path: + matches = [p for p in entries if p.name == f"{name}-{version}.jar"] + if len(matches) != 1: + raise RuntimeError(f"Expected exactly one {name}:{version}, got {matches}") + if not matches[0].is_file(): + raise RuntimeError(f"Resolved artifact is absent: {matches[0]}") + return matches[0].resolve() + + +def jar_facts(path: Path) -> dict[str, str]: + return {"path": str(path), "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} + + +def snapshot_candidate(path: Path, output: Path) -> Path: + expected = jar_facts(path)["sha256"] + directory = output / "candidate-artifacts" + directory.mkdir(exist_ok=True) + target = directory / path.name + if path.resolve() != target.resolve(): + shutil.copyfile(path, target) + if jar_facts(target)["sha256"] != expected or jar_facts(path)["sha256"] != expected: + raise RuntimeError(f"Candidate changed while being snapshotted: {path}") + return target.resolve() + + +def candidate_jar(root: Path, module: str, name: str) -> Path: + pom = ET.parse(root / "pom.xml") + version = pom.findtext("{http://maven.apache.org/POM/4.0.0}version") + if not version: + raise RuntimeError("Cannot resolve the reactor version from pom.xml") + path = root / module / "target" / f"{name}-{version}.jar" + if not path.is_file(): + raise RuntimeError(f"Build the candidate first; artifact missing: {path}") + return path.resolve() + + +def resolve_classpaths(fixture: Path, output: Path, maven: str) -> dict[str, list[Path]]: + classpaths: dict[str, list[Path]] = {} + for version in API_VERSIONS: + target = output / f"dependencies-{version}.txt" + execute([ + maven, "-B", "-f", str(fixture / "pom.xml"), + "org.apache.maven.plugins:maven-dependency-plugin:3.11.0:build-classpath", + f"-Dotel.api.version={version}", f"-Dmdep.outputFile={target}", + ], output / f"resolve-{version}.log") + classpaths[version] = [Path(p).resolve() for p in target.read_text().strip().split(os.pathsep)] + artifact(classpaths[version], "opentelemetry-api", version) + artifact(classpaths[version], "opentelemetry-context", version) + return classpaths + + +def probe_environment(view: str) -> dict[str, str]: + env = os.environ.copy() + # The fixture sets its own plugin registration/global provider. Do not inherit + # Lambda-hosted CI tracing or a developer's auto-agent/plugin configuration. + for key in ("_X_AMZN_TRACE_ID", "DURABLE_EXECUTION_PLUGINS", "JAVA_TOOL_OPTIONS", + "JDK_JAVA_OPTIONS", "OTEL_JAVAAGENT_EXTENSIONS", "AWS_LAMBDA_EXEC_WRAPPER"): + env.pop(key, None) + env["DURABLE_EXECUTION_PLUGINS"] = f"{view},compat-healthy" + return env + + +def run_matrix(args: argparse.Namespace) -> int: + root = args.root.resolve() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + fixture = root / "otel-plugin/src/test/compatibility/b1" + cp = resolve_classpaths(fixture, output, args.maven) + released_core = artifact(cp["1.66.0"], CORE, RELEASED_VERSION) + released_plugin = artifact(cp["1.66.0"], PLUGIN, RELEASED_VERSION) + new_core = args.new_core.resolve() if args.new_core else candidate_jar(root, "sdk", CORE) + new_plugin = args.new_plugin.resolve() if args.new_plugin else candidate_jar(root, "otel-plugin", PLUGIN) + for jar in (new_core, new_plugin): + if not jar.is_file(): + raise RuntimeError(f"Candidate artifact missing: {jar}") + candidate_inputs = {"core": jar_facts(new_core), "plugin": jar_facts(new_plugin)} + new_core = snapshot_candidate(new_core, output) + new_plugin = snapshot_candidate(new_plugin, output) + classes = output / "classes" + classes.mkdir(exist_ok=True) + execute([args.javac, "--release", "17", "-classpath", os.pathsep.join(map(str, cp["1.66.0"])), + "-d", str(classes), str(fixture / "InstalledApiProbe.java")], output / "compile.log") + services = classes / "META-INF/services" + services.mkdir(parents=True, exist_ok=True) + (services / "software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider").write_text( + PROBE + "$HealthyProvider\n") + report: dict[str, object] = { + "released_core": jar_facts(released_core), "released_plugin": jar_facts(released_plugin), + "new_core": jar_facts(new_core), "new_plugin": jar_facts(new_plugin), + "candidate_inputs": candidate_inputs, + "cases": [], "agent_coverage": "This matrix is visible-API skew, not a deployed Java-agent test.", + } + cases: list[dict[str, object]] = report["cases"] # type: ignore[assignment] + failures = 0 + pairs = {"old-old": (released_core, released_plugin), "new-old": (new_core, released_plugin), + "old-new": (released_core, new_plugin), "new-new": (new_core, new_plugin)} + for version in API_VERSIONS: + api = artifact(cp[version], "opentelemetry-api", version) + context = artifact(cp[version], "opentelemetry-context", version) + dependencies = [p for p in cp[version] if p.name not in + (f"{CORE}-{RELEASED_VERSION}.jar", f"{PLUGIN}-{RELEASED_VERSION}.jar")] + for label, (core, plugin) in pairs.items(): + for view in ("otel-invocation", "otel-execution"): + name = f"{label}-api{version}-{view}" + negative = label == "old-old" and version == "1.49.0" + case: dict[str, object] = {"name": name, "expected_negative_control": negative, + "api": jar_facts(api), "context": jar_facts(context)} + command = [args.java, "-cp", os.pathsep.join(map(str, [classes, core, plugin, *dependencies])), + PROBE, str(core), str(plugin), str(api), str(context), view, + str(negative).lower(), str(version == "1.66.0").lower()] + try: + log = output / f"{name}.log" + execute(command, log, env=probe_environment(view), timeout=90) + contents = log.read_text(errors="replace") + if "COMPAT_PASS " not in contents: + raise RuntimeError("Probe did not report successful completion") + if negative and "NEGATIVE_CONTROL_REPRODUCED" not in contents: + raise RuntimeError("Released negative control did not reproduce the reported failure") + case["passed"] = True + except RuntimeError as error: + failures += 1 + case.update(passed=False, error=str(error)) + cases.append(case) + (output / "results.json").write_text(json.dumps(report, indent=2) + "\n") + print(f"{'PASS' if case['passed'] else 'FAIL'} {name}", flush=True) + report["passed"] = failures == 0 + report["failure_count"] = failures + (output / "results.json").write_text(json.dumps(report, indent=2) + "\n") + print(f"Installed artifact matrix: {len(cases) - failures}/{len(cases)} passed; {output / 'results.json'}") + return 1 if failures else 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument("--output", type=Path, default=Path("target/otel-api-compatibility")) + parser.add_argument("--new-core", type=Path) + parser.add_argument("--new-plugin", type=Path) + parser.add_argument("--maven", default=shutil.which("mvn") or "mvn") + parser.add_argument("--java", default=shutil.which("java") or "java") + parser.add_argument("--javac", default=shutil.which("javac") or "javac") + try: + return run_matrix(parser.parse_args()) + except (RuntimeError, OSError, subprocess.SubprocessError) as error: + print(f"Compatibility harness failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a3e02cc11..b7bf5bef5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,6 +81,10 @@ jobs: - name: Build and test run: mvn -B install --file pom.xml + - name: Verify installed OpenTelemetry API compatibility + if: ${{ matrix.java == 17 }} + run: python3 .github/scripts/verify_otel_api_compatibility.py + - name: Setup uv for coverage badge if: ${{ matrix.java == 17 }} # cicirello/jacoco-badge-generator is a Docker-based action; the diff --git a/otel-plugin/src/test/compatibility/b1/InstalledApiProbe.java b/otel-plugin/src/test/compatibility/b1/InstalledApiProbe.java new file mode 100644 index 000000000..9d31b1d58 --- /dev/null +++ b/otel-plugin/src/test/compatibility/b1/InstalledApiProbe.java @@ -0,0 +1,200 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.ServiceLoader; +import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.LoggerFactory; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.DurableExecutionPluginProvider; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.testing.TestResult; + +/** Compiled against the actual released 2.2.1 SPI, then run unchanged in fresh matrix JVMs. */ +public final class InstalledApiProbe { + private static final AtomicInteger HEALTHY_CREATED = new AtomicInteger(); + private static final AtomicInteger HEALTHY_STARTS = new AtomicInteger(); + private static final AtomicInteger HEALTHY_ENDS = new AtomicInteger(); + + private InstalledApiProbe() {} + + public static void main(String[] args) throws Exception { + verifyArtifacts(args); + var view = args[4]; + var root = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + var logs = new ListAppender(); + logs.start(); + root.addAppender(logs); + try { + exercise(view, Boolean.parseBoolean(args[5]), Boolean.parseBoolean(args[6]), logs); + System.out.println("COMPAT_PASS " + view + " negative=" + args[5] + " api=" + args[2] + + " core=" + args[0] + " plugin=" + args[1]); + } finally { + root.detachAppender(logs); + logs.stop(); + GlobalOpenTelemetry.resetForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); + } + } + + private static void verifyArtifacts(String[] args) throws Exception { + checkSource(DurableExecutionPlugin.class, Path.of(args[0])); + checkSource(GlobalOpenTelemetry.class, Path.of(args[2])); + checkSource(Context.class, Path.of(args[3])); + var provider = ServiceLoader.load(DurableExecutionPluginProvider.class).stream() + .map(ServiceLoader.Provider::get) + .filter(value -> value.getName().equals(args[4])) + .findFirst().orElseThrow(); + check(provider.getApiVersion() == DurableExecutionPluginProvider.API_VERSION, "released SPI version"); + checkSource(provider.getPluginType(), Path.of(args[1])); + } + + private static void exercise(String view, boolean negative, boolean compatible, ListAppender logs) { + GlobalOpenTelemetry.resetForTest(); + // Reproduce #763's documented visible-API skew, not a claim of a deployed agent test. + OtelPluginAutoConfigurationState.markInstalled(); + var handlerCalls = new AtomicInteger(); + var sideEffects = new AtomicInteger(); + var runner = createRunner(handlerCalls, sideEffects); + check(HEALTHY_CREATED.get() == 1, "existing SPI must create the healthy plugin once per configuration"); + var first = runner.run("compatibility-input"); + if (negative) { + assertNegative(first, handlerCalls, sideEffects); + return; + } + check(first.getStatus() == ExecutionStatus.PENDING, "plugin failure must preserve first invocation"); + check(HEALTHY_STARTS.get() == 1 && HEALTHY_ENDS.get() == 1 && handlerCalls.get() == 1, + "healthy plugin and handler must run"); + if (!compatible) { + synchronized (logs) { + check(hasCompatibilityDiagnostic(List.copyOf(logs.list)), "incompatible API must be diagnosed"); + } + } + resumeAndCheck(runner, view, compatible, handlerCalls, sideEffects); + } + + private static LocalDurableTestRunner createRunner( + AtomicInteger handlerCalls, AtomicInteger sideEffects) { + return LocalDurableTestRunner.create(String.class, (input, ctx) -> { + handlerCalls.incrementAndGet(); + var saved = ctx.step("saved", String.class, step -> { + sideEffects.incrementAndGet(); + return input; + }); + ctx.wait("resume", Duration.ofSeconds(1)); + return saved; + }); + } + + private static void assertNegative( + TestResult result, AtomicInteger handlerCalls, AtomicInteger sideEffects) { + check(result.getStatus() == ExecutionStatus.FAILED, "old/old older API must reproduce customer failure"); + var failure = result.getError().orElseThrow(); + check(failure.errorType().endsWith("NoSuchMethodError") + && failure.errorMessage().contains("GlobalOpenTelemetry.isSet"), + "negative control must reproduce the exact unsupported API: " + failure); + check(HEALTHY_STARTS.get() == 0 && handlerCalls.get() == 0 && sideEffects.get() == 0, + "old linkage failure must precede the healthy start hook and handler"); + System.out.println("NEGATIVE_CONTROL_REPRODUCED NoSuchMethodError GlobalOpenTelemetry.isSet"); + } + + private static void resumeAndCheck(LocalDurableTestRunner runner, String view, boolean compatible, + AtomicInteger handlerCalls, AtomicInteger sideEffects) { + var exporter = InMemorySpanExporter.create(); + var tracing = registerLateGlobal(compatible, exporter); + try { + runner.advanceTime(); + var last = runner.runUntilComplete("compatibility-input"); + check(last.getStatus() == ExecutionStatus.SUCCEEDED, "handler must complete after resume"); + check("compatibility-input".equals(last.getResult(String.class)), "handler output must be preserved"); + check(HEALTHY_STARTS.get() == 2 && HEALTHY_ENDS.get() == 2 && handlerCalls.get() == 2, + "healthy hooks and handler must remain active on resume"); + check(sideEffects.get() == 1, "completed user step must not repeat on resume"); + check(HEALTHY_CREATED.get() == 1, "resume must preserve the existing 2.x plugin instance lifetime"); + var spans = exporter.getFinishedSpanItems(); + if (compatible) check(spans.stream().anyMatch(span -> span.getName().equals("Workflow")), + "compatible global provider must export Workflow spans in " + view); + else check(spans.isEmpty(), "unsupported global API must disable the affected instrumentation"); + } finally { + if (tracing != null) tracing.close(); + } + } + + private static SdkTracerProvider registerLateGlobal(boolean compatible, InMemorySpanExporter exporter) { + // Registration must succeed: an early plugin must not freeze the global as no-op. + if (!compatible) { + GlobalOpenTelemetry.set(OpenTelemetry.noop()); + return null; + } + var builder = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + DeterministicIdGenerator.installOn(builder); + DurableSampler.installOn(builder); + var tracing = builder.build(); + OpenTelemetrySdk.builder().setTracerProvider(tracing).buildAndRegisterGlobal(); + return tracing; + } + + /** A real old-SPI service provider, discovered alongside the actual released/candidate OTel provider. */ + public static final class HealthyProvider implements DurableExecutionPluginProvider { + @Override + public String getName() { return "compat-healthy"; } + + @Override + public int getApiVersion() { return API_VERSION; } + + @Override + public Class getPluginType() { return HealthyPlugin.class; } + + @Override + public DurableExecutionPlugin createPlugin() { + HEALTHY_CREATED.incrementAndGet(); + return new HealthyPlugin(); + } + } + + public static final class HealthyPlugin implements DurableExecutionPlugin { + @Override + public void onInvocationStart(InvocationInfo info) { HEALTHY_STARTS.incrementAndGet(); } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { HEALTHY_ENDS.incrementAndGet(); } + } + + private static boolean hasCompatibilityDiagnostic(List events) { + return events.stream().anyMatch(event -> { + var text = event.getFormattedMessage(); + var error = event.getThrowableProxy(); + if (error != null) text += " " + error.getClassName() + " " + error.getMessage(); + return event.getLevel().isGreaterOrEqual(Level.WARN) + && text.contains("OpenTelemetry") + && (text.contains("API") || text.contains("isSet") || text.contains("getOrNoop") + || text.contains("NoSuchMethodError")); + }); + } + + private static void checkSource(Class type, Path expected) throws Exception { + var actual = Path.of(type.getProtectionDomain().getCodeSource().getLocation().toURI()).toRealPath(); + check(actual.equals(expected.toRealPath()), type.getName() + " loaded from wrong artifact: " + actual); + } + + private static void check(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/otel-plugin/src/test/compatibility/b1/README.md b/otel-plugin/src/test/compatibility/b1/README.md new file mode 100644 index 000000000..a0d8ab922 --- /dev/null +++ b/otel-plugin/src/test/compatibility/b1/README.md @@ -0,0 +1,39 @@ +# Installed OpenTelemetry API compatibility + +Run after building the SDK and OTel plugin: + +```sh +python3 .github/scripts/verify_otel_api_compatibility.py +``` + +The driver resolves real released core, testing, and plugin **2.2.1** artifacts +through Maven. It compiles one probe against the released SPI and runs fresh JVMs +with old/old, new/old, old/new, and new/new core/plugin pairs, in both OTel views, +using actual API/context **1.49.0** and **1.66.0** jars. Candidate jars come from +the reactor build, or explicit `--new-core` / `--new-plugin` paths. + +The sixteen required cases verify: + +- Selected core, plugin, API, and context classes load from the intended jars. +- The existing service-provider API discovers and creates a healthy plugin, with + the same instance lifetime across suspension/resume. +- Old/old plus API 1.49 reproduces the exact `GlobalOpenTelemetry.isSet` + `NoSuchMethodError` and customer failure, before healthy start hooks/user code. +- Fixed combinations isolate that mismatch, report a diagnostic, and preserve + healthy hooks, handler output, and completed-step replay behavior. +- An early unsupported/uninitialized global provider does not install a no-op + global: subsequent registration must succeed. API 1.66 then exports real + Workflow spans; unsupported API 1.49 disables the affected instrumentation. + +Resolution, compilation, timeout, and probe failures all fail the command. There +is no network-dependent skip. Logs, artifact SHA-256 hashes, negative-control +results, and the case summary are written to `target/otel-api-compatibility`. +The fixture POM is independent of the reactor and changes no production version +or dependency floor. + +This matrix reproduces visible-API/classpath skew with real artifacts. Its test +marker enables the documented plugin auto-configuration path; it does **not** +claim to deploy or validate every Java-agent version. Actual agent validation +must identify the released agent version, extension jar, visible API, runtime, +and observed behavior separately. Provider registration remains fail-fast for +invalid configuration; the fixture does not introduce invocation factories. diff --git a/otel-plugin/src/test/compatibility/b1/pom.xml b/otel-plugin/src/test/compatibility/b1/pom.xml new file mode 100644 index 000000000..bd6992fe4 --- /dev/null +++ b/otel-plugin/src/test/compatibility/b1/pom.xml @@ -0,0 +1,52 @@ + + + + + 4.0.0 + software.amazon.lambda.durable.compatibility + otel-installed-api-fixture + 1.0-SNAPSHOT + pom + + 1.66.0 + + + + software.amazon.lambda.durable + aws-durable-execution-sdk-java-plugin-otel + 2.2.1 + + + software.amazon.lambda.durable + aws-durable-execution-sdk-java-testing + 2.2.1 + + + io.opentelemetry + opentelemetry-api + ${otel.api.version} + + + io.opentelemetry + opentelemetry-context + ${otel.api.version} + + + io.opentelemetry + opentelemetry-sdk + 1.66.0 + + + io.opentelemetry + opentelemetry-sdk-testing + 1.66.0 + + + ch.qos.logback + logback-classic + 1.6.5 + + + From 3b2d6f6ae6965527995446573290be2ccd308531 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 01:34:41 +0000 Subject: [PATCH 04/17] ci: run OTel conformance on validated CodeBuild Java 21 --- .github/workflows/otel-conformance-tests.yml | 33 +++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/otel-conformance-tests.yml b/.github/workflows/otel-conformance-tests.yml index 24b7a75b2..f91ad632c 100644 --- a/.github/workflows/otel-conformance-tests.yml +++ b/.github/workflows/otel-conformance-tests.yml @@ -68,8 +68,9 @@ jobs: actions: write contents: read id-token: write - uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@98b802cdb172614f98e217f7464784d47b9bb484 + uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@f5855f2d0f60be996973173cf479c3567f83e30f with: + runs_on: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ubuntu-latest' || format('codebuild-github-actions-runner-{0}-{1}', github.run_id, github.run_attempt) }} language: java resource_prefix: j sdk_repository: aws/aws-durable-execution-sdk-java @@ -81,13 +82,37 @@ jobs: # checked out (.build/durable-sdk). examples_dir: .build/durable-sdk/conformance-tests-otel setup_command: | - if [ -z "${JAVA_HOME_21_X64:-}" ]; then - echo "The runner does not provide Java 21" + set -euo pipefail + if [ -n "${JAVA_HOME_21_X64:-}" ]; then + export JAVA_HOME="$JAVA_HOME_21_X64" + elif [ -x /usr/lib/jvm/java-21-amazon-corretto/bin/java ]; then + export JAVA_HOME=/usr/lib/jvm/java-21-amazon-corretto + elif [ -z "${JAVA_HOME:-}" ]; then + export JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v java)")")")" + fi + if [ ! -x "$JAVA_HOME/bin/java" ] || [ ! -x "$JAVA_HOME/bin/javac" ]; then + echo "Selected JAVA_HOME=$JAVA_HOME is not a complete JDK" exit 1 fi - export JAVA_HOME="$JAVA_HOME_21_X64" export PATH="$JAVA_HOME/bin:$PATH" + java_spec=$(java -XshowSettings:properties -version 2>&1 | awk '$1 == "java.specification.version" {print $3}') + if [ "$java_spec" != "21" ]; then + echo "Java 21 is required; selected JAVA_HOME=$JAVA_HOME reports specification version $java_spec" + exit 1 + fi + javac_version=$(javac -version 2>&1) + if [[ ! "$javac_version" =~ ^javac[[:space:]]21([.]|[[:space:]]|$) ]]; then + echo "Java 21 javac is required; selected compiler reports $javac_version" + exit 1 + fi java -version + javac -version + if [ -n "${GITHUB_ENV:-}" ]; then + echo "JAVA_HOME=$JAVA_HOME" >> "$GITHUB_ENV" + fi + if [ -n "${GITHUB_PATH:-}" ]; then + echo "$JAVA_HOME/bin" >> "$GITHUB_PATH" + fi prepare_command: | JAVA_SDK_VERSION=$( mvn -B -q \ From 3a49c5a96d4121899c9336ae39f38f23d2fd1219 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 01:34:41 +0000 Subject: [PATCH 05/17] ci: run OTel conformance on validated CodeBuild Java 21 --- .github/workflows/otel-conformance-tests.yml | 33 +++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/otel-conformance-tests.yml b/.github/workflows/otel-conformance-tests.yml index 87077a82f..452db089d 100644 --- a/.github/workflows/otel-conformance-tests.yml +++ b/.github/workflows/otel-conformance-tests.yml @@ -68,8 +68,9 @@ jobs: actions: write contents: read id-token: write - uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@43872f8d5dd917fe3ee6d01a6c953c1ff67c5fe4 + uses: aws/aws-durable-execution-conformance-tests/.github/workflows/opentelemetry-orchestrator.yml@a66037abbbfa55fde97f714e30f0bc262edefd63 with: + runs_on: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'ubuntu-latest' || format('codebuild-github-actions-runner-{0}-{1}', github.run_id, github.run_attempt) }} language: java resource_prefix: j sdk_repository: aws/aws-durable-execution-sdk-java @@ -81,13 +82,37 @@ jobs: # checked out (.build/durable-sdk). examples_dir: .build/durable-sdk/conformance-tests-otel setup_command: | - if [ -z "${JAVA_HOME_21_X64:-}" ]; then - echo "The runner does not provide Java 21" + set -euo pipefail + if [ -n "${JAVA_HOME_21_X64:-}" ]; then + export JAVA_HOME="$JAVA_HOME_21_X64" + elif [ -x /usr/lib/jvm/java-21-amazon-corretto/bin/java ]; then + export JAVA_HOME=/usr/lib/jvm/java-21-amazon-corretto + elif [ -z "${JAVA_HOME:-}" ]; then + export JAVA_HOME="$(dirname "$(dirname "$(readlink -f "$(command -v java)")")")" + fi + if [ ! -x "$JAVA_HOME/bin/java" ] || [ ! -x "$JAVA_HOME/bin/javac" ]; then + echo "Selected JAVA_HOME=$JAVA_HOME is not a complete JDK" exit 1 fi - export JAVA_HOME="$JAVA_HOME_21_X64" export PATH="$JAVA_HOME/bin:$PATH" + java_spec=$(java -XshowSettings:properties -version 2>&1 | awk '$1 == "java.specification.version" {print $3}') + if [ "$java_spec" != "21" ]; then + echo "Java 21 is required; selected JAVA_HOME=$JAVA_HOME reports specification version $java_spec" + exit 1 + fi + javac_version=$(javac -version 2>&1) + if [[ ! "$javac_version" =~ ^javac[[:space:]]21([.]|[[:space:]]|$) ]]; then + echo "Java 21 javac is required; selected compiler reports $javac_version" + exit 1 + fi java -version + javac -version + if [ -n "${GITHUB_ENV:-}" ]; then + echo "JAVA_HOME=$JAVA_HOME" >> "$GITHUB_ENV" + fi + if [ -n "${GITHUB_PATH:-}" ]; then + echo "$JAVA_HOME/bin" >> "$GITHUB_PATH" + fi prepare_command: | JAVA_SDK_VERSION=$( mvn -B -q \ From 7c50701be4d4d62d77cddcab1f09301243612ade Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 02:52:45 +0000 Subject: [PATCH 06/17] ci: trigger OTel compatibility checks for module and harness changes --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7bf5bef5..61e067dba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,6 +29,8 @@ on: - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'insight-plugin/**' + - 'otel-plugin/**' + - '.github/scripts/verify_otel_api_compatibility.py' - 'examples/**' - 'pom.xml' push: @@ -42,6 +44,8 @@ on: - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'insight-plugin/**' + - 'otel-plugin/**' + - '.github/scripts/verify_otel_api_compatibility.py' - 'examples/**' - 'pom.xml' From f4e4e1db81c6fb0fbbaf25815d73757abf3fda7c Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 03:04:55 +0000 Subject: [PATCH 07/17] ci: use read-only resolver for review-comment intake --- .github/workflows/ai-pr-review-address.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ai-pr-review-address.yml b/.github/workflows/ai-pr-review-address.yml index 2b6e25d79..5bd13defb 100644 --- a/.github/workflows/ai-pr-review-address.yml +++ b/.github/workflows/ai-pr-review-address.yml @@ -23,7 +23,10 @@ jobs: contents: read issues: read pull-requests: read - uses: aws/aws-durable-execution-ci/.github/workflows/ai-pr-review-address.yml@d6b017da14385908951d23e26c790b28a4e5f9f0 + uses: aws/aws-durable-execution-ci/.github/workflows/ai-work-item-resolver.yml@d6b017da14385908951d23e26c790b28a4e5f9f0 + with: + work-scope: review + upload-work-items: true address: if: >- From 987a4e8e628c296ca7f36c2e4ae67b4950bccd2c Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:35:34 -0700 Subject: [PATCH 08/17] fix: wake fatal scope observers without ending owner cleanup --- otel-plugin/README.md | 6 +- .../durable/otel/DurableSamplingDecision.java | 5 +- .../durable/otel/ExecutionOtelPlugin.java | 10 +- .../durable/otel/InvocationOtelPlugin.java | 10 +- .../durable/otel/OtelPluginSupport.java | 5 +- .../otel/CrossLoaderParentSamplingTest.java | 137 +++++++++++++++++ ...urableSamplingDecisionClassLoaderTest.java | 2 +- .../otel/FatalScopeCleanupBudgetTest.java | 142 ++++++++++++++++++ .../otel/HandlerMdcIntegrationTest.java | 99 ++++++++++++ .../durable/otel/ScopeFatalSignalTest.java | 93 ++++++++++++ .../execution/ApiRequestDelayedBatcher.java | 30 +++- .../durable/execution/CheckpointManager.java | 13 +- .../durable/execution/DurableExecutor.java | 32 ++-- .../durable/execution/ExecutionManager.java | 2 + .../execution/HandlerScopeHandoffTest.java | 105 +++++++++++++ .../execution/InFlightFatalAbortTest.java | 105 +++++++++++++ .../PluginFatalCheckpointAbortTest.java | 7 +- 17 files changed, 770 insertions(+), 33 deletions(-) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CrossLoaderParentSamplingTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FatalScopeCleanupBudgetTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ScopeFatalSignalTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/execution/InFlightFatalAbortTest.java diff --git a/otel-plugin/README.md b/otel-plugin/README.md index 04294f8a6..bc42e4fa0 100644 --- a/otel-plugin/README.md +++ b/otel-plugin/README.md @@ -265,14 +265,14 @@ Remote backend server span (Root / Parent) When no valid remote parent can be constructed, a synthetic execution root anchors the trace instead and both spans parent onto it: ``` -Synthetic execution root +DurableExecutionRoot (materialized and re-exported each invocation) ├── Workflow ├── Invocation 1 ├── Invocation 2 └── Invocation N ``` -- **Execution ancestor** — the common parent both the Workflow and Invocation spans resolve onto. A valid remote server span (`Root` and `Parent`) is used directly, whether or not `Sampled` is present; only when a valid remote parent cannot be constructed does a synthetic execution root take its place. It is a non-recording context, not an exported span. +- **Execution ancestor** — the common parent both the Workflow and Invocation spans resolve onto. A valid remote server span (`Root` and `Parent`) is used directly, whether or not `Sampled` is present; only when a valid remote parent cannot be constructed does a synthetic execution root take its place. The remote ancestor is used as a non-recording context; the synthetic ancestor is materialized as `DurableExecutionRoot` and re-exported on each invocation with its stable span ID and execution start time, subject to sampling. - **Workflow span** — one logical span per durable execution, joining the execution trace with a stable span ID derived from the ARN. Exported only on the terminal invocation (SUCCEEDED/FAILED). - **Invocation span** — one per Lambda invocation, parented to the ambient span only when it is on the execution trace, otherwise to the execution ancestor - **Operation span** — one per durable operation, named after your step/wait names @@ -282,7 +282,7 @@ Operation and attempt spans link to the Workflow span. `ExecutionOtelPlugin` rev ### Sampling -The plugin decides sampling once per invocation and applies that single decision to every durable span (Workflow, Invocation, operation, attempt), so the configured sampler is not re-invoked per span and the full decision — including `RECORD_ONLY` — is preserved. The decision follows this precedence, highest first: +The plugin decides sampling once per invocation and applies that single decision to every durable span (DurableExecutionRoot, Workflow, Invocation, operation, attempt), so the configured sampler is not re-invoked per span and the full decision — including `RECORD_ONLY` — is preserved. The decision follows this precedence, highest first: 1. **Backend decision** — `Sampled=1` / `Sampled=0` in the propagated header is authoritative and always preserved, regardless of the configured sampler. 2. **Same-trace ambient span** — when the header carries no usable `Sampled` value but a valid ambient span (for example an auto-instrumentation Lambda handler span) is already on the execution's trace, the plugin follows that span's decision: sampled → sampled; unsampled but still recording → `RECORD_ONLY`; unsampled and not recording → dropped. diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java index 9c72c39bd..2341f8bdb 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DurableSamplingDecision.java @@ -29,8 +29,9 @@ * identity, so a key created in one loader is not equal to the key created in the other. To bridge this: * *

      - *
    1. Context key — used when both sides share a class loader (an application-owned provider). It preserves - * the full {@link SamplingResult}, including any attributes a custom sampler attached. + *
    2. Context key — used when both sides share a class loader (an application-owned provider). Agent-backed + * spans omit this carrier because another loader cannot consume its key. It preserves the full + * {@link SamplingResult}, including any attributes a custom sampler attached. *
    3. Thread-scoped system property — a cross-class-loader fallback modelled on * {@link DeterministicIdGenerator}'s scoped-ID bridge. The intent is published on the thread that creates the * durable span until its sampler consumes the value (before synchronous span processors run), keyed by thread ID diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index dcc82cf51..0074ba61c 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -424,7 +424,13 @@ public void onInvocationEnd(InvocationEndInfo info) { workflowSpan.end(); } OtelPluginSupport.exportExecutionRoot( - tracer, idGenerator, executionAncestor, durableExecutionArn, executionStartTime, samplingIntent); + tracer, + idGenerator, + executionAncestor, + durableExecutionArn, + executionStartTime, + samplingIntent, + sdkTracerProvider != null); // Flush spans before Lambda freezes if (sdkTracerProvider != null) { @@ -711,7 +717,7 @@ private Context resolveParentContext(String parentId) { */ private Context withDurableDecision(Context context) { var intent = samplingIntent; - return intent != null ? DurableSamplingDecision.store(context, intent) : context; + return intent != null && sdkTracerProvider != null ? DurableSamplingDecision.store(context, intent) : context; } private TraceFlags effectiveTraceFlags() { diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 9da13db78..7b3fd3684 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -428,7 +428,13 @@ public void onInvocationEnd(InvocationEndInfo info) { workflowSpan.end(); } OtelPluginSupport.exportExecutionRoot( - tracer, idGenerator, executionAncestor, durableExecutionArn, executionStartTime, samplingIntent); + tracer, + idGenerator, + executionAncestor, + durableExecutionArn, + executionStartTime, + samplingIntent, + sdkTracerProvider != null); if (sdkTracerProvider != null) { // Flush spans before Lambda freezes @@ -725,7 +731,7 @@ private Context resolveParentContext(String parentId) { */ private Context withDurableDecision(Context context) { var intent = samplingIntent; - return intent != null ? DurableSamplingDecision.store(context, intent) : context; + return intent != null && sdkTracerProvider != null ? DurableSamplingDecision.store(context, intent) : context; } /** diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index eeb3ca1b5..9adf0af9e 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -36,13 +36,14 @@ static void exportExecutionRoot( SpanContext ancestor, String arn, Instant start, - DurableSamplingDecision.Intent intent) { + DurableSamplingDecision.Intent intent, + boolean useContextCarrier) { if (ancestor == null || ancestor.isRemote()) { return; } var builder = tracer.spanBuilder("DurableExecutionRoot") .setSpanKind(SpanKind.INTERNAL) - .setParent(DurableSamplingDecision.store(Context.root(), intent)) + .setParent(useContextCarrier ? DurableSamplingDecision.store(Context.root(), intent) : Context.root()) .setAttribute(DURABLE_EXECUTION_ARN, arn) .setAttribute(DURABLE_EXECUTION_SYNTHETIC_ROOT, true) .setStartTimestamp(start); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CrossLoaderParentSamplingTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CrossLoaderParentSamplingTest.java new file mode 100644 index 000000000..282e415f1 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/CrossLoaderParentSamplingTest.java @@ -0,0 +1,137 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; + +class CrossLoaderParentSamplingTest { + @AfterEach + void clearGlobalState() { + GlobalOpenTelemetry.resetForTest(); + DurableSamplingDecision.clearSharedStateForTest(); + DeterministicIdGenerator.clearSharedStateForTest(); + OtelPluginAutoConfigurationState.resetInstalledForTest(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void agentProcessorCannotForwardDurableDecisionToApplicationProvider(boolean executionView) throws Exception { + var observations = new ArrayList(); + try (var agentLoader = DurableSamplingDecisionClassLoaderTest.pluginClassLoader(); + var application = SdkTracerProvider.builder() + .setSampler(DurableSampler.wrap(Sampler.alwaysOff())) + .build(); + var agent = agentProvider(agentLoader, application.get("application"), observations)) { + GlobalOpenTelemetry.set(opaqueTelemetry(agent)); + OtelPluginAutoConfigurationState.markInstalled(); + var config = OtelPluginConfig.builder() + .contextExtractor(() -> new ExtractedContext( + "aabbccddee112233445566778899aabb", null, ExtractedContext.Sampling.SAMPLED)) + .enableMdc(false) + .build(); + var factory = executionView ? ExecutionOtelPlugin.factory(config) : InvocationOtelPlugin.factory(config); + var arn = "arn:aws:lambda:us-east-1:123:function:test/durable/exec"; + var info = new InvocationInfo("request", arn, true, Instant.EPOCH); + var plugin = factory.createPlugin(info); + plugin.onInvocationStart(info); + plugin.onInvocationEnd(new InvocationEndInfo("request", arn, true, InvocationStatus.SUCCEEDED, null)); + assertTrue(observations.stream().anyMatch(o -> o.name().equals("Workflow"))); + assertTrue(observations.stream().anyMatch(o -> o.name().equals("DurableExecutionRoot"))); + for (var observation : observations) { + assertFalse( + observation.auxiliaryRecording(), "forwarded parent leaked sampling for " + observation.name()); + } + } + } + + private static SdkTracerProvider agentProvider( + ClassLoader loader, Tracer application, List observations) throws Exception { + var samplerClass = Class.forName(DurableSampler.class.getName(), true, loader); + assertNotSame(DurableSampler.class, samplerClass); + var wrap = samplerClass.getDeclaredMethod("wrap", Sampler.class); + wrap.setAccessible(true); + var sampler = (Sampler) wrap.invoke(null, Sampler.alwaysOn()); + var builder = SdkTracerProvider.builder() + .setSampler(sampler) + .addSpanProcessor(new ForwardingProcessor(application, observations)); + var ids = Class.forName(DeterministicIdGenerator.class.getName(), true, loader); + var install = ids.getDeclaredMethod("installOn", SdkTracerProviderBuilder.class); + install.setAccessible(true); + install.invoke(null, builder); + return builder.build(); + } + + private static OpenTelemetry opaqueTelemetry(SdkTracerProvider provider) { + var opaque = new TracerProvider() { + @Override + public Tracer get(String name) { + return provider.get(name); + } + + @Override + public Tracer get(String name, String version) { + return provider.get(name, version); + } + }; + return new OpenTelemetry() { + @Override + public TracerProvider getTracerProvider() { + return opaque; + } + + @Override + public ContextPropagators getPropagators() { + return ContextPropagators.noop(); + } + }; + } + + private record Observation(String name, boolean auxiliaryRecording) {} + + private record ForwardingProcessor(Tracer application, List observations) implements SpanProcessor { + @Override + public void onStart(Context parent, ReadWriteSpan span) { + var auxiliary = application + .spanBuilder("processor-auxiliary") + .setParent(parent) + .startSpan(); + observations.add(new Observation(span.getName(), auxiliary.isRecording())); + auxiliary.end(); + } + + @Override + public boolean isStartRequired() { + return true; + } + + @Override + public void onEnd(ReadableSpan span) {} + + @Override + public boolean isEndRequired() { + return false; + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java index bcb9e6c32..936020c15 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DurableSamplingDecisionClassLoaderTest.java @@ -87,7 +87,7 @@ void decisionCrossesClassLoaderBoundary_viaScopedProperty() throws Exception { * holds its own copies, mirroring the two plugin class loaders) while delegating OpenTelemetry and JDK classes to * the parent so those types are shared and interoperable across loaders. */ - private static URLClassLoader pluginClassLoader() { + static URLClassLoader pluginClassLoader() { var classesDir = DurableSamplingDecisionClassLoaderTest.class .getProtectionDomain() .getCodeSource() diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FatalScopeCleanupBudgetTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FatalScopeCleanupBudgetTest.java new file mode 100644 index 000000000..14858a3e4 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/FatalScopeCleanupBudgetTest.java @@ -0,0 +1,142 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.HandlerScoped; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class FatalScopeCleanupBudgetTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @SuppressWarnings("removal") + void laterFatalStillWaitsForEarlierScopeCleanupBeforeEndDispatch(boolean threadDeath) throws Exception { + var enteredEarlier = new CountDownLatch(1); + var releaseEarlier = new CountDownLatch(1); + var endCalled = new CountDownLatch(1); + var scopeClosed = new AtomicBoolean(); + var endedBeforeClose = new AtomicBoolean(); + var ends = new AtomicInteger(); + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("later scope close"); + var callers = Executors.newSingleThreadExecutor(); + var workers = new ThreadPoolExecutor(1, 1, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>()) { + @Override + public void execute(Runnable command) { + super.execute(command); + // Let both the fatal publication and earlier blocking close happen before caller finalization begins. + await(enteredEarlier); + } + }; + try { + var config = DurableConfig.builder() + .withExecutorService(workers) + .withPlugins( + info -> new ScopePlugin( + () -> { + enteredEarlier.countDown(); + await(releaseEarlier); + scopeClosed.set(true); + }, + scopeClosed, + endedBeforeClose, + ends, + endCalled, + fatal), + info -> new ScopePlugin( + () -> { + throw fatal; + }, + scopeClosed, + endedBeforeClose, + ends, + endCalled, + fatal)) + .build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + context.wait("pause", Duration.ofSeconds(1)); + return input; + }, + config); + var result = callers.submit(() -> runner.run("input")); + assertTrue(enteredEarlier.await(3, TimeUnit.SECONDS)); + assertFalse(endCalled.await(100, TimeUnit.MILLISECONDS), "earlier cleanup must receive its handoff budget"); + releaseEarlier.countDown(); + var failure = assertThrows(ExecutionException.class, () -> result.get(3, TimeUnit.SECONDS)); + assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(failure)); + assertFalse(endedBeforeClose.get(), "finalization must follow completed in-budget scope cleanup"); + assertEquals(2, ends.get(), "both plugins finalize once with the original fatal"); + } finally { + releaseEarlier.countDown(); + callers.shutdownNow(); + workers.shutdownNow(); + assertTrue(callers.awaitTermination(3, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } + + @HandlerScoped(ScopeOpener.class) + public static final class ScopePlugin implements DurableExecutionPlugin { + private final AutoCloseable closer; + private final AtomicBoolean closed; + private final AtomicBoolean endedBeforeClose; + private final AtomicInteger ends; + private final CountDownLatch endCalled; + private final Error fatal; + + ScopePlugin( + AutoCloseable closer, + AtomicBoolean closed, + AtomicBoolean endedBeforeClose, + AtomicInteger ends, + CountDownLatch endCalled, + Error fatal) { + this.closer = closer; + this.closed = closed; + this.endedBeforeClose = endedBeforeClose; + this.ends = ends; + this.endCalled = endCalled; + this.fatal = fatal; + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + if (!closed.get()) endedBeforeClose.set(true); + if (info.invocationStatus() != InvocationStatus.RETRYING || info.executionError() != fatal) { + endedBeforeClose.set(true); + } + ends.incrementAndGet(); + endCalled.countDown(); + } + } + + public static final class ScopeOpener implements Function { + @Override + public AutoCloseable apply(Object value) { + return ((ScopePlugin) value).closer; + } + } + + private static void await(CountDownLatch latch) { + try { + if (!latch.await(3, TimeUnit.SECONDS)) throw new AssertionError("latch timed out"); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError(failure); + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerMdcIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerMdcIntegrationTest.java index 237e859af..2b9d301ff 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerMdcIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/HandlerMdcIntegrationTest.java @@ -6,11 +6,16 @@ import io.opentelemetry.sdk.trace.SdkTracerProvider; import java.time.Duration; +import java.util.Arrays; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; @@ -18,7 +23,10 @@ import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; @@ -174,6 +182,97 @@ public T deserialize(String data, TypeToken type) { }; } + @ParameterizedTest + @ValueSource(strings = {"success", "failure", "inputFailure"}) + void endHookObservesTaskMdcBeforeReusedWorkerAmbientStateIsRestored(String outcome) throws Exception { + var worker = Executors.newSingleThreadExecutor(); + var callerThread = new AtomicReference(); + var caller = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "mdc-end-caller"); + callerThread.set(thread); + return thread; + }); + var started = new CountDownLatch(1); + var release = new CountDownLatch(1); + var startThread = new AtomicReference(); + var endThread = new AtomicReference(); + var endMdc = new AtomicReference>(); + var ambient = Map.of("worker", "ambient"); + var plugin = new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo info) { + startThread.set(Thread.currentThread()); + MDC.put("start-hook", "request"); + started.countDown(); + awaitMdcLatch(release); + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + endThread.set(Thread.currentThread()); + var values = MDC.getCopyOfContextMap(); + endMdc.set(values == null ? Map.of() : values); + MDC.put("end-hook", "must-not-leak"); + } + }; + try { + worker.submit(() -> MDC.setContextMap(ambient)).get(3, TimeUnit.SECONDS); + var config = DurableConfig.builder() + .withExecutorService(worker) + .withPlugins(info -> plugin) + .withSerDes(outcome.equals("inputFailure") ? failingInputSerDes() : new JacksonSerDes()) + .build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + if (outcome.equals("failure")) throw new IllegalStateException("body failure"); + return "done"; + }, + config); + var result = caller.submit(() -> runner.run("input")); + assertTrue(started.await(3, TimeUnit.SECONDS)); + awaitMdcCallerJoin(callerThread.get()); + release.countDown(); + assertEquals( + outcome.equals("success") ? ExecutionStatus.SUCCEEDED : ExecutionStatus.FAILED, + result.get(5, TimeUnit.SECONDS).getStatus()); + assertSame(startThread.get(), endThread.get()); + // Input failure skips DurableLogger; entered handlers retain its established MDC-clearing behavior. + assertEquals( + outcome.equals("inputFailure") ? Map.of("worker", "ambient", "start-hook", "request") : Map.of(), + endMdc.get(), + "End hooks must not see prematurely restored ambient worker MDC"); + assertEquals(ambient, worker.submit(MDC::getCopyOfContextMap).get(3, TimeUnit.SECONDS)); + } finally { + release.countDown(); + caller.shutdownNow(); + worker.shutdownNow(); + assertTrue(caller.awaitTermination(3, TimeUnit.SECONDS)); + assertTrue(worker.awaitTermination(3, TimeUnit.SECONDS)); + } + } + + private static void awaitMdcLatch(CountDownLatch latch) { + try { + assertTrue(latch.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new AssertionError(error); + } + } + + private static void awaitMdcCallerJoin(Thread caller) { + var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (System.nanoTime() < deadline) { + if (caller.getState() == Thread.State.WAITING + && Arrays.stream(caller.getStackTrace()) + .anyMatch(frame -> frame.getClassName().equals(CompletableFuture.class.getName()) + && frame.getMethodName().equals("join"))) return; + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + } + fail("Caller did not attach its completion observer before releasing the handler"); + } + private static void assertRestored( ExecutorService executor, Map workerMdc, Map callerMdc) throws Exception { var callerAfter = MDC.getCopyOfContextMap(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ScopeFatalSignalTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ScopeFatalSignalTest.java new file mode 100644 index 000000000..b5da1ea7b --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/ScopeFatalSignalTest.java @@ -0,0 +1,93 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.HandlerScoped; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class ScopeFatalSignalTest { + @ParameterizedTest + @CsvSource({"false,false", "true,false", "false,true", "true,true"}) + @SuppressWarnings("removal") + void fatalCloseWakesCallerAndStillAllowsBoundedEarlierCleanup(boolean threadDeath, boolean releaseInBudget) + throws Exception { + var earlierEntered = new CountDownLatch(1); + var release = new CountDownLatch(1); + var closed = new AtomicBoolean(); + var ownerFatal = new AtomicReference(); + var ownerObserved = new CountDownLatch(1); + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("later scope close"); + var workers = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "scope-fatal-owner"); + thread.setUncaughtExceptionHandler((owner, failure) -> { + ownerFatal.set(failure); + ownerObserved.countDown(); + }); + return thread; + }); + var callers = Executors.newSingleThreadExecutor(); + try { + var earlier = new Scoped(() -> { + earlierEntered.countDown(); + if (!release.await(5, TimeUnit.SECONDS)) throw new AssertionError("release timed out"); + closed.set(true); + }); + var later = new Scoped(() -> { + throw fatal; + }); + var config = DurableConfig.builder() + .withExecutorService(workers) + .withPlugins(info -> earlier, info -> later) + .build(); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config); + var result = callers.submit(() -> runner.run("input")); + assertTrue(earlierEntered.await(3, TimeUnit.SECONDS)); + assertThrows( + TimeoutException.class, + () -> result.get(100, TimeUnit.MILLISECONDS), + "earlier cleanup must receive its handoff budget"); + if (releaseInBudget) release.countDown(); + var failure = assertThrows( + ExecutionException.class, + () -> result.get(2, TimeUnit.SECONDS), + "reported fatal must wake the invocation without waiting forever for earlier cleanup"); + assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(failure)); + assertEquals(releaseInBudget, closed.get()); + } finally { + release.countDown(); + callers.shutdown(); + workers.shutdown(); + assertTrue(callers.awaitTermination(3, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + assertTrue(ownerObserved.await(3, TimeUnit.SECONDS)); + assertSame(fatal, ownerFatal.get(), "the fatal still escapes its actual owner"); + } + + @HandlerScoped(Opener.class) + public static final class Scoped implements DurableExecutionPlugin { + private final AutoCloseable closer; + + Scoped(AutoCloseable closer) { + this.closer = closer; + } + } + + public static final class Opener implements Function { + @Override + public AutoCloseable apply(Object plugin) { + return ((Scoped) plugin).closer; + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ApiRequestDelayedBatcher.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ApiRequestDelayedBatcher.java index 50f5248c9..9488a7947 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ApiRequestDelayedBatcher.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ApiRequestDelayedBatcher.java @@ -6,8 +6,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -39,6 +41,8 @@ public class ApiRequestDelayedBatcher { private final Supplier pluginFatal; private final AtomicReference terminalFailure = new AtomicReference<>(); + private final CompletableFuture abortSignal = new CompletableFuture<>(); + private final Set> activeRequests = ConcurrentHashMap.newKeySet(); /** Accumulated requests to be executed in future */ private final List> delayedBatch; @@ -122,6 +126,8 @@ CompletableFuture submit(T request, Duration flushDelay) { /** Flushes pending batch and waits for completion */ void shutdown() { + var observedFatal = getFatalFailure(); + if (observedFatal != null) abortPending(observedFatal); synchronized (delayedBatch) { // cancel the flush timer if it has not been triggered this.delayedBatchFlushTimer.cancel(false); @@ -130,7 +136,8 @@ void shutdown() { } // wait for previous batches to be flushed - flushingQueueFuture.join(); + // A terminal invocation failure must not wait for an already admitted backend call to return. + CompletableFuture.anyOf(flushingQueueFuture, abortSignal).join(); var fatal = getFatalFailure(); if (fatal != null) throw new CompletionException(fatal); } @@ -215,7 +222,11 @@ private void flushQueue() { } // An external fatal publisher can drain the queue between the peek and this poll. - var next = flushingQueue.poll(); + Item next; + synchronized (delayedBatch) { + next = flushingQueue.poll(); + if (next != null) activeRequests.add(next.result()); + } if (next == null) break; flushingItems.add(next); flushingSize += itemSizeInByte; @@ -244,6 +255,8 @@ private void flushQueue() { abortPending(fatal); throw fatal; } + } finally { + flushingItems.forEach(item -> activeRequests.remove(item.result())); } } } @@ -259,18 +272,21 @@ private void rethrowPluginFatalIfPresent() { if (fatal != null) throw fatal; } - /** Settles delayed and queued requests without waiting for their timer or an in-flight backend call. */ + /** Settles delayed, queued, and active request futures without interrupting an admitted backend call. */ void abortPending(Error fatal) { terminalFailure.compareAndSet(null, fatal); var original = terminalFailure.get(); - var pending = new ArrayList>(); + var pending = new ArrayList>(); synchronized (delayedBatch) { delayedBatchFlushTimer.cancel(false); - pending.addAll(delayedBatch); + delayedBatch.forEach(item -> pending.add(item.result())); delayedBatch.clear(); Item item; - while ((item = flushingQueue.poll()) != null) pending.add(item); + while ((item = flushingQueue.poll()) != null) pending.add(item.result()); + // Queue removal and active registration share this lock, so an item cannot escape the abort. + pending.addAll(activeRequests); } - pending.forEach(item -> item.result().completeExceptionally(original)); + pending.forEach(future -> future.completeExceptionally(original)); + abortSignal.completeExceptionally(original); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java index e7196342d..b5d81ee8c 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java @@ -44,6 +44,7 @@ class CheckpointManager { private final DurableConfig config; private final BooleanSupplier tryStartCheckpointProcessing; private final Runnable finishCheckpointProcessing; + private final Object checkpointLock = new Object(); private String checkpointToken; CheckpointManager( @@ -191,8 +192,10 @@ void shutdown() { pollingFutures.clear(); } + var fatal = checkpointApiRequestDelayedBatcher.getFatalFailure(); + Throwable shutdownFailure = fatal != null ? fatal : new IllegalStateException("CheckpointManager shutdown"); for (var futures : allFutures) { - futures.forEach(f -> f.completeExceptionally(new IllegalStateException("CheckpointManager shutdown"))); + futures.forEach(f -> f.completeExceptionally(shutdownFailure)); } // wait for all non-polling checkpoint requests to complete @@ -231,7 +234,8 @@ List fetchAllPages(CheckpointUpdatedExecutionState checkpointUpdatedE } private void checkpointBatch(List updates) { - synchronized (pollingFutures) { + // Backend serialization must not hold the poller registry lock needed by terminal shutdown. + synchronized (checkpointLock) { // filter the null values from pollers var request = updates.stream().filter(Objects::nonNull).toList(); @@ -276,7 +280,10 @@ private void checkpointBatch(List updates) { // complete the registered pollingFutures for (var operation : operations) { - var pollers = pollingFutures.remove(operation.id()); + List> pollers; + synchronized (pollingFutures) { + pollers = pollingFutures.remove(operation.id()); + } if (pollers != null) { completedFutures += pollers.size(); pollers.forEach(poller -> poller.complete(operation)); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index 407d21d94..936d40ba4 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -294,13 +294,24 @@ private static CompletableFuture supplyAsync( var caller = Thread.currentThread(); Runnable work = (Runnable & CompletableFuture.AsynchronousCompletionTask) () -> { try { - T value; - // Preserve the instrumented owner's context even when startup fails. Without plugins, retain - // the existing worker cleanup without adding MDC snapshot reads. - try (SafeCloseable ignored = preservePluginMdc ? restoreMdcOnClose() : MDC::clear) { - value = task.get(); + if (!preservePluginMdc) { + T value; + try (SafeCloseable ignored = MDC::clear) { + value = task.get(); + } + result.complete(value); + return; + } + // Preserve worker MDC through plugin startup, handler work, and completion callbacks. + try (SafeCloseable ignored = restoreMdcOnClose()) { + try { + result.complete(task.get()); + } catch (Throwable failure) { + // End hooks triggered by this completion must observe the task's MDC before restoration. + result.completeExceptionally(failure); + throw failure; + } } - result.complete(value); } catch (Throwable failure) { result.completeExceptionally(failure); // A direct executor is already on the invocation caller; its fatal result is rethrown below after @@ -363,7 +374,8 @@ private static R finalizeAfterHandlerScopes( // Only the caller waits: a signaling handler must be free to unwind and close its scopes. awaitHandlerScopes( executionFuture, handlerFuture, hasHandlerScope, lambdaContext, pluginCount, pluginFatal); - finish.run(); + // A completed handler owns its registered end callback even if the caller wakes first. + if (!handlerFuture.isDone()) finish.run(); } return finalized.join(); } @@ -391,8 +403,10 @@ static CompletableFuture awaitHandlerScopes( // instrumentation. var failure = executionFuture.handle((result, error) -> error).join(); var fatal = pluginFatal.get(); - if (fatal != null) return CompletableFuture.failedFuture(fatal); - if (failure == null || !hasHandlerScope.get()) return executionFuture; + // A later scope can fail before earlier scopes finish closing. Preserve their existing cleanup budget. + if (!hasHandlerScope.get() || (failure == null && fatal == null)) { + return fatal == null ? executionFuture : CompletableFuture.failedFuture(fatal); + } var reserve = SHUTDOWN_RESPONSE_RESERVE_MILLIS + PLUGIN_FINALIZATION_RESERVE_MILLIS * pluginCount; var budgetMillis = lambdaContext == null ? 500L diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index 4ca9a9d58..006992fe5 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -743,6 +743,8 @@ private void failFromPlugin(Error fatal) { /** Stops pending work while leaving the root worker responsible for completing its scope cleanup. */ void recordHandlerScopeFatal(Error fatal) { var original = recordPluginFatal(fatal, true); + // Wake the invocation caller so it can apply the bounded handoff while the owner continues cleanup. + executionExceptionFuture.completeExceptionally(original); stopAllOperations(original); checkpointManager.abortPending(original); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/HandlerScopeHandoffTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/HandlerScopeHandoffTest.java index 0f107f043..f90436fbb 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/HandlerScopeHandoffTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/HandlerScopeHandoffTest.java @@ -9,10 +9,15 @@ import java.time.Duration; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -85,6 +90,106 @@ void noScopeDoesNotConsultTheRemainingTimeOrWaitForTheHandler() { assertFalse(handler.isDone()); } + @Test + void knownFatalStillGivesEarlierScopeCleanupItsHandoffBudget() throws Exception { + var fatal = new InternalError("later scope failed"); + var handler = new CompletableFuture(); + var callers = Executors.newSingleThreadExecutor(); + try { + var result = callers.submit(() -> DurableExecutor.awaitHandlerScopes( + CompletableFuture.failedFuture(fatal), + handler, + new AtomicBoolean(true), + null, + 2, + new AtomicReference(fatal))); + assertThrows(TimeoutException.class, () -> result.get(100, TimeUnit.MILLISECONDS)); + handler.completeExceptionally(fatal); + var chosen = result.get(1, TimeUnit.SECONDS); + assertSame( + fatal, assertThrows(CompletionException.class, chosen::join).getCause()); + } finally { + handler.completeExceptionally(fatal); + callers.shutdownNow(); + assertTrue(callers.awaitTermination(3, TimeUnit.SECONDS)); + } + } + + @Test + void completedHandoffWaitsForTheRegisteredOwnerFinalizer() throws Exception { + var callbackRegistered = new CountDownLatch(1); + var callbacks = new LinkedBlockingQueue(); + var handler = new CompletableFuture() { + @Override + public CompletableFuture whenComplete(BiConsumer action) { + callbackRegistered.countDown(); + // A completed future may publish its result before its registered completion action executes. + return super.whenCompleteAsync(action, callbacks::add); + } + }; + var local = new ThreadLocal(); + var owner = new AtomicReference(); + var finalizerThread = new AtomicReference(); + var finalizerValue = new AtomicReference(); + var workers = Executors.newSingleThreadExecutor(); + var callers = Executors.newSingleThreadExecutor(); + var method = DurableExecutor.class.getDeclaredMethod( + "finalizeAfterHandlerScopes", + CompletableFuture.class, + CompletableFuture.class, + AtomicBoolean.class, + Context.class, + int.class, + AtomicReference.class, + BiFunction.class); + method.setAccessible(true); + BiFunction end = (value, failure) -> { + finalizerThread.set(Thread.currentThread()); + finalizerValue.set(local.get()); + local.remove(); + return "pending"; + }; + try { + var result = callers.submit(() -> method.invoke( + null, + CompletableFuture.failedFuture(control(false)), + handler, + new AtomicBoolean(true), + null, + 2, + new AtomicReference(), + end)); + assertTrue(callbackRegistered.await(3, TimeUnit.SECONDS)); + workers.submit(() -> { + owner.set(Thread.currentThread()); + local.set("invocation"); + handler.complete("handler finished"); + }) + .get(3, TimeUnit.SECONDS); + assertTrue(handler.isDone()); + assertThrows( + TimeoutException.class, + () -> result.get(100, TimeUnit.MILLISECONDS), + "The waiter must not steal a completed handoff's pending completion callback"); + var callback = callbacks.poll(3, TimeUnit.SECONDS); + assertNotNull(callback); + workers.submit(callback).get(3, TimeUnit.SECONDS); + assertEquals("pending", result.get(3, TimeUnit.SECONDS)); + assertSame(owner.get(), finalizerThread.get()); + assertEquals("invocation", finalizerValue.get()); + assertNull(workers.submit(local::get).get(3, TimeUnit.SECONDS), "The reused owner must be clean"); + } finally { + handler.complete("cleanup"); + Runnable callback; + while ((callback = callbacks.poll()) != null) + workers.submit(callback).get(3, TimeUnit.SECONDS); + callers.shutdownNow(); + workers.shutdownNow(); + assertTrue(callers.awaitTermination(3, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } + private static CompletableFuture handoff( CompletableFuture execution, CompletableFuture handler) { return DurableExecutor.awaitHandlerScopes( diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/InFlightFatalAbortTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/InFlightFatalAbortTest.java new file mode 100644 index 000000000..b6b7d5b40 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/InFlightFatalAbortTest.java @@ -0,0 +1,105 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.execution; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +import java.time.Duration; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.CheckpointDurableExecutionResponse; +import software.amazon.awssdk.services.lambda.model.OperationAction; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.awssdk.services.lambda.model.OperationUpdate; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.client.DurableExecutionClient; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class InFlightFatalAbortTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @SuppressWarnings("removal") + void activeBatchAndAlreadyWaitingShutdownObserveFatalWithoutBackendReturn(boolean threadDeath) throws Exception { + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("active batch fatal"); + var batcher = new ApiRequestDelayedBatcher(10, 100, String::length, values -> block(entered, release)); + var closing = Executors.newSingleThreadExecutor(); + var request = batcher.submit("START", Duration.ZERO); + try { + assertTrue(entered.await(3, TimeUnit.SECONDS)); + var shutdown = closing.submit(batcher::shutdown); + assertThrows(TimeoutException.class, () -> shutdown.get(100, TimeUnit.MILLISECONDS)); + batcher.abortPending(fatal); + assertFatal(request, fatal); + assertFatal(shutdown, fatal); + assertEquals(1, release.getCount(), "fatal handling must not need the backend to return"); + assertFatal(batcher.submit("later", Duration.ZERO), fatal); + } finally { + release.countDown(); + closing.shutdown(); + assertTrue(closing.awaitTermination(3, TimeUnit.SECONDS)); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @SuppressWarnings("removal") + void checkpointShutdownDoesNotWaitForBackendHeldPollerLockAfterFatal(boolean threadDeath) throws Exception { + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("checkpoint fatal"); + var signal = new AtomicReference(); + var client = mock(DurableExecutionClient.class); + when(client.checkpoint(any(), any(), any())).thenAnswer(call -> { + block(entered, release); + return CheckpointDurableExecutionResponse.builder() + .checkpointToken("next") + .build(); + }); + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withCheckpointDelay(Duration.ZERO) + .build(); + var manager = new CheckpointManager(config, "arn", "token", ops -> {}, () -> true, () -> {}, signal::get); + var closing = Executors.newSingleThreadExecutor(); + var request = manager.checkpoint(OperationUpdate.builder() + .id("step") + .type(OperationType.STEP) + .action(OperationAction.START) + .build()); + try { + assertTrue(entered.await(3, TimeUnit.SECONDS)); + var shutdown = closing.submit(manager::shutdown); + assertThrows(TimeoutException.class, () -> shutdown.get(100, TimeUnit.MILLISECONDS)); + signal.set(fatal); + manager.abortPending(fatal); + assertFatal(request, fatal); + assertFatal(shutdown, fatal); + assertEquals(1, release.getCount()); + } finally { + release.countDown(); + closing.shutdown(); + assertTrue(closing.awaitTermination(3, TimeUnit.SECONDS)); + } + } + + private static void assertFatal(Future result, Error fatal) { + var failure = assertThrows(ExecutionException.class, () -> result.get(1, TimeUnit.SECONDS)); + assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(failure)); + } + + private static void block(CountDownLatch entered, CountDownLatch release) { + entered.countDown(); + try { + if (!release.await(5, TimeUnit.SECONDS)) throw new AssertionError("backend release timed out"); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError("backend must not be interrupted", failure); + } + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/PluginFatalCheckpointAbortTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/PluginFatalCheckpointAbortTest.java index e52d3704e..1745af0cc 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/PluginFatalCheckpointAbortTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/PluginFatalCheckpointAbortTest.java @@ -144,8 +144,11 @@ void scopeFatalAbortsCheckpointsAndPollersWithoutCompletingRootCleanupEarly() th assertAll( () -> assertFalse(admitted, "a published fatal must reject backend admission during scope cleanup"), () -> assertDoesNotThrow(() -> manager.deregisterActiveThread(null)), - () -> assertFalse(outcome.isDone(), "a fatal must not become a suspension before owner cleanup")); - assertFalse(outcome.isDone(), "the root must retain ownership of its remaining scope cleanup"); + () -> assertSame( + fatal, + ExceptionHelper.unwrapAsyncFailure( + assertThrows(CompletionException.class, outcome::join)))); + assertFalse(owner.isDone(), "the root must retain ownership of its remaining scope cleanup"); owner.completeExceptionally(fatal); assertSame( fatal, ExceptionHelper.unwrapAsyncFailure(assertThrows(CompletionException.class, outcome::join))); From 9333fd47d2618c8cd635a91ca77cfdc855046b76 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:19:39 -0700 Subject: [PATCH 09/17] fix: bound fatal peer cleanup and contain malformed failure chains --- .../durable/PluginFatalRunningPeerTest.java | 122 +++++++++++ .../durable/execution/DurableExecutor.java | 9 +- .../durable/execution/ExecutionManager.java | 41 +++- .../operation/BaseDurableOperation.java | 26 +++ .../lambda/durable/plugin/PluginRunner.java | 28 ++- .../lambda/durable/util/ExceptionHelper.java | 69 +++++-- .../OperationOwnerInterruptionTest.java | 116 +++++++++++ .../durable/util/UnusualAsyncFailureTest.java | 191 ++++++++++++++++++ 8 files changed, 571 insertions(+), 31 deletions(-) create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalRunningPeerTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/operation/OperationOwnerInterruptionTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/util/UnusualAsyncFailureTest.java diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalRunningPeerTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalRunningPeerTest.java new file mode 100644 index 000000000..2adfe7ddb --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalRunningPeerTest.java @@ -0,0 +1,122 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class PluginFatalRunningPeerTest { + @ParameterizedTest + @CsvSource({"false,false,true", "true,false,true", "false,true,true", "true,true,true", "false,false,false"}) + @SuppressWarnings("removal") + void runningPeerCleanupIsBoundedOnlyAfterPluginFatal( + boolean threadDeath, boolean ignoresInterrupt, boolean triggerFatal) throws Exception { + var peerEntered = new CountDownLatch(1); + var releasePeer = new CountDownLatch(1); + var interrupted = new AtomicBoolean(); + var exited = new AtomicBoolean(); + var ends = new AtomicInteger(); + var manager = new AtomicReference(); + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("peer hook fatal"); + DurableExecutionPluginFactory fault = info -> new DurableExecutionPlugin() { + @Override + public void onUserFunctionStart(UserFunctionStartInfo start) { + if ("trigger".equals(start.name())) throw fatal; + } + + @Override + public void onInvocationEnd(InvocationEndInfo end) { + ends.incrementAndGet(); + } + }; + var workers = Executors.newCachedThreadPool(task -> { + var thread = new Thread(task, "running-peer-worker"); + thread.setDaemon(true); + return thread; + }); + var callers = Executors.newSingleThreadExecutor(); + try { + var config = DurableConfig.builder() + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(fault) + .build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + manager.set(((DurableContextImpl) context).getExecutionManager()); + var peer = context.stepAsync("peer", String.class, step -> { + peerEntered.countDown(); + try { + while (releasePeer.getCount() != 0) { + try { + releasePeer.await(); + } catch (InterruptedException error) { + interrupted.set(true); + if (!ignoresInterrupt) return "interrupted"; + } + } + return "released"; + } finally { + exited.set(true); + } + }); + await(peerEntered); + if (triggerFatal) { + context.stepAsync("trigger", String.class, step -> "unreachable"); + return peer.get(); + } + return input; + }, + config); + var response = callers.submit(() -> runner.run("input")); + assertTrue(peerEntered.await(3, TimeUnit.SECONDS)); + if (triggerFatal) { + var failure = assertThrows(ExecutionException.class, () -> response.get(2, TimeUnit.SECONDS)); + assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(failure)); + assertTrue(interrupted.get(), "the running peer receives cooperative cancellation"); + assertEquals( + !ignoresInterrupt, + exited.get(), + "uncooperative work must not hold invocation shutdown forever"); + } else { + assertThrows(TimeoutException.class, () -> response.get(100, TimeUnit.MILLISECONDS)); + assertFalse(interrupted.get(), "ordinary shutdown must not interrupt user work"); + releasePeer.countDown(); + assertEquals( + ExecutionStatus.SUCCEEDED, + response.get(2, TimeUnit.SECONDS).getStatus()); + assertTrue(exited.get()); + } + assertEquals(1, ends.get()); + manager.get().getPluginRunner().onInvocationEnd(null); + assertEquals(1, ends.get(), "the completed invocation must release its plugin instances"); + } finally { + releasePeer.countDown(); + workers.shutdownNow(); + callers.shutdownNow(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + assertTrue(callers.awaitTermination(3, TimeUnit.SECONDS)); + } + } + + private static void await(CountDownLatch latch) { + try { + if (!latch.await(3, TimeUnit.SECONDS)) throw new AssertionError("peer not started"); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new AssertionError(error); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index 936d40ba4..639e85db7 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -326,14 +326,7 @@ private static CompletableFuture supplyAsync( } private static Throwable normalizeInvocationFailure(Throwable failure) { - var unwrapped = ExceptionHelper.unwrapAsyncFailure(failure); - if (isFatal(unwrapped)) return unwrapped; - // ExecutionException can be an application failure itself. Only CompletionException is the SDK's - // ordinary transport here; retain the application's error type, message, and serialized cause chain. - while (failure instanceof CompletionException && failure.getCause() != null) { - failure = failure.getCause(); - } - return failure; + return ExceptionHelper.unwrapInvocationFailure(failure); } @SuppressWarnings("removal") diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index 006992fe5..6f18952e4 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -18,6 +18,8 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -77,6 +79,7 @@ public class ExecutionManager implements SafeCloseable { // factory when the invocation starts, and releases them in close(), so instances never outlive the invocation. private final PluginRunner pluginRunner; private final AtomicReference pluginFatal; + private final CompletableFuture pluginFatalSignal = new CompletableFuture<>(); private final Object pluginFinalizationLock = new Object(); private boolean invocationFinalized; private Error lateScopeFatal; @@ -664,6 +667,7 @@ public void close() { } private void validateRunningThreads() { + var cleanupDeadline = Long.MAX_VALUE; // This will detect stuck user thread and thread leaks in the thread pool for (BaseDurableOperation op : registeredOperations.values()) { var userHandlerFuture = op.getRunningUserHandler(); @@ -672,9 +676,23 @@ private void validateRunningThreads() { // the operations that run them have never been waiting for and the execution has completed. logger.info("Waiting for operation to complete before shutting down: {}", op.getOperationId()); try { - userHandlerFuture.get(); + if (pluginFatal.get() == null) { + CompletableFuture.anyOf(userHandlerFuture, pluginFatalSignal) + .handle((result, failure) -> null) + .get(); + } + cleanupDeadline = fatalCleanupDeadline(cleanupDeadline); + if (cleanupDeadline == Long.MAX_VALUE) userHandlerFuture.get(); + else userHandlerFuture.get(Math.max(0L, cleanupDeadline - System.nanoTime()), TimeUnit.NANOSECONDS); + } catch (TimeoutException e) { + logger.warn("Operation owner exceeded the fatal cleanup budget: {}", op.getOperationId()); + break; } catch (InterruptedException | CancellationException e) { - // if the user handler is stuck + if (pluginFatal.get() != null) { + if (e instanceof InterruptedException) + Thread.currentThread().interrupt(); + break; + } throw new IllegalStateException( "Stuck running user handler when shutting down: " + op.getOperationId()); } catch (Exception e) { @@ -693,6 +711,15 @@ private void validateRunningThreads() { } } + private long fatalCleanupDeadline(long current) { + if (current != Long.MAX_VALUE || pluginFatal.get() == null) return current; + // One budget across all peers, with response headroom when the runtime exposes its deadline. + var budgetMillis = lambdaContext == null + ? 500L + : Math.min(500L, Math.max(0L, (long) lambdaContext.getRemainingTimeInMillis() - 1_000L)); + return System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(budgetMillis); + } + /** Returns {@code true} if the given status represents a terminal (final) operation state. */ public static boolean isTerminalStatus(OperationStatus status) { return status == OperationStatus.SUCCEEDED @@ -758,12 +785,15 @@ Error beginInvocationFinalization() { } private Error recordPluginFatal(Error fatal, boolean fromScope) { + Error original; synchronized (pluginFinalizationLock) { if (pluginFatal.compareAndSet(null, fatal) && fromScope && invocationFinalized) { lateScopeFatal = fatal; } - return pluginFatal.get(); + original = pluginFatal.get(); } + pluginFatalSignal.complete(null); + return original; } private boolean isLateScopeFatal(Throwable failure) { @@ -779,7 +809,10 @@ public void rethrowPluginFatalIfPresent() { } private void stopAllOperations(Throwable cause) { - registeredOperations.values().forEach(op -> op.getCompletionFuture().completeExceptionally(cause)); + registeredOperations.values().forEach(op -> { + op.getCompletionFuture().completeExceptionally(cause); + op.interruptRunningUserHandler(); + }); } /** diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java index c5a87d259..fa0c1040b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java @@ -62,6 +62,9 @@ public abstract class BaseDurableOperation { protected final AtomicBoolean replayCompletedOperation = new AtomicBoolean(false); private final DurableContextImpl durableContext; private final AtomicReference> runningUserHandler = new AtomicReference<>(null); + private final Object handlerOwnerLock = new Object(); + private Thread handlerOwner; + private CompletableFuture handlerOwnerCompletion; protected BaseDurableOperation( OperationIdentifier operationIdentifier, @@ -329,14 +332,27 @@ protected void runUserHandler(Runnable runnable, ThreadType threadType) { var completion = new CompletableFuture(); runningUserHandler.set(completion); Runnable observed = (Runnable & CompletableFuture.AsynchronousCompletionTask) () -> { + synchronized (handlerOwnerLock) { + handlerOwner = Thread.currentThread(); + handlerOwnerCompletion = completion; + } try { wrapped.run(); + executionManager.rethrowPluginFatalIfPresent(); completion.complete(null); } catch (Throwable failure) { completion.completeExceptionally(failure); // Settle accepted work before propagating on either an async worker or the direct caller. The // invocation boundary still finalizes plugins, but the operation call must not return normally. executionManager.rethrowPluginFatalIfPresent(); + } finally { + synchronized (handlerOwnerLock) { + // A completion callback can start the next attempt before this runnable exits. + if (handlerOwnerCompletion == completion) { + handlerOwner = null; + handlerOwnerCompletion = null; + } + } } }; try { @@ -558,6 +574,16 @@ protected void validateReplay(Operation checkpointed) { } } + /** + * Requests cooperative owner interruption after an invocation has failed fatally. The actual completion future is + * left pending until the owner unwinds. Synchronization prevents interrupting a worker after it has been reused. + */ + public void interruptRunningUserHandler() { + synchronized (handlerOwnerLock) { + if (handlerOwner != null && handlerOwner != Thread.currentThread()) handlerOwner.interrupt(); + } + } + public CompletableFuture getRunningUserHandler() { return runningUserHandler.get(); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java index 94b42e528..ff2fb77ca 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java @@ -212,7 +212,26 @@ private static void contain(Throwable t, String message) { if (cause instanceof ThreadDeath fatal) { throw fatal; } - logger.warn(message, t); + try { + logger.warn(message, t); + } catch (Throwable loggingFailure) { + rethrowDiagnosticFatal(loggingFailure); + try { + logger.warn( + "{} ({}; exception details unavailable)", + message, + t.getClass().getName()); + } catch (Throwable fallbackFailure) { + rethrowDiagnosticFatal(fallbackFailure); + } + } + } + + @SuppressWarnings("removal") + private static void rethrowDiagnosticFatal(Throwable failure) { + var cause = ExceptionHelper.unwrapAsyncFailure(failure); + if (cause instanceof VirtualMachineError fatal) throw fatal; + if (cause instanceof ThreadDeath fatal) throw fatal; } /** @@ -288,13 +307,12 @@ private static void closeHandlerScopes(ArrayDeque scopes, Consume @SuppressWarnings("removal") private static void reportHandlerScopeFailure(String message, Throwable failure, Consumer onScopeFatal) { - var cause = ExceptionHelper.unwrapAsyncFailure(failure); - if (cause instanceof VirtualMachineError || cause instanceof ThreadDeath) { - var fatal = (Error) cause; + try { + contain(failure, message); + } catch (VirtualMachineError | ThreadDeath fatal) { onScopeFatal.accept(fatal); throw fatal; } - logger.warn(message, failure); } public void onInvocationStart(InvocationInfo info) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/util/ExceptionHelper.java b/sdk/src/main/java/software/amazon/lambda/durable/util/ExceptionHelper.java index 87d7dc4ca..e35045ee8 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/util/ExceptionHelper.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/util/ExceptionHelper.java @@ -3,6 +3,8 @@ package software.amazon.lambda.durable.util; import java.util.Arrays; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; @@ -33,24 +35,63 @@ public static void sneakyThrow(Throwable exception) throws * @return the original Throwable that is not a CompletionException */ public static Throwable unwrapCompletableFuture(Throwable throwable) { - while (throwable instanceof CompletionException) { - throwable = throwable.getCause(); - } - return throwable; + return unwrap(throwable, UnwrapMode.COMPLETION); + } + + /** Inspects asynchronous wrappers for fatal causes without looping on cyclic or unreadable cause chains. */ + public static Throwable unwrapAsyncFailure(Throwable failure) { + return unwrap(failure, UnwrapMode.ASYNC); } /** - * Unwraps completion/future wrappers to inspect potentially fatal causes. Callers must retain ordinary application - * {@link ExecutionException} values when reporting or serializing failures. Non-wrapper failures and cause-less - * wrappers keep their identity. - * - * @param failure the failure to inspect, possibly null - * @return the underlying asynchronous failure, or the original non-wrapper failure + * Removes completion transport wrappers while retaining application ExecutionException values, except when an + * asynchronous wrapper contains a fatal JVM error. Each cause accessor is read at most once. */ - public static Throwable unwrapAsyncFailure(Throwable failure) { - while ((failure instanceof CompletionException || failure instanceof ExecutionException) - && failure.getCause() != null) { - failure = failure.getCause(); + public static Throwable unwrapInvocationFailure(Throwable failure) { + return unwrap(failure, UnwrapMode.INVOCATION); + } + + private enum UnwrapMode { + COMPLETION, + ASYNC, + INVOCATION + } + + @SuppressWarnings("removal") + private static Throwable unwrap(Throwable failure, UnwrapMode mode) { + if (!(failure instanceof CompletionException) + && !(mode != UnwrapMode.COMPLETION && failure instanceof ExecutionException)) return failure; + var visited = Collections.newSetFromMap(new IdentityHashMap()); + Throwable applicationWrapper = null; + while (failure instanceof CompletionException + || (mode != UnwrapMode.COMPLETION && failure instanceof ExecutionException)) { + if (!visited.add(failure)) break; + if (mode == UnwrapMode.INVOCATION && applicationWrapper == null && failure instanceof ExecutionException) { + applicationWrapper = failure; + } + Throwable cause; + try { + cause = failure.getCause(); + } catch (VirtualMachineError | ThreadDeath fatal) { + // Fatal inspectors return the fatal so their caller can notify the invocation before rethrowing it. + if (mode == UnwrapMode.COMPLETION) throw fatal; + return fatal; + } catch (Throwable unreadableCause) { + // A malformed diagnostic must not replace the failure that crossed the plugin boundary. + break; + } + if (cause == null) { + // Preserve the released completion-only helper's cause-less-wrapper behavior. + if (mode == UnwrapMode.COMPLETION) return null; + break; + } + failure = cause; + } + if (mode == UnwrapMode.INVOCATION + && applicationWrapper != null + && !(failure instanceof VirtualMachineError) + && !(failure instanceof ThreadDeath)) { + return applicationWrapper; } return failure; } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/OperationOwnerInterruptionTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/OperationOwnerInterruptionTest.java new file mode 100644 index 000000000..c1fddf8bf --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/OperationOwnerInterruptionTest.java @@ -0,0 +1,116 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.operation; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.execution.ThreadContext; +import software.amazon.lambda.durable.execution.ThreadType; +import software.amazon.lambda.durable.model.OperationIdentifier; +import software.amazon.lambda.durable.model.OperationSubType; + +class OperationOwnerInterruptionTest { + @Test + void completingOlderAttemptCannotLoseNewOwnerOrInterruptReusedWorker() throws Exception { + var firstOwner = new AtomicReference(); + var firstExited = new CountDownLatch(1); + var firstRelease = new CountDownLatch(1); + var secondEntered = new CountDownLatch(1); + var secondRelease = new CountDownLatch(1); + var secondInterrupted = new CountDownLatch(1); + var unrelatedEntered = new CountDownLatch(1); + var unrelatedRelease = new CountDownLatch(1); + var unrelatedInterrupted = new AtomicBoolean(); + var workers = new ThreadPoolExecutor(2, 2, 0, TimeUnit.SECONDS, new LinkedBlockingQueue<>()) { + @Override + protected void afterExecute(Runnable task, Throwable failure) { + if (Thread.currentThread() == firstOwner.get()) firstExited.countDown(); + } + }; + try { + var context = mock(DurableContextImpl.class); + var manager = mock(ExecutionManager.class); + when(context.getExecutionManager()).thenReturn(manager); + when(manager.getCurrentThreadContext()).thenReturn(new ThreadContext("op", ThreadType.STEP)); + when(context.getDurableConfig()) + .thenReturn( + DurableConfig.builder().withExecutorService(workers).build()); + var operation = new TestOperation(context); + operation.run(() -> { + firstOwner.set(Thread.currentThread()); + await(firstRelease); + }); + var first = operation.getRunningUserHandler(); + first.whenComplete((ignored, failure) -> { + operation.run(() -> { + secondEntered.countDown(); + try { + secondRelease.await(); + } catch (InterruptedException expected) { + secondInterrupted.countDown(); + } + }); + await(secondEntered); + }); + firstRelease.countDown(); + assertTrue(firstExited.await(3, TimeUnit.SECONDS)); + assertTrue(secondEntered.await(3, TimeUnit.SECONDS)); + var unrelated = workers.submit(() -> { + assertSame(firstOwner.get(), Thread.currentThread()); + unrelatedEntered.countDown(); + try { + unrelatedRelease.await(); + } catch (InterruptedException unexpected) { + unrelatedInterrupted.set(true); + } + }); + assertTrue(unrelatedEntered.await(3, TimeUnit.SECONDS)); + operation.interruptRunningUserHandler(); + assertTrue(secondInterrupted.await(1, TimeUnit.SECONDS), "the second attempt remains the registered owner"); + operation.getRunningUserHandler().get(1, TimeUnit.SECONDS); + operation.interruptRunningUserHandler(); + assertFalse(unrelatedInterrupted.get(), "an idle/completed attempt cannot interrupt a reused worker"); + unrelatedRelease.countDown(); + unrelated.get(1, TimeUnit.SECONDS); + } finally { + firstRelease.countDown(); + secondRelease.countDown(); + unrelatedRelease.countDown(); + workers.shutdownNow(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError(failure); + } + } + + private static final class TestOperation extends BaseDurableOperation { + TestOperation(DurableContextImpl context) { + super(OperationIdentifier.of("op", "op", OperationSubType.STEP), context, null); + } + + @Override + protected void start() {} + + @Override + protected void replay(Operation existing) {} + + void run(Runnable action) { + runUserHandler(action, ThreadType.STEP); + } + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/util/UnusualAsyncFailureTest.java b/sdk/src/test/java/software/amazon/lambda/durable/util/UnusualAsyncFailureTest.java new file mode 100644 index 000000000..c9f6c157f --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/util/UnusualAsyncFailureTest.java @@ -0,0 +1,191 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.util; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.List; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.plugin.*; + +class UnusualAsyncFailureTest { + @Test + void cyclicCauseIsBoundedAndEachAccessorIsReadOnce() { + var first = new CountingWrapper(); + var second = new CountingWrapper(); + first.next = second; + second.next = first; + assertSame(first, ExceptionHelper.unwrapAsyncFailure(first)); + assertEquals(1, first.reads.get()); + assertEquals(1, second.reads.get()); + } + + @ParameterizedTest + @ValueSource(strings = {"runtime", "linkage", "assertion"}) + void unreadableNonfatalCauseDoesNotReplaceThePluginFailure(String type) { + var failure = poisonous(type); + assertSame(failure, ExceptionHelper.unwrapAsyncFailure(failure)); + } + + @ParameterizedTest + @ValueSource(strings = {"runtime", "linkage", "assertion"}) + void hookFailureWithUnreadableCauseStillAllowsHealthyPlugin(String type) { + var healthy = new AtomicInteger(); + var runner = new PluginRunner(List.of( + info -> new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo ignored) { + throw poisonous(type); + } + }, + info -> new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo ignored) { + healthy.incrementAndGet(); + } + })); + assertDoesNotThrow(() -> runner.onInvocationStart(null)); + assertEquals(1, healthy.get()); + } + + @Test + void completionOnlyCyclesAreBoundedAndCauseLessBehaviorIsPreserved() { + var first = new CountingWrapper(); + var second = new CountingWrapper(); + first.next = second; + second.next = first; + assertSame(first, ExceptionHelper.unwrapCompletableFuture(first)); + assertEquals(1, first.reads.get()); + assertEquals(1, second.reads.get()); + var empty = new CompletionException((Throwable) null); + var nested = new CompletionException(empty); + assertNull(ExceptionHelper.unwrapCompletableFuture(nested)); + assertSame(empty, ExceptionHelper.unwrapInvocationFailure(nested)); + } + + @Test + void invocationNormalizationKeepsApplicationWrapperAndReadsEachCauseOnce() { + var business = new IllegalArgumentException("business"); + var checkedReads = new AtomicInteger(); + var checked = new ExecutionException(business) { + @Override + public synchronized Throwable getCause() { + assertEquals(1, checkedReads.incrementAndGet()); + return business; + } + }; + var transport = new CountingWrapper(); + transport.next = checked; + assertSame(checked, ExceptionHelper.unwrapInvocationFailure(transport)); + assertEquals(1, transport.reads.get()); + assertEquals(1, checkedReads.get()); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @SuppressWarnings("removal") + void fatalAccessorIsReportedAtTheScopeBoundary(boolean threadDeath) { + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("cause accessor"); + var wrapper = new CompletionException((Throwable) null) { + @Override + public synchronized Throwable getCause() { + throw fatal; + } + }; + assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(wrapper)); + assertSame( + fatal, ExceptionHelper.unwrapInvocationFailure(new CompletionException(new ExecutionException(fatal)))); + var reported = new AtomicReference(); + var runner = new PluginRunner(List.of(info -> new Scoped(wrapper))); + runner.onInvocationStart(null); + assertSame( + fatal, + assertThrows(Error.class, () -> runner.runHandler(() -> "unreachable", () -> {}, reported::set))); + assertSame(fatal, reported.get()); + } + + @ParameterizedTest + @CsvSource({ + "factory,runtime", + "factory,linkage", + "factory,assertion", + "scope,runtime", + "scope,linkage", + "scope,assertion" + }) + void unreadableCauseIsContainedInFactoryAndScope(String boundary, String type) { + var healthy = new AtomicInteger(); + var wrapper = poisonous(type); + var runner = new PluginRunner(List.of( + info -> { + if (boundary.equals("factory")) throw wrapper; + return new Scoped(wrapper); + }, + info -> new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo ignored) { + healthy.incrementAndGet(); + } + })); + assertDoesNotThrow(() -> runner.onInvocationStart(null)); + assertEquals("result", assertDoesNotThrow(() -> runner.runHandler(() -> "result"))); + assertEquals(1, healthy.get()); + } + + @HandlerScoped(Opener.class) + public static final class Scoped implements DurableExecutionPlugin { + private final CompletionException failure; + + Scoped(CompletionException failure) { + this.failure = failure; + } + } + + public static final class Opener implements Function { + @Override + public AutoCloseable apply(Object plugin) { + return () -> { + throw ((Scoped) plugin).failure; + }; + } + } + + private static CompletionException poisonous(String type) { + return new CompletionException((Throwable) null) { + @Override + public synchronized Throwable getCause() { + switch (type) { + case "linkage": + throw new NoSuchMethodError("cause accessor"); + case "assertion": + throw new AssertionError("cause accessor"); + default: + throw new IllegalStateException("cause accessor"); + } + } + }; + } + + private static final class CountingWrapper extends CompletionException { + private final AtomicInteger reads = new AtomicInteger(); + private Throwable next; + + CountingWrapper() { + super((Throwable) null); + } + + @Override + public synchronized Throwable getCause() { + if (reads.incrementAndGet() > 8) throw new AssertionError("unbounded cause traversal"); + return next; + } + } +} From 58e3740634189a90aa161e2e96ed5436aefe9fcc Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 05:44:26 +0000 Subject: [PATCH 10/17] fix: preserve operation-owner cleanup during fatal cancellation --- .../PluginPeerCleanupRegressionTest.java | 172 ++++++++++++++++++ .../durable/execution/DurableExecutor.java | 1 + .../durable/execution/ExecutionManager.java | 17 +- .../operation/BaseDurableOperation.java | 55 ++++-- .../lambda/durable/plugin/PluginRunner.java | 11 ++ .../OperationOwnerInterruptionTest.java | 45 +++++ 6 files changed, 285 insertions(+), 16 deletions(-) create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginPeerCleanupRegressionTest.java diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginPeerCleanupRegressionTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginPeerCleanupRegressionTest.java new file mode 100644 index 000000000..8f98d37c3 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginPeerCleanupRegressionTest.java @@ -0,0 +1,172 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.util.Arrays; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class PluginPeerCleanupRegressionTest { + @Test + void suspensionDoesNotInterruptChildOwners() throws Exception { + var interruptions = new AtomicInteger(); + var workers = + new ThreadPoolExecutor(0, 16, 1, TimeUnit.SECONDS, new SynchronousQueue<>(), task -> new Thread(task) { + @Override + public void interrupt() { + interruptions.incrementAndGet(); + super.interrupt(); + } + }); + try { + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + var first = context.runInChildContextAsync("first", String.class, child -> { + child.wait("pause", Duration.ofSeconds(10)); + return "a"; + }); + var second = context.runInChildContextAsync("second", String.class, child -> { + child.wait("pause", Duration.ofSeconds(10)); + return "b"; + }); + return first.get() + second.get(); + }, + DurableConfig.builder().withExecutorService(workers).build()); + assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); + assertEquals(0, interruptions.get(), "durable suspension must not interrupt operation owners"); + var result = runner.runUntilComplete("input"); + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("ab", result.getResult(String.class)); + assertEquals(0, interruptions.get()); + } finally { + workers.shutdownNow(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void fatalPeerClosesUserScopeOnOwnerBeforeInvocationEnd(boolean failCleanup) throws Exception { + var peerEntered = new CountDownLatch(1); + var releasePeer = new CountDownLatch(1); + var peerThread = new AtomicReference(); + var localScope = new ThreadLocal(); + var scopeAfterTask = new AtomicReference(); + var observedAfterTask = new CountDownLatch(1); + var peerEnds = new AtomicInteger(); + var healthyEnds = new AtomicInteger(); + var endBeforeInvocation = new AtomicBoolean(); + var wrongOwner = new AtomicBoolean(); + var endError = new AtomicReference(); + var endOutcome = new AtomicReference(); + var fatal = new InternalError("peer fatal"); + var cleanupFatal = new InternalError("cleanup fatal"); + var workers = new ThreadPoolExecutor(0, 16, 1, TimeUnit.SECONDS, new SynchronousQueue<>()) { + @Override + public void execute(Runnable task) { + super.execute(() -> { + try { + task.run(); + } catch (InternalError expected) { + // A custom executor may contain task failures and reuse its worker. + } finally { + if (Thread.currentThread() == peerThread.get()) { + scopeAfterTask.set(localScope.get()); + observedAfterTask.countDown(); + } + } + }); + } + }; + DurableExecutionPluginFactory observer = info -> new DurableExecutionPlugin() { + @Override + public void onUserFunctionStart(UserFunctionStartInfo start) { + if ("peer".equals(start.name())) { + peerThread.set(Thread.currentThread()); + localScope.set(true); + } + if ("trigger".equals(start.name())) throw fatal; + } + + @Override + public void onUserFunctionEnd(UserFunctionEndInfo end) { + if ("peer".equals(end.name())) { + wrongOwner.set(Thread.currentThread() != peerThread.get()); + endError.set(end.error()); + endOutcome.set(end.outcome()); + localScope.remove(); + peerEnds.incrementAndGet(); + if (failCleanup) throw cleanupFatal; + } + } + + @Override + public void onInvocationEnd(InvocationEndInfo end) { + endBeforeInvocation.set(peerEnds.get() == 1); + } + }; + try { + var config = DurableConfig.builder() + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(observer, info -> new DurableExecutionPlugin() { + @Override + public void onUserFunctionEnd(UserFunctionEndInfo end) { + if ("peer".equals(end.name())) { + assertSame(fatal, end.error()); + healthyEnds.incrementAndGet(); + } + } + }) + .build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + var peer = context.stepAsync("peer", String.class, step -> { + peerEntered.countDown(); + try { + if (!releasePeer.await(3, TimeUnit.SECONDS)) + throw new AssertionError("peer not interrupted"); + } catch (InterruptedException interrupted) { + throw new IllegalStateException("cooperative peer", interrupted); + } + return "released"; + }); + try { + assertTrue(peerEntered.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + throw new AssertionError(failure); + } + context.stepAsync("trigger", String.class, step -> "unreachable"); + return peer.get(); + }, + config); + var thrown = assertThrows(Throwable.class, () -> runner.run("input")); + assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(thrown)); + assertTrue(observedAfterTask.await(3, TimeUnit.SECONDS)); + assertEquals(1, peerEnds.get(), "every entered user scope must receive its end hook"); + assertEquals(1, healthyEnds.get(), "one failed cleanup cannot skip the remaining plugins"); + assertFalse(wrongOwner.get()); + assertSame(fatal, endError.get()); + assertEquals(UserFunctionOutcome.FAILED, endOutcome.get()); + assertNull(scopeAfterTask.get(), "the reused executor worker must not retain the attempt scope"); + assertTrue(endBeforeInvocation.get(), "cooperative peer cleanup precedes invocation finalization"); + if (failCleanup) assertTrue(Arrays.asList(fatal.getSuppressed()).contains(cleanupFatal)); + } finally { + releasePeer.countDown(); + workers.shutdownNow(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index 639e85db7..7e4f9759d 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -440,6 +440,7 @@ private static void fireOnInvocationEnd( if (pluginRunner.isEmpty()) { return; } + executionManager.awaitFatalOperationCleanup(); // Freeze the caller outcome immediately before finalization. Observe instrumentation fatals already // reported at this boundary, retaining an earlier invocation fatal. Later scope failures still escape their // owner thread, but cannot rewrite the outcome whose one end snapshot has already been dispatched. diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index 6f18952e4..20ee3f9ef 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -80,6 +80,7 @@ public class ExecutionManager implements SafeCloseable { private final PluginRunner pluginRunner; private final AtomicReference pluginFatal; private final CompletableFuture pluginFatalSignal = new CompletableFuture<>(); + private long fatalOperationCleanupDeadline = Long.MAX_VALUE; private final Object pluginFinalizationLock = new Object(); private boolean invocationFinalized; private Error lateScopeFatal; @@ -670,6 +671,7 @@ private void validateRunningThreads() { var cleanupDeadline = Long.MAX_VALUE; // This will detect stuck user thread and thread leaks in the thread pool for (BaseDurableOperation op : registeredOperations.values()) { + if (pluginFatal.get() != null && op.isRunningUserHandlerOwner()) continue; var userHandlerFuture = op.getRunningUserHandler(); if (userHandlerFuture != null && !userHandlerFuture.isDone()) { // Some user threads can still be running because @@ -711,13 +713,22 @@ private void validateRunningThreads() { } } - private long fatalCleanupDeadline(long current) { + private synchronized long fatalCleanupDeadline(long current) { if (current != Long.MAX_VALUE || pluginFatal.get() == null) return current; + if (fatalOperationCleanupDeadline != Long.MAX_VALUE) return fatalOperationCleanupDeadline; // One budget across all peers, with response headroom when the runtime exposes its deadline. var budgetMillis = lambdaContext == null ? 500L : Math.min(500L, Math.max(0L, (long) lambdaContext.getRemainingTimeInMillis() - 1_000L)); - return System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(budgetMillis); + fatalOperationCleanupDeadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(budgetMillis); + return fatalOperationCleanupDeadline; + } + + /** Gives cooperative operation owners their existing fatal cleanup allowance before plugin finalization. */ + void awaitFatalOperationCleanup() { + if (pluginFatal.get() == null) return; + registeredOperations.values().forEach(BaseDurableOperation::interruptRunningUserHandler); + validateRunningThreads(); } /** Returns {@code true} if the given status represents a terminal (final) operation state. */ @@ -811,7 +822,7 @@ public void rethrowPluginFatalIfPresent() { private void stopAllOperations(Throwable cause) { registeredOperations.values().forEach(op -> { op.getCompletionFuture().completeExceptionally(cause); - op.interruptRunningUserHandler(); + if (pluginFatal.get() != null) op.interruptRunningUserHandler(); }); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java index fa0c1040b..cffa0a30b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java @@ -65,6 +65,7 @@ public abstract class BaseDurableOperation { private final Object handlerOwnerLock = new Object(); private Thread handlerOwner; private CompletableFuture handlerOwnerCompletion; + private CompletableFuture interruptedHandlerCompletion; protected BaseDurableOperation( OperationIdentifier operationIdentifier, @@ -339,20 +340,16 @@ protected void runUserHandler(Runnable runnable, ThreadType threadType) { try { wrapped.run(); executionManager.rethrowPluginFatalIfPresent(); + clearHandlerOwner(completion); completion.complete(null); } catch (Throwable failure) { + clearHandlerOwner(completion); completion.completeExceptionally(failure); // Settle accepted work before propagating on either an async worker or the direct caller. The // invocation boundary still finalizes plugins, but the operation call must not return normally. executionManager.rethrowPluginFatalIfPresent(); } finally { - synchronized (handlerOwnerLock) { - // A completion callback can start the next attempt before this runnable exits. - if (handlerOwnerCompletion == completion) { - handlerOwner = null; - handlerOwnerCompletion = null; - } - } + clearHandlerOwner(completion); } }; try { @@ -367,6 +364,17 @@ protected void runUserHandler(Runnable runnable, ThreadType threadType) { } } + private void clearHandlerOwner(CompletableFuture completion) { + synchronized (handlerOwnerLock) { + // Revoke interruption before publishing completion and invoking callbacks. A callback may start a new + // attempt, so an older runnable's finally must not clear that new owner. + if (handlerOwnerCompletion == completion) { + handlerOwner = null; + handlerOwnerCompletion = null; + } + } + } + /** * Runs a user-provided function inside the plugin user-function hook boundary. * @@ -390,13 +398,18 @@ protected T runUserFunction(Integer attempt, Supplier userFunction) { executionManager.wasObservedAtInvocationStart(getOperationId()), attempt); pluginRunner.onUserFunctionStart(startInfo); + T result; try { - T result = userFunction.get(); - pluginRunner.onUserFunctionEnd( - PluginInfoConverter.toUserFunctionEndInfo(startInfo, UserFunctionOutcome.SUCCEEDED, null)); - return result; - } catch (Throwable e) { + result = userFunction.get(); executionManager.rethrowPluginFatalIfPresent(); + } catch (Throwable e) { + try { + executionManager.rethrowPluginFatalIfPresent(); + } catch (Error fatal) { + pluginRunner.onUserFunctionEndAfterFatal( + PluginInfoConverter.toUserFunctionEndInfo(startInfo, UserFunctionOutcome.FAILED, fatal), fatal); + throw fatal; + } var error = ExceptionHelper.unwrapCompletableFuture(e); if (error == null) { error = e; @@ -408,6 +421,9 @@ protected T runUserFunction(Integer attempt, Supplier userFunction) { ExceptionHelper.sneakyThrow(e); return null; // unreachable — sneakyThrow always throws } + pluginRunner.onUserFunctionEnd( + PluginInfoConverter.toUserFunctionEndInfo(startInfo, UserFunctionOutcome.SUCCEEDED, null)); + return result; } /** @@ -580,7 +596,20 @@ protected void validateReplay(Operation checkpointed) { */ public void interruptRunningUserHandler() { synchronized (handlerOwnerLock) { - if (handlerOwner != null && handlerOwner != Thread.currentThread()) handlerOwner.interrupt(); + if (handlerOwner != null + && handlerOwner != Thread.currentThread() + && !handlerOwnerCompletion.isDone() + && interruptedHandlerCompletion != handlerOwnerCompletion) { + interruptedHandlerCompletion = handlerOwnerCompletion; + handlerOwner.interrupt(); + } + } + } + + /** Whether this caller is the current operation attempt's owner, which must never await itself. */ + public boolean isRunningUserHandlerOwner() { + synchronized (handlerOwnerLock) { + return handlerOwner == Thread.currentThread(); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java index ff2fb77ca..424b456bf 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java @@ -402,4 +402,15 @@ private static void closeStartedUserFunctions( public void onUserFunctionEnd(UserFunctionEndInfo info) { runOperationHook(p -> p.onUserFunctionEnd(info)); } + + /** Completes same-owner attempt cleanup after a peer fatal while retaining the invocation's original failure. */ + public void onUserFunctionEndAfterFatal(UserFunctionEndInfo info, Error original) { + for (var plugin : plugins) { + try { + runPlugin(plugin, p -> p.onUserFunctionEnd(info)); + } catch (Error cleanupFailure) { + if (cleanupFailure != original) original.addSuppressed(cleanupFailure); + } + } + } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/OperationOwnerInterruptionTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/OperationOwnerInterruptionTest.java index c1fddf8bf..021cd464a 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/OperationOwnerInterruptionTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/OperationOwnerInterruptionTest.java @@ -89,6 +89,51 @@ protected void afterExecute(Runnable task, Throwable failure) { } } + @Test + void completedAttemptIsNotInterruptedWhileItsCompletionCallbackIsStillRunning() throws Exception { + var releaseBody = new CountDownLatch(1); + var callbackEntered = new CountDownLatch(1); + var releaseCallback = new CountDownLatch(1); + var callbackExited = new CountDownLatch(1); + var callbackInterrupted = new AtomicBoolean(); + var workers = Executors.newSingleThreadExecutor(); + try { + var context = mock(DurableContextImpl.class); + var manager = mock(ExecutionManager.class); + when(context.getExecutionManager()).thenReturn(manager); + when(manager.getCurrentThreadContext()).thenReturn(new ThreadContext("op", ThreadType.STEP)); + when(context.getDurableConfig()) + .thenReturn( + DurableConfig.builder().withExecutorService(workers).build()); + var operation = new TestOperation(context); + operation.run(() -> await(releaseBody)); + var completion = operation.getRunningUserHandler(); + completion.whenComplete((ignored, failure) -> { + callbackEntered.countDown(); + try { + releaseCallback.await(); + } catch (InterruptedException unexpected) { + callbackInterrupted.set(true); + } finally { + callbackExited.countDown(); + } + }); + releaseBody.countDown(); + assertTrue(callbackEntered.await(3, TimeUnit.SECONDS)); + assertTrue(completion.isDone()); + operation.interruptRunningUserHandler(); + releaseCallback.countDown(); + assertTrue(callbackExited.await(3, TimeUnit.SECONDS)); + workers.submit(() -> {}).get(3, TimeUnit.SECONDS); + assertFalse(callbackInterrupted.get(), "completed ownership must not authorize a later interrupt"); + } finally { + releaseBody.countDown(); + releaseCallback.countDown(); + workers.shutdownNow(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } + private static void await(CountDownLatch latch) { try { assertTrue(latch.await(3, TimeUnit.SECONDS)); From 78373abd264dea4c4c5ee5b2cc7c8982639c71af Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 06:01:37 +0000 Subject: [PATCH 11/17] fix: reject user function admission after a peer plugin fatal --- .../PluginFatalStartAdmissionTest.java | 98 +++++++++++++++++++ .../operation/BaseDurableOperation.java | 1 + 2 files changed, 99 insertions(+) create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalStartAdmissionTest.java diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalStartAdmissionTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalStartAdmissionTest.java new file mode 100644 index 000000000..cc93fc982 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/PluginFatalStartAdmissionTest.java @@ -0,0 +1,98 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class PluginFatalStartAdmissionTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + @SuppressWarnings("removal") + void peerFatalDuringStartHookPreventsLaterUserBody(boolean threadDeath) throws Exception { + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("peer fatal during start"); + var startEntered = new CountDownLatch(1); + var releaseStart = new CountDownLatch(1); + var peerExited = new CountDownLatch(1); + var peerOwner = new AtomicReference(); + var bodyCalls = new AtomicInteger(); + var workers = new ThreadPoolExecutor(0, 16, 1, TimeUnit.SECONDS, new SynchronousQueue<>()) { + @Override + public void execute(Runnable task) { + super.execute(() -> { + try { + task.run(); + } catch (VirtualMachineError | ThreadDeath expected) { + // The caller must receive the same fatal; keep this custom executor reusable for observation. + } finally { + if (Thread.currentThread() == peerOwner.get()) peerExited.countDown(); + } + }); + } + }; + var callers = Executors.newSingleThreadExecutor(); + DurableExecutionPluginFactory plugin = info -> new DurableExecutionPlugin() { + @Override + public void onUserFunctionStart(UserFunctionStartInfo start) { + if ("blocked".equals(start.name())) { + peerOwner.set(Thread.currentThread()); + startEntered.countDown(); + boolean interrupted = false; + while (releaseStart.getCount() != 0) { + try { + releaseStart.await(); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + if (interrupted) Thread.currentThread().interrupt(); + } + if ("trigger".equals(start.name())) throw fatal; + } + }; + try { + var config = DurableConfig.builder() + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(plugin) + .build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + var blocked = context.stepAsync("blocked", String.class, step -> { + bodyCalls.incrementAndGet(); + return "side effect"; + }); + try { + assertTrue(startEntered.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + throw new AssertionError(failure); + } + context.stepAsync("trigger", String.class, step -> "unreachable"); + return blocked.get(); + }, + config); + var result = callers.submit(() -> runner.run("input")); + var failure = assertThrows(ExecutionException.class, () -> result.get(3, TimeUnit.SECONDS)); + assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(failure)); + assertEquals(0, bodyCalls.get()); + releaseStart.countDown(); + assertTrue(peerExited.await(3, TimeUnit.SECONDS)); + assertEquals(0, bodyCalls.get(), "a hook returning after the fatal must not admit user side effects"); + } finally { + releaseStart.countDown(); + callers.shutdownNow(); + workers.shutdownNow(); + assertTrue(callers.awaitTermination(3, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java index cffa0a30b..6a2703d35 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java @@ -400,6 +400,7 @@ protected T runUserFunction(Integer attempt, Supplier userFunction) { pluginRunner.onUserFunctionStart(startInfo); T result; try { + executionManager.rethrowPluginFatalIfPresent(); result = userFunction.get(); executionManager.rethrowPluginFatalIfPresent(); } catch (Throwable e) { From 5cf04d1877553652d21d0f688cb0ddbb97359578 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 06:39:09 +0000 Subject: [PATCH 12/17] fix: retain owner cleanup through bounded plugin failures --- docs/migration-2.x-to-3.x.md | 2 +- .../durable/otel/ExecutionOtelPlugin.java | 28 ++-- .../durable/otel/InvocationOtelPlugin.java | 27 ++-- .../durable/otel/OtelPluginSupport.java | 17 ++ .../InvocationInfoHeaderIntegrationTest.java | 20 ++- .../otel/LateUserFunctionCleanupTest.java | 149 ++++++++++++++++++ .../otel/UserFunctionEndCleanupTest.java | 131 +++++++++++++++ .../durable/UserFunctionEndHandoffTest.java | 124 +++++++++++++++ .../durable/execution/DurableExecutor.java | 11 +- .../durable/execution/ExecutionManager.java | 7 +- .../durable/execution/RuntimeTraceHeader.java | 16 +- .../operation/BaseDurableOperation.java | 2 +- .../lambda/durable/plugin/PluginRunner.java | 82 ++++++++-- .../execution/HandlerScopeHandoffTest.java | 2 + .../execution/RuntimeTraceHeaderTest.java | 55 +++++++ .../plugin/PluginLinkageErrorTest.java | 4 +- .../plugin/UserFunctionStartFailureTest.java | 5 +- 17 files changed, 629 insertions(+), 53 deletions(-) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LateUserFunctionCleanupTest.java create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionEndCleanupTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/UserFunctionEndHandoffTest.java diff --git a/docs/migration-2.x-to-3.x.md b/docs/migration-2.x-to-3.x.md index 319316c63..0e6698171 100644 --- a/docs/migration-2.x-to-3.x.md +++ b/docs/migration-2.x-to-3.x.md @@ -226,7 +226,7 @@ What actually changed: - `DurableConfig.getPluginRunner()` is removed. `DurableConfig.getPluginFactories()` replaces it and returns an immutable `List` in dispatch order. - `PluginRunner.getPlugins()` is removed. A runner holds no plugin instances until `onInvocationStart(InvocationInfo)` materializes them, and there is no accessor for them. - `PluginRunner`'s constructor takes `List` instead of `List`. -- `PluginRunner.releasePlugins()` is added. The SDK calls it when the invocation returns, which is what bounds a plugin instance's lifetime to one invocation. +- `PluginRunner.releasePlugins()` is added. The SDK calls it when the invocation returns, releasing invocation-level references. If fatal cleanup exceeds its bounded allowance, a started operation retains that invocation's end-hook recipients until its owner unwinds. These cleanup references do not create or share plugin instances with another invocation. - `ExecutionManager.getPluginRunner()` exists in `3.x` and returns the runner for the current invocation. It is new in this release, not a renamed `2.x` method, and `ExecutionManager` is an internal coordination class. It is reachable only through `BaseContextImpl.getExecutionManager()`, which is declared on the implementation class and not on the `DurableContext` or `BaseContext` interfaces that handlers are given. What to do instead, by what you were trying to achieve: diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java index 0074ba61c..6af8b1933 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExecutionOtelPlugin.java @@ -69,11 +69,12 @@ * derived from the execution ARN and start time rather than carried in the plugin. * *

      Lifetime. One instance serves exactly one Lambda invocation: {@link #factory()} and its overloads return a - * {@link DurableExecutionPluginFactory} that the SDK calls once per invocation, and the instance is dropped when the - * invocation returns. Everything about the invocation — the execution ARN, the resolved execution trace and ancestor, - * the sampling intent, the Invocation span, the deferred Workflow span context — is therefore a {@code final} field, - * resolved in the constructor from the {@link InvocationInfo} the factory receives. Nothing is reset between - * invocations because nothing is carried between them. + * {@link DurableExecutionPluginFactory} that the SDK calls once per invocation. Invocation-level references are + * released at return; an unfinished operation retains its matching cleanup until its owner unwinds. Everything about + * the invocation — the execution ARN, the resolved execution trace and ancestor, the sampling intent, the Invocation + * span, the deferred Workflow span context — is therefore a {@code final} field, resolved in the constructor from the + * {@link InvocationInfo} the factory receives. Nothing is reset between invocations because nothing is carried between + * them. * *

      What belongs to the execution environment stays in the factory's {@link OtelPluginEnvironment}: the configuration, * the ID generator, and either the application-owned tracer provider (built once) or the lazily resolved ADOT global @@ -380,17 +381,15 @@ public void onInvocationEnd(InvocationEndInfo info) { MdcSpanEnricher.clear(); } - // Release OTel context on worker threads, then end any attempt spans still open so no recording span is - // abandoned. Attempt spans normally start and end within one user-function call, so this is a safeguard. + // Close scopes owned by this thread, retaining other owners' scopes until their matching end hooks run. for (var scope : attemptScopes.values()) { scope.close(); } for (var span : attemptSpans.values()) { span.end(); } - // The placeholder and attempt registries are not emptied: an operation that never completed has no recording - // span to abandon, every attempt span above has been ended, and this instance is dropped when the invocation - // returns, so there is nothing to recycle them for. + // Every attempt span has ended. Only unfinished owners may retain this invocation's instance for scope + // cleanup; these registries are never recycled for another invocation. // End the invocation span every invocation. invocationSpan.setAttribute( @@ -531,7 +530,8 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { if ("CONTEXT".equals(info.type())) { var operationContext = operationContexts.get(info.id()); if (operationContext != null) { - var scope = Span.wrap(operationContext).makeCurrent(); + var scope = OtelPluginSupport.onOwnerThread( + Span.wrap(operationContext).makeCurrent()); var key = attemptKey(info.id(), info.attempt()); attemptScopes.put(key, scope); } @@ -571,7 +571,7 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { attemptSpans.put(key, span); // Make span current on this thread so auto-instrumented calls become children - var scope = span.makeCurrent(); + var scope = OtelPluginSupport.onOwnerThread(span.makeCurrent()); attemptScopes.put(key, scope); if (enableMdc) { @@ -581,7 +581,7 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { @Override public void onUserFunctionEnd(UserFunctionEndInfo info) { - if (disabled()) return; + if (invocationSpan == null) return; var key = attemptKey(info.id(), info.attempt()); @@ -591,6 +591,8 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) { scope.close(); } + if (ended) return; + // CONTEXT operations don't have attempt spans — scope cleanup is all we need if ("CONTEXT".equals(info.type())) { return; diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java index 7b3fd3684..346739f35 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java @@ -65,12 +65,12 @@ * * *

      Lifetime. One instance serves exactly one Lambda invocation: {@link #factory()} and its overloads return a - * {@link DurableExecutionPluginFactory} that the SDK calls once per invocation, and the instance is dropped when the - * invocation returns. Everything about the invocation — the execution ARN, the resolved execution trace and ancestor, - * the sampling intent, the Invocation span, the deferred Workflow span context — is therefore a {@code final} field, - * resolved in the constructor from the {@link InvocationInfo} the factory receives (the same instance - * {@link #onInvocationStart(InvocationInfo)} then receives). Nothing is reset between invocations because nothing is - * carried between them. + * {@link DurableExecutionPluginFactory} that the SDK calls once per invocation. Invocation-level references are + * released at return; an unfinished operation retains its matching cleanup until its owner unwinds. Everything about + * the invocation — the execution ARN, the resolved execution trace and ancestor, the sampling intent, the Invocation + * span, the deferred Workflow span context — is therefore a {@code final} field, resolved in the constructor from the + * {@link InvocationInfo} the factory receives (the same instance {@link #onInvocationStart(InvocationInfo)} then + * receives). Nothing is reset between invocations because nothing is carried between them. * *

      What belongs to the execution environment stays in the factory's {@link OtelPluginEnvironment}: the configuration, * the ID generator, and either the application-owned tracer provider (built once) or the lazily resolved ADOT global @@ -570,7 +570,7 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { // Still set the operation span as current so auto-instrumented calls become children var operationSpan = operationSpans.get(info.id()); if (operationSpan != null) { - var scope = operationSpan.makeCurrent(); + var scope = OtelPluginSupport.onOwnerThread(operationSpan.makeCurrent()); var key = attemptKey(info.id(), info.attempt()); attemptScopes.put(key, scope); } @@ -610,7 +610,7 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { attemptSpans.put(key, span); // Make span current on this thread so auto-instrumented calls become children - var scope = span.makeCurrent(); + var scope = OtelPluginSupport.onOwnerThread(span.makeCurrent()); attemptScopes.put(key, scope); // Inject trace context into MDC for log-trace correlation @@ -621,7 +621,7 @@ public void onUserFunctionStart(UserFunctionStartInfo info) { @Override public void onUserFunctionEnd(UserFunctionEndInfo info) { - if (disabled()) return; + if (invocationSpan == null) return; var key = attemptKey(info.id(), info.attempt()); @@ -636,6 +636,9 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) { MDC.remove(MdcSpanEnricher.MDC_SPAN_ID); } + // A bounded fatal return can finish spans before this operation owner finally unwinds. + if (ended) return; + // CONTEXT operations don't have attempt spans — scope cleanup is all we need if ("CONTEXT".equals(info.type())) { return; @@ -677,7 +680,7 @@ private boolean disabled() { } private void endOpenSpansChildFirst() { - // Attempt spans are children of operation spans, so release their scopes and end them first. + // Close scopes owned by this thread, retaining other owners' scopes for their matching end hooks. for (var scope : attemptScopes.values()) { scope.close(); } @@ -694,8 +697,8 @@ private void endOpenSpansChildFirst() { span.end(); } } - // The registries are not emptied afterwards: every span they held has been ended above, and this instance is - // dropped when the invocation returns, so there is nothing to recycle them for. + // Every span has ended. Only unfinished owners may retain this invocation's instance for scope cleanup; + // these registries are never recycled for another invocation. } /** diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java index 9adf0af9e..127853756 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginSupport.java @@ -13,6 +13,7 @@ import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.api.trace.TracerProvider; import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.samplers.SamplingResult; import java.nio.file.Files; @@ -59,6 +60,22 @@ static void exportExecutionRoot( private OtelPluginSupport() {} + /** Keeps finalization on another thread from consuming an operation owner's eventual scope cleanup. */ + static Scope onOwnerThread(Scope scope) { + var owner = Thread.currentThread(); + return new Scope() { + // Only the owner can read or mutate this flag; other threads return before accessing it. + private boolean closed; + + @Override + public void close() { + if (Thread.currentThread() != owner || closed) return; + closed = true; + scope.close(); + } + }; + } + /** Creates a new DeterministicIdGenerator for the application-side state bridge. */ static DeterministicIdGenerator createDefaultIdGenerator() { return new DeterministicIdGenerator(); diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationInfoHeaderIntegrationTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationInfoHeaderIntegrationTest.java index a598d393f..a76a11350 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationInfoHeaderIntegrationTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/InvocationInfoHeaderIntegrationTest.java @@ -11,6 +11,7 @@ import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import java.time.Instant; import java.util.Map; +import java.util.concurrent.CompletionException; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -39,8 +40,16 @@ class InvocationInfoHeaderIntegrationTest { @ParameterizedTest @CsvSource({ - "factory,present", "hook,present", "factory,empty", "hook,empty", - "factory,null", "hook,null", "factory,unavailable", "hook,unavailable" + "factory,unreadable-cause", + "hook,unreadable-cause", + "factory,present", + "hook,present", + "factory,empty", + "hook,empty", + "factory,null", + "hook,null", + "factory,unavailable", + "hook,unavailable" }) void ordinaryFactoryAndHookCanReadTheCapturedHeader(String reader, String carrier) { var runtimeThread = Thread.currentThread(); @@ -54,6 +63,13 @@ void ordinaryFactoryAndHookCanReadTheCapturedHeader(String reader, String carrie when(runtime.getXrayTraceId()).thenAnswer(ignored -> { capturedThread.set(Thread.currentThread()); return switch (carrier) { + case "unreadable-cause" -> + throw new CompletionException("unreadable cause", null) { + @Override + public synchronized Throwable getCause() { + throw new IllegalStateException("cause accessor failed"); + } + }; case "present" -> HEADER; case "empty" -> ""; default -> null; diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LateUserFunctionCleanupTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LateUserFunctionCleanupTest.java new file mode 100644 index 000000000..76c88d2d7 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/LateUserFunctionCleanupTest.java @@ -0,0 +1,149 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.time.Duration; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class LateUserFunctionCleanupTest { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void ownerUnwindingAfterFatalReturnStillClosesItsOriginalAttemptScope(boolean executionView) throws Exception { + var fatal = new InternalError("peer fatal"); + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + var exited = new CountDownLatch(1); + var owner = new AtomicReference(); + var before = new AtomicReference(); + var afterTask = new AtomicReference(); + var endedOn = new AtomicReference(); + var endError = new AtomicReference(); + var ends = new AtomicInteger(); + var factories = new AtomicInteger(); + var endedInstance = new AtomicInteger(); + var exporter = InMemorySpanExporter.create(); + var provider = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var config = OtelPluginConfig.builder() + .enableMdc(true) + .contextExtractor(() -> new ExtractedContext( + "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED)) + .build(); + var otel = executionView + ? ExecutionOtelPlugin.factory(provider, config) + : InvocationOtelPlugin.factory(provider, config); + DurableExecutionPluginFactory observer = info -> { + var instance = factories.incrementAndGet(); + return new DurableExecutionPlugin() { + @Override + public void onUserFunctionStart(UserFunctionStartInfo start) { + if ("peer".equals(start.name())) { + owner.set(Thread.currentThread()); + before.set(Span.current().getSpanContext()); + } + } + + @Override + public void onUserFunctionEnd(UserFunctionEndInfo end) { + if ("peer".equals(end.name())) { + endedOn.set(Thread.currentThread()); + endError.set(end.error()); + ends.incrementAndGet(); + endedInstance.set(instance); + } + } + }; + }; + DurableExecutionPluginFactory fault = info -> new DurableExecutionPlugin() { + @Override + public void onUserFunctionStart(UserFunctionStartInfo start) { + if ("trigger".equals(start.name())) throw fatal; + } + }; + var workers = new ThreadPoolExecutor(0, 16, 1, TimeUnit.SECONDS, new SynchronousQueue<>()) { + @Override + public void execute(Runnable task) { + super.execute(() -> { + try { + task.run(); + } catch (InternalError expected) { + // Reusable custom workers make leaked thread-local scope visible after task exit. + } finally { + if (Thread.currentThread() == owner.get()) { + afterTask.set(Span.current().getSpanContext()); + exited.countDown(); + } + } + }); + } + }; + try { + var durable = DurableConfig.builder() + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(observer, otel, fault) + .build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + var peer = context.stepAsync("peer", String.class, step -> { + entered.countDown(); + while (release.getCount() != 0) { + try { + release.await(); + } catch (InterruptedException ignored) { + // Hold the actual user body past the invocation's bounded cleanup allowance. + } + } + return "late"; + }); + try { + assertTrue(entered.await(3, TimeUnit.SECONDS)); + } catch (InterruptedException failure) { + throw new AssertionError(failure); + } + context.stepAsync("trigger", String.class, step -> "unreachable"); + return peer.get(); + }, + durable); + assertSame(fatal, assertThrows(InternalError.class, () -> runner.run("input"))); + assertEquals(0, ends.get(), "the peer is still held after invocation return"); + var other = LocalDurableTestRunner.create(String.class, (input, context) -> "other", durable); + assertEquals(ExecutionStatus.SUCCEEDED, other.run("other").getStatus()); + assertEquals(2, factories.get(), "a new invocation creates its own plugin while old cleanup is held"); + release.countDown(); + assertTrue(exited.await(3, TimeUnit.SECONDS)); + assertAll( + () -> assertEquals(1, ends.get(), "the started attempt must retain its own end-hook recipients"), + () -> assertEquals( + before.get(), afterTask.get(), "the original OTel scope must close on its owner")); + assertSame(owner.get(), endedOn.get()); + assertSame(fatal, endError.get()); + assertEquals(1, endedInstance.get(), "late cleanup belongs to the original invocation's plugin"); + assertEquals(2, factories.get(), "owner cleanup does not invoke factories again"); + assertEquals( + 1, + exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().contains("peer") + && span.getName().contains(" attempt ")) + .count()); + } finally { + release.countDown(); + workers.shutdownNow(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } +} diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionEndCleanupTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionEndCleanupTest.java new file mode 100644 index 000000000..6a2e4d030 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/UserFunctionEndCleanupTest.java @@ -0,0 +1,131 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.time.Duration; +import java.util.Arrays; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class UserFunctionEndCleanupTest { + @ParameterizedTest + @CsvSource({"false,false", "false,true", "true,false", "true,true"}) + @SuppressWarnings("removal") + void earlierEndFatalCannotSkipLaterOtelOwnerCleanup(boolean executionView, boolean threadDeath) throws Exception { + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("first user end"); + var cleanupFatal = new InternalError("later user end"); + var owner = new AtomicReference(); + var before = new AtomicReference(); + var after = new AtomicReference(); + var afterTask = new AtomicReference(); + var endOwner = new AtomicReference(); + var ends = new AtomicInteger(); + var bodies = new AtomicInteger(); + var exited = new CountDownLatch(1); + var exporter = InMemorySpanExporter.create(); + var provider = SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)); + var config = OtelPluginConfig.builder() + .enableMdc(true) + .contextExtractor(() -> new ExtractedContext( + "12345678901234567890123456789012", "1234567890123456", ExtractedContext.Sampling.SAMPLED)) + .build(); + var otel = executionView + ? ExecutionOtelPlugin.factory(provider, config) + : InvocationOtelPlugin.factory(provider, config); + DurableExecutionPluginFactory first = info -> new DurableExecutionPlugin() { + @Override + public void onUserFunctionStart(UserFunctionStartInfo start) { + if ("work".equals(start.name())) { + owner.set(Thread.currentThread()); + before.set(Span.current().getSpanContext()); + } + } + + @Override + public void onUserFunctionEnd(UserFunctionEndInfo end) { + if ("work".equals(end.name())) throw fatal; + } + }; + DurableExecutionPluginFactory last = info -> new DurableExecutionPlugin() { + @Override + public void onUserFunctionEnd(UserFunctionEndInfo end) { + if ("work".equals(end.name())) { + ends.incrementAndGet(); + endOwner.set(Thread.currentThread()); + after.set(Span.current().getSpanContext()); + } + } + }; + DurableExecutionPluginFactory secondFailure = info -> new DurableExecutionPlugin() { + @Override + public void onUserFunctionEnd(UserFunctionEndInfo end) { + if ("work".equals(end.name())) throw cleanupFatal; + } + }; + var workers = new ThreadPoolExecutor(0, 16, 1, TimeUnit.SECONDS, new SynchronousQueue<>()) { + @Override + public void execute(Runnable task) { + super.execute(() -> { + try { + task.run(); + } catch (VirtualMachineError | ThreadDeath expected) { + // Custom executors may retain workers after a task reports its fatal to the invocation. + } finally { + if (Thread.currentThread() == owner.get()) { + afterTask.set(Span.current().getSpanContext()); + exited.countDown(); + } + } + }); + } + }; + try { + var durable = DurableConfig.builder() + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(first, otel, secondFailure, last) + .build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.step("work", String.class, step -> { + bodies.incrementAndGet(); + return "done"; + }), + durable); + var thrown = assertThrows(Error.class, () -> runner.run("input")); + assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(thrown)); + // Resource shutdown may additionally attach its transport failure; retain this cleanup failure once. + assertEquals( + 1, + Arrays.stream(fatal.getSuppressed()) + .filter(error -> error == cleanupFatal) + .count()); + assertTrue(exited.await(3, TimeUnit.SECONDS)); + assertEquals(1, bodies.get()); + assertEquals(1, ends.get(), "the later observer must receive the same single end event"); + assertSame(owner.get(), endOwner.get()); + assertEquals( + before.get(), after.get(), "OTel must restore its attempt scope on the owner before later hooks"); + assertEquals( + before.get(), afterTask.get(), "the executor reuse boundary must not retain the attempt scope"); + assertTrue(exporter.getFinishedSpanItems().stream() + .anyMatch(span -> span.getName().contains(" attempt "))); + } finally { + workers.shutdownNow(); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } +} diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/UserFunctionEndHandoffTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/UserFunctionEndHandoffTest.java new file mode 100644 index 000000000..ac9ab85f0 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/UserFunctionEndHandoffTest.java @@ -0,0 +1,124 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class UserFunctionEndHandoffTest { + @ParameterizedTest + @CsvSource({"false,false", "true,false", "false,true", "true,true"}) + void fatalWakesCallerWithoutFinalizingInsideTheOperationOwner(boolean exceedsBudget, boolean partialStart) + throws Exception { + var fatal = new InternalError("first end fatal"); + var cleanupEntered = new CountDownLatch(1); + var releaseCleanup = new CountDownLatch(1); + var cleanupExited = new CountDownLatch(1); + var owner = new AtomicReference(); + var endThread = new AtomicReference(); + var invocationEnded = new AtomicBoolean(); + var cleanupFinished = new AtomicBoolean(); + var cleanupBeforeEnd = new AtomicBoolean(); + var cleanupOrder = new CopyOnWriteArrayList(); + var bodyCalls = new AtomicInteger(); + DurableExecutionPluginFactory first = info -> new DurableExecutionPlugin() { + @Override + public void onUserFunctionStart(UserFunctionStartInfo start) { + owner.set(Thread.currentThread()); + if (partialStart) throw fatal; + } + + @Override + public void onUserFunctionEnd(UserFunctionEndInfo end) { + owner.set(Thread.currentThread()); + if (!partialStart) throw fatal; + } + }; + DurableExecutionPluginFactory heldCleanup = info -> new DurableExecutionPlugin() { + @Override + public void onUserFunctionEnd(UserFunctionEndInfo end) { + cleanupEntered.countDown(); + while (releaseCleanup.getCount() != 0) { + try { + releaseCleanup.await(); + } catch (InterruptedException ignored) { + // Exercise an end hook that does not cooperate with the bounded cancellation request. + } + } + cleanupFinished.set(true); + cleanupOrder.add("held"); + } + + @Override + public void onInvocationEnd(InvocationEndInfo end) { + endThread.set(Thread.currentThread()); + cleanupBeforeEnd.set(cleanupFinished.get()); + invocationEnded.set(true); + } + }; + DurableExecutionPluginFactory earlier = info -> new DurableExecutionPlugin() { + @Override + public void onUserFunctionEnd(UserFunctionEndInfo end) { + cleanupOrder.add("earlier"); + cleanupExited.countDown(); + } + }; + var workers = Executors.newCachedThreadPool(task -> { + var thread = new Thread(task, "end-handoff-worker"); + thread.setDaemon(true); + thread.setUncaughtExceptionHandler((ignored, failure) -> {}); + return thread; + }); + var callers = Executors.newSingleThreadExecutor(); + try { + var plugins = partialStart + ? new DurableExecutionPluginFactory[] {earlier, heldCleanup, first} + : new DurableExecutionPluginFactory[] {first, heldCleanup, earlier}; + var config = DurableConfig.builder() + .withExecutorService(workers) + .withCheckpointDelay(Duration.ZERO) + .withPlugins(plugins) + .build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.step("work", String.class, step -> { + bodyCalls.incrementAndGet(); + return "done"; + }), + config); + var response = callers.submit(() -> runner.run("input")); + assertTrue(cleanupEntered.await(3, TimeUnit.SECONDS)); + if (!exceedsBudget) { + assertFalse(invocationEnded.get(), "cooperative owner cleanup must precede invocation finalization"); + releaseCleanup.countDown(); + } + var failure = assertThrows(ExecutionException.class, () -> response.get(2, TimeUnit.SECONDS)); + assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(failure)); + assertEquals(partialStart ? 0 : 1, bodyCalls.get()); + assertTrue(invocationEnded.get()); + if (exceedsBudget) + assertNotSame( + owner.get(), endThread.get(), "a held operation cannot finalize inline ahead of its cleanup"); + assertEquals(!exceedsBudget, cleanupBeforeEnd.get()); + if (exceedsBudget) assertFalse(cleanupFinished.get(), "the caller must not wait for uncooperative cleanup"); + releaseCleanup.countDown(); + assertTrue(cleanupExited.await(3, TimeUnit.SECONDS)); + assertEquals(List.of("held", "earlier"), cleanupOrder, "earlier starts unwind in LIFO order"); + } finally { + releaseCleanup.countDown(); + callers.shutdownNow(); + workers.shutdownNow(); + assertTrue(callers.awaitTermination(3, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index 7e4f9759d..073f01eb0 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -150,6 +150,7 @@ public static DurableExecutionOutput execute( lambdaContext, config.getPluginFactories().size(), pluginFatal, + executionManager, (result, ex) -> { if (ex != null) { // an exception thrown from handlerFuture or suspension/termination occurred @@ -347,8 +348,10 @@ private static R finalizeAfterHandlerScopes( Context lambdaContext, int pluginCount, AtomicReference pluginFatal, + ExecutionManager executionManager, BiFunction finalizer) { var started = new AtomicBoolean(); + var operationOwnerHandoff = new AtomicBoolean(); var finalized = new CompletableFuture(); // Attach before waiting so legacy hooks retain normal CompletableFuture completion-thread dispatch. var ready = executionFuture.handle((value, failure) -> { @@ -358,7 +361,11 @@ private static R finalizeAfterHandlerScopes( var fatal = pluginFatal.get(); return finalizer.apply(value, fatal == null ? failure : fatal); }); - if (!hasHandlerScope.get() || handlerFuture.isDone()) finish.run(); + if (pluginFatal.get() != null && executionManager.isRunningOperationOwner()) { + // Reporting a fatal must not finalize inline while this operation still owns attempt cleanup. + // Wake the invocation caller, which can apply the existing bounded owner handoff independently. + operationOwnerHandoff.set(true); + } else if (!hasHandlerScope.get() || handlerFuture.isDone()) finish.run(); else handlerFuture.whenComplete((ignored, ignoredFailure) -> finish.run()); return finish; }); @@ -368,7 +375,7 @@ private static R finalizeAfterHandlerScopes( awaitHandlerScopes( executionFuture, handlerFuture, hasHandlerScope, lambdaContext, pluginCount, pluginFatal); // A completed handler owns its registered end callback even if the caller wakes first. - if (!handlerFuture.isDone()) finish.run(); + if (operationOwnerHandoff.get() || !handlerFuture.isDone()) finish.run(); } return finalized.join(); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index 20ee3f9ef..c952dfd31 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -76,7 +76,8 @@ public class ExecutionManager implements SafeCloseable { // ===== Plugins ===== // Created per invocation, alongside this manager: the runner materializes one plugin instance per configured - // factory when the invocation starts, and releases them in close(), so instances never outlive the invocation. + // factory when the invocation starts and releases the runner's references in close(). Started operation owners + // retain only their matching cleanup recipients until they unwind, including after a bounded fatal return. private final PluginRunner pluginRunner; private final AtomicReference pluginFatal; private final CompletableFuture pluginFatalSignal = new CompletableFuture<>(); @@ -731,6 +732,10 @@ void awaitFatalOperationCleanup() { validateRunningThreads(); } + boolean isRunningOperationOwner() { + return registeredOperations.values().stream().anyMatch(BaseDurableOperation::isRunningUserHandlerOwner); + } + /** Returns {@code true} if the given status represents a terminal (final) operation state. */ public static boolean isTerminalStatus(OperationStatus status) { return status == OperationStatus.SUCCEEDED diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/RuntimeTraceHeader.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/RuntimeTraceHeader.java index fa5c4086d..4426a4c7a 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/RuntimeTraceHeader.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/RuntimeTraceHeader.java @@ -58,11 +58,17 @@ private static void rethrowFatal(Throwable failure) { if (cause instanceof VirtualMachineError fatal) throw fatal; if (cause instanceof ThreadDeath fatal) throw fatal; if (!(cause instanceof CompletionException - || cause instanceof ExecutionException - || cause instanceof InvocationTargetException - || cause instanceof UndeclaredThrowableException) - || cause.getCause() == null) return; - cause = cause.getCause(); + || cause instanceof ExecutionException + || cause instanceof InvocationTargetException + || cause instanceof UndeclaredThrowableException)) return; + try { + cause = cause.getCause(); + } catch (VirtualMachineError | ThreadDeath fatal) { + throw fatal; + } catch (Throwable unreadableCause) { + return; + } + if (cause == null) return; } } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java index 6a2703d35..fa3a6e4c5 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java @@ -391,7 +391,7 @@ private void clearHandlerOwner(CompletableFuture completion) { * @return the user function's result */ protected T runUserFunction(Integer attempt, Supplier userFunction) { - var pluginRunner = getPluginRunner(); + var pluginRunner = getPluginRunner().captureUserFunctionHooks(); var startInfo = PluginInfoConverter.toUserFunctionStartInfo( operationIdentifier, durableContext.getParentId(), diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java index 424b456bf..f0cf847b7 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/PluginRunner.java @@ -37,6 +37,7 @@ public class PluginRunner { private static final Logger logger = LoggerFactory.getLogger(PluginRunner.class); + private static final UserFunctionHooks EMPTY_USER_FUNCTION_HOOKS = new UserFunctionHooks(List.of(), fatal -> {}); private final List pluginFactories; private final Consumer operationFatalObserver; @@ -130,8 +131,8 @@ private static DurableExecutionPlugin createPlugin(DurableExecutionPluginFactory } /** - * Drops this invocation's plugin instances. Called when the invocation returns so the instances are unreachable - * from the SDK and cannot leak into the next invocation the environment hosts. + * Drops invocation-level plugin references when the invocation returns. A started operation retains only its + * matching hook recipients until its owner unwinds; these recipients are never reused by another invocation. * *

      No containment here: this only replaces the field, and calls nothing on the plugins it drops. There is no * {@code close()} in the plugin contract, so releasing cannot run plugin code and cannot fail. @@ -368,19 +369,26 @@ public void onOperationChange(OperationChangeInfo info) { runOperationHook(p -> p.onOperationChange(info)); } - @SuppressWarnings("removal") public void onUserFunctionStart(UserFunctionStartInfo info) { + startUserFunctions(plugins, info, operationFatalObserver); + } + + @SuppressWarnings("removal") + private static void startUserFunctions( + List plugins, UserFunctionStartInfo info, Consumer operationFatalObserver) { var started = new ArrayDeque(); try { - run(plugin -> { - plugin.onUserFunctionStart(info); - started.push(plugin); - }); + for (var plugin : plugins) { + runPlugin(plugin, current -> { + current.onUserFunctionStart(info); + started.push(current); + }); + } } catch (VirtualMachineError | ThreadDeath fatal) { - // Unwind attempt scopes on their owner before publishing the fatal: publication can start invocation - // finalization on another thread, where it is too late to restore these thread-local scopes safely. - closeStartedUserFunctions(started, info, fatal); + // Wake the caller before an earlier plugin's cleanup can block. Operation-owner fatal handoff keeps + // finalization off this cleanup thread, while earlier starts still unwind in LIFO order here. operationFatalObserver.accept(fatal); + closeStartedUserFunctions(started, info, fatal); throw fatal; } } @@ -400,17 +408,67 @@ private static void closeStartedUserFunctions( } public void onUserFunctionEnd(UserFunctionEndInfo info) { - runOperationHook(p -> p.onUserFunctionEnd(info)); + endUserFunctions(plugins, info, operationFatalObserver); + } + + private static void endUserFunctions( + List plugins, UserFunctionEndInfo info, Consumer operationFatalObserver) { + var fatal = closeUserFunctions(plugins, info, null, operationFatalObserver); + if (fatal != null) { + throw fatal; + } } /** Completes same-owner attempt cleanup after a peer fatal while retaining the invocation's original failure. */ public void onUserFunctionEndAfterFatal(UserFunctionEndInfo info, Error original) { + closeUserFunctions(plugins, info, original, operationFatalObserver); + } + + private static Error closeUserFunctions( + List plugins, + UserFunctionEndInfo info, + Error original, + Consumer operationFatalObserver) { + var first = original; for (var plugin : plugins) { try { runPlugin(plugin, p -> p.onUserFunctionEnd(info)); } catch (Error cleanupFailure) { - if (cleanupFailure != original) original.addSuppressed(cleanupFailure); + if (first == null) { + first = cleanupFailure; + operationFatalObserver.accept(first); + } else if (cleanupFailure != first) first.addSuppressed(cleanupFailure); } } + return first; + } + + /** Captures this invocation's attempt hook recipients without invoking factories or sharing plugin instances. */ + public UserFunctionHooks captureUserFunctionHooks() { + var snapshot = plugins; + return snapshot.isEmpty() ? EMPTY_USER_FUNCTION_HOOKS : new UserFunctionHooks(snapshot, operationFatalObserver); + } + + /** An operation owner retains these recipients through its matching end, including after a bounded fatal return. */ + public static final class UserFunctionHooks { + private final List plugins; + private final Consumer operationFatalObserver; + + private UserFunctionHooks(List plugins, Consumer operationFatalObserver) { + this.plugins = plugins; + this.operationFatalObserver = operationFatalObserver; + } + + public void onUserFunctionStart(UserFunctionStartInfo info) { + startUserFunctions(plugins, info, operationFatalObserver); + } + + public void onUserFunctionEnd(UserFunctionEndInfo info) { + endUserFunctions(plugins, info, operationFatalObserver); + } + + public void onUserFunctionEndAfterFatal(UserFunctionEndInfo info, Error original) { + closeUserFunctions(plugins, info, original, operationFatalObserver); + } } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/HandlerScopeHandoffTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/HandlerScopeHandoffTest.java index f90436fbb..c5b6ff47c 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/HandlerScopeHandoffTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/HandlerScopeHandoffTest.java @@ -141,6 +141,7 @@ public CompletableFuture whenComplete(BiConsumer end = (value, failure) -> { @@ -158,6 +159,7 @@ public CompletableFuture whenComplete(BiConsumer(), + mock(ExecutionManager.class), end)); assertTrue(callbackRegistered.await(3, TimeUnit.SECONDS)); workers.submit(() -> { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/RuntimeTraceHeaderTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/RuntimeTraceHeaderTest.java index cf396a111..17b4bde7e 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/RuntimeTraceHeaderTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/RuntimeTraceHeaderTest.java @@ -18,6 +18,7 @@ import java.nio.file.Path; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; import javax.tools.ToolProvider; import org.junit.jupiter.api.Test; @@ -25,6 +26,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.lambda.durable.util.ExceptionHelper; class RuntimeTraceHeaderTest { @Test @@ -223,6 +225,59 @@ private static Stream fatalAccessorFailures() { Arguments.of(new CompletionException(new InvocationTargetException(fatal)), fatal))); } + @ParameterizedTest + @MethodSource("unreadableCauses") + void unreadableNonfatalWrapperCauseIsAuthoritativeAbsence(Throwable unreadable) { + var reads = new AtomicInteger(); + var wrapper = new CompletionException("unreadable", null) { + @Override + public synchronized Throwable getCause() { + reads.incrementAndGet(); + ExceptionHelper.sneakyThrow(unreadable); + return null; + } + }; + var context = mock(RuntimeContext.class); + when(context.getXrayTraceId()).thenThrow(wrapper); + assertEquals("", RuntimeTraceHeader.capture(context)); + assertEquals(1, reads.get()); + } + + private static Stream unreadableCauses() { + return Stream.of(new IllegalStateException("unreadable"), new AssertionError("unreadable")); + } + + @Test + void wrapperCauseIsReadOnceAndFatalIdentityIsPreserved() { + var reads = new AtomicInteger(); + var fatal = new InternalError("original cause"); + var wrapper = new CompletionException("changing cause", null) { + @Override + public synchronized Throwable getCause() { + if (reads.incrementAndGet() > 1) throw new IllegalStateException("second cause read"); + return fatal; + } + }; + var context = mock(RuntimeContext.class); + when(context.getXrayTraceId()).thenThrow(wrapper); + assertSame(fatal, assertThrows(InternalError.class, () -> RuntimeTraceHeader.capture(context))); + assertEquals(1, reads.get()); + } + + @Test + void fatalCauseAccessorFailureRetainsIdentity() { + var fatal = new InternalError("cause accessor fatal"); + var wrapper = new CompletionException("unreadable", null) { + @Override + public synchronized Throwable getCause() { + throw fatal; + } + }; + var context = mock(RuntimeContext.class); + when(context.getXrayTraceId()).thenThrow(wrapper); + assertSame(fatal, assertThrows(InternalError.class, () -> RuntimeTraceHeader.capture(context))); + } + private abstract static class RuntimeContext implements Context { @Override public String getXrayTraceId() { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginLinkageErrorTest.java b/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginLinkageErrorTest.java index 4b2352fa2..2806c63d7 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginLinkageErrorTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/plugin/PluginLinkageErrorTest.java @@ -35,9 +35,9 @@ void fatalErrorsRetainTheirIdentity( assertSame(failure, assertThrows(Error.class, () -> dispatch.accept(runner))); assertEquals( - hook.equals("onInvocationEnd") ? 1 : 0, + hook.equals("onInvocationEnd") || hook.equals("onUserFunctionEnd") ? 1 : 0, healthyCalls.get(), - "Invocation finalization still gives every plugin its cleanup opportunity"); + "End hooks still give every plugin its cleanup opportunity; other fatal hooks stop dispatch"); } static Stream linkageFailures() { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/plugin/UserFunctionStartFailureTest.java b/sdk/src/test/java/software/amazon/lambda/durable/plugin/UserFunctionStartFailureTest.java index cfbdd07ca..44119a8c8 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/plugin/UserFunctionStartFailureTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/plugin/UserFunctionStartFailureTest.java @@ -28,7 +28,7 @@ static Stream failures() { @ParameterizedTest @MethodSource("failures") - void completedStartHooksUnwindOnOwnerBeforeFatalPublication(boolean wrapped, Error fatal) { + void fatalIsPublishedBeforeCompletedStartHooksUnwindInOwnerOrder(boolean wrapped, Error fatal) { var owner = Thread.currentThread(); var scopes = new ArrayDeque(); var calls = new ArrayList(); @@ -58,7 +58,8 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) { var start = start(); assertSame(fatal, assertThrows(Error.class, () -> runner.onUserFunctionStart(start))); assertSame(fatal, reported.get()); - assertEquals(0, cleanupAtPublication.get()); + assertEquals(2, cleanupAtPublication.get(), "publication must wake the caller before cleanup can block"); + assertTrue(scopes.isEmpty(), "all earlier starts must still unwind on their owner before dispatch returns"); assertEquals(List.of("first:start", "second:start", "second:end", "first:end"), calls); assertEquals(2, endInfos.size()); for (var end : endInfos) { From 95fc394f132431472126d6d399107e1f704d6a9b Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 18:51:37 +0000 Subject: [PATCH 13/17] fix: accept concrete handler scope return types --- .../lambda/durable/plugin/HandlerScoped.java | 2 +- .../plugin/ConcreteHandlerScopeTest.java | 50 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/plugin/ConcreteHandlerScopeTest.java diff --git a/sdk/src/main/java/software/amazon/lambda/durable/plugin/HandlerScoped.java b/sdk/src/main/java/software/amazon/lambda/durable/plugin/HandlerScoped.java index e4ba959ca..c630341be 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/plugin/HandlerScoped.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/plugin/HandlerScoped.java @@ -26,5 +26,5 @@ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) public @interface HandlerScoped { - Class> value(); + Class> value(); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/plugin/ConcreteHandlerScopeTest.java b/sdk/src/test/java/software/amazon/lambda/durable/plugin/ConcreteHandlerScopeTest.java new file mode 100644 index 000000000..ad8d98504 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/plugin/ConcreteHandlerScopeTest.java @@ -0,0 +1,50 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.plugin; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Instant; +import java.util.List; +import java.util.function.Function; +import org.junit.jupiter.api.Test; + +class ConcreteHandlerScopeTest { + @Test + void concreteScopeReturnTypeOpensAndClosesOnItsOwner() { + var plugin = new ConcretePlugin(); + var runner = new PluginRunner(List.of(info -> plugin)); + runner.onInvocationStart(new InvocationInfo("request", "arn:test", true, Instant.now())); + assertEquals("result", runner.runHandler(() -> { + assertNotNull(plugin.scope); + assertFalse(plugin.scope.closed); + return "result"; + })); + assertTrue(plugin.scope.closed); + assertSame(Thread.currentThread(), plugin.scope.owner); + } + + @HandlerScoped(ConcreteOpener.class) + public static final class ConcretePlugin implements DurableExecutionPlugin { + ConcreteScope scope; + } + + public static final class ConcreteOpener implements Function { + @Override + public ConcreteScope apply(ConcretePlugin plugin) { + return plugin.scope = new ConcreteScope(); + } + } + + public static final class ConcreteScope implements AutoCloseable { + final Thread owner = Thread.currentThread(); + boolean closed; + + @Override + public void close() { + assertSame(owner, Thread.currentThread()); + assertFalse(closed); + closed = true; + } + } +} From 6e512e794ecc5555ed92cc57482ebe9b8842b5b5 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 18:51:37 +0000 Subject: [PATCH 14/17] fix: preserve execution outcomes across end MDC failures --- .../durable/otel/MdcFailureBoundaryTest.java | 314 ++++++++++++++++++ .../durable/execution/DurableExecutor.java | 61 +++- 2 files changed, 374 insertions(+), 1 deletion(-) create mode 100644 otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFailureBoundaryTest.java diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFailureBoundaryTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFailureBoundaryTest.java new file mode 100644 index 000000000..7082945d6 --- /dev/null +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFailureBoundaryTest.java @@ -0,0 +1,314 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.otel; + +import static org.junit.jupiter.api.Assertions.*; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.lang.reflect.UndeclaredThrowableException; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.AbstractExecutorService; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.MDC; +import org.slf4j.spi.MDCAdapter; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.InvocationEndInfo; +import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class MdcFailureBoundaryTest { + @Test + void workerCaptureFailureDoesNotEndAnInvocationThatNeverStarted() throws Exception { + var original = MDC.getMDCAdapter(); + var injected = new AtomicBoolean(); + var failure = new IllegalStateException("worker MDC capture failed"); + var plugin = new RecordingPlugin(); + var bodyCalls = new AtomicInteger(); + var executor = new CompletingWorker(); + try { + replaceAdapter(proxy(original, (method) -> { + if (method.equals("getCopyOfContextMap") + && Thread.currentThread().getName().equals("mdc-initialization-worker") + && injected.compareAndSet(false, true)) throw failure; + return null; + })); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + bodyCalls.incrementAndGet(); + return "done"; + }, + DurableConfig.builder() + .withExecutorService(executor) + .withPlugins(info -> plugin) + .build()); + var result = runner.run("input"); + assertTrue(injected.get()); + assertEquals(ExecutionStatus.FAILED, result.getStatus()); + assertEquals(failure.getMessage(), result.getError().orElseThrow().errorMessage()); + assertEquals(0, bodyCalls.get()); + assertEquals(0, plugin.starts.get()); + assertEquals(0, plugin.ends.get(), "no end hook may consume stale state before start dispatch"); + } finally { + try { + executor.shutdownNow(); + assertTrue(executor.awaitTermination(3, TimeUnit.SECONDS)); + } finally { + replaceAdapter(original); + } + } + } + + @ParameterizedTest + @CsvSource({ + "getCopyOfContextMap,false", + "clear,false", + "setContextMap,false", + "getCopyOfContextMap,true", + "clear,true", + "setContextMap,true" + }) + void nonfatalEndMdcFailuresPreserveTheSelectedOutcome(String method, boolean suspend) throws Exception { + runEndFailure(method, suspend, new IllegalStateException("end MDC failure")); + } + + @SuppressWarnings("removal") + @ParameterizedTest + @CsvSource({ + "getCopyOfContextMap,false", + "clear,false", + "setContextMap,false", + "getCopyOfContextMap,true", + "clear,true", + "setContextMap,true" + }) + void fatalEndMdcFailuresStillEscape(String method, boolean wrapped) throws Exception { + Error fatal = wrapped ? new ThreadDeath() : new VirtualMachineError("fatal end MDC failure") {}; + runEndFailure(method, false, wrapped ? new CompletionException(new ExecutionException(fatal)) : fatal, fatal); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void unreadableOrdinaryCausePreservesSuccessAndSuspension(boolean suspend) throws Exception { + var reads = new AtomicInteger(); + var failure = new CompletionException("unreadable", null) { + @Override + public synchronized Throwable getCause() { + reads.incrementAndGet(); + throw new IllegalStateException("diagnostic unavailable"); + } + }; + runEndFailure("getCopyOfContextMap", suspend, failure); + assertEquals(1, reads.get()); + } + + @SuppressWarnings("removal") + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void fatalCauseAccessorEscapesWithOriginalIdentity(boolean threadDeath) throws Exception { + var reads = new AtomicInteger(); + Error fatal = threadDeath ? new ThreadDeath() : new VirtualMachineError("fatal accessor") {}; + var failure = new CompletionException("unreadable", null) { + @Override + public synchronized Throwable getCause() { + reads.incrementAndGet(); + throw fatal; + } + }; + runEndFailure("setContextMap", false, failure, fatal); + assertEquals(1, reads.get()); + } + + @SuppressWarnings("removal") + @ParameterizedTest + @CsvSource({"reflection,false", "reflection,true", "proxy,false", "proxy,true"}) + void jdkReflectiveWrappersKeepFatalIdentity(String wrapper, boolean threadDeath) throws Exception { + Error fatal = threadDeath ? new ThreadDeath() : new VirtualMachineError("wrapped fatal") {}; + Throwable failure = wrapper.equals("reflection") + ? new InvocationTargetException(fatal) + : new UndeclaredThrowableException(fatal); + runEndFailure("clear", false, failure, fatal); + } + + @Test + void cyclicCauseIsReadOnlyOnceAndPreservesOutcome() throws Exception { + var reads = new AtomicInteger(); + var cycle = new CompletionException("cycle", null) { + @Override + public synchronized Throwable getCause() { + reads.incrementAndGet(); + return this; + } + }; + runEndFailure("getCopyOfContextMap", false, cycle); + assertEquals(1, reads.get()); + } + + @Test + void ordinaryApplicationCauseIsNotTreatedAsATransportWrapper() throws Exception { + runEndFailure("clear", false, new IllegalStateException("ordinary diagnostic", new VirtualMachineError() {})); + } + + private static void runEndFailure(String method, boolean suspend, Throwable failure) throws Exception { + runEndFailure(method, suspend, failure, null); + } + + private static void runEndFailure(String method, boolean suspend, Throwable failure, Error expectedFatal) + throws Exception { + var original = MDC.getMDCAdapter(); + var callerBefore = MDC.getCopyOfContextMap(); + var injected = new AtomicBoolean(); + var plugin = new RecordingPlugin(); + var worker = Executors.newSingleThreadExecutor(); + try { + replaceAdapter(proxy(original, name -> { + var inEnd = Arrays.stream(Thread.currentThread().getStackTrace()) + .anyMatch(frame -> frame.getClassName().endsWith(".DurableExecutor") + && frame.getMethodName().equals("fireOnInvocationEnd")); + if (inEnd && name.equals(method) && injected.compareAndSet(false, true)) throw failure; + // Select both legitimate MDC restoration branches, independently of the finalizer's thread. + if (inEnd && name.equals("getCopyOfContextMap")) + return method.equals("setContextMap") ? Map.of("ambient", "saved") : NullSnapshot.INSTANCE; + return null; + })); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + if (suspend) context.wait("resume", Duration.ofSeconds(1)); + return "done"; + }, + DurableConfig.builder() + .withExecutorService(worker) + .withPlugins(info -> plugin) + .build()); + if (expectedFatal != null) { + assertSame(expectedFatal, assertThrows(Error.class, () -> runner.run("input"))); + assertEquals(method.equals("getCopyOfContextMap") ? 0 : 1, plugin.ends.get()); + } else { + var result = runner.run("input"); + assertEquals(suspend ? ExecutionStatus.PENDING : ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals(1, plugin.ends.get()); + } + assertTrue(injected.get(), "the actual execute() end boundary must exercise the adapter failure"); + assertEquals(1, plugin.starts.get()); + } finally { + try { + worker.shutdownNow(); + assertTrue(worker.awaitTermination(3, TimeUnit.SECONDS)); + } finally { + replaceAdapter(original); + if (callerBefore == null) MDC.clear(); + else MDC.setContextMap(callerBefore); + } + } + } + + private enum NullSnapshot { + INSTANCE + } + + @FunctionalInterface + private interface Fault { + Object apply(String method) throws Throwable; + } + + private static MDCAdapter proxy(MDCAdapter delegate, Fault fault) { + return (MDCAdapter) Proxy.newProxyInstance( + MDCAdapter.class.getClassLoader(), new Class[] {MDCAdapter.class}, (proxy, method, args) -> { + var value = fault.apply(method.getName()); + if (value == NullSnapshot.INSTANCE) return null; + if (value != null) return value; + try { + return method.invoke(delegate, args); + } catch (InvocationTargetException ex) { + throw ex.getCause(); + } + }); + } + + private static void replaceAdapter(MDCAdapter adapter) throws Exception { + var setter = MDC.class.getDeclaredMethod("setMDCAdapter", MDCAdapter.class); + setter.setAccessible(true); + setter.invoke(null, adapter); + } + + private static final class RecordingPlugin implements DurableExecutionPlugin { + final AtomicInteger starts = new AtomicInteger(); + final AtomicInteger ends = new AtomicInteger(); + + @Override + public void onInvocationStart(InvocationInfo info) { + starts.incrementAndGet(); + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + ends.incrementAndGet(); + } + } + /** Executes on a real worker, completing its first task before returning to the invocation caller. */ + private static final class CompletingWorker extends AbstractExecutorService { + private final ExecutorService delegate = + Executors.newSingleThreadExecutor(task -> new Thread(task, "mdc-initialization-worker")); + private final AtomicBoolean first = new AtomicBoolean(true); + + @Override + public void execute(Runnable task) { + if (!first.compareAndSet(true, false)) { + delegate.execute(task); + return; + } + var done = new CompletableFuture(); + delegate.execute(() -> { + try { + task.run(); + } finally { + done.complete(null); + } + }); + done.orTimeout(3, TimeUnit.SECONDS).join(); + } + + @Override + public void shutdown() { + delegate.shutdown(); + } + + @Override + public List shutdownNow() { + return delegate.shutdownNow(); + } + + @Override + public boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(timeout, unit); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index 073f01eb0..db086c292 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -4,7 +4,11 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.UndeclaredThrowableException; import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; @@ -434,6 +438,61 @@ private static SafeCloseable restoreMdcOnClose() { }; } + /** MDC is ancillary to end dispatch; ordinary adapter failures must not replace the selected outcome. */ + private static SafeCloseable preserveEndMdc() { + SafeCloseable restore; + try { + restore = restoreMdcOnClose(); + } catch (Throwable failure) { + reportEndMdcFailure(failure); + return () -> {}; + } + return () -> { + try { + restore.close(); + } catch (Throwable failure) { + reportEndMdcFailure(failure); + } + }; + } + + private static void reportEndMdcFailure(Throwable failure) { + throwIfMdcFatal(failure); + try { + // Do not ask a malformed Throwable for its message or cause again while logging. + logger.warn( + "Could not preserve MDC around invocation-end dispatch; preserving execution outcome ({})", + failure.getClass().getName()); + } catch (Throwable loggingFailure) { + // An MDC-backed logger may encounter the same adapter failure while reporting it. + throwIfMdcFatal(loggingFailure); + } + } + + @SuppressWarnings("removal") + private static void throwIfMdcFatal(Throwable failure) { + if (failure instanceof VirtualMachineError fatal) throw fatal; + if (failure instanceof ThreadDeath fatal) throw fatal; + var seen = Collections.newSetFromMap(new IdentityHashMap()); + var cause = failure; + while (seen.add(cause)) { + if (cause instanceof VirtualMachineError fatal) throw fatal; + if (cause instanceof ThreadDeath fatal) throw fatal; + if (!(cause instanceof CompletionException + || cause instanceof ExecutionException + || cause instanceof InvocationTargetException + || cause instanceof UndeclaredThrowableException)) return; + try { + cause = cause.getCause(); + } catch (VirtualMachineError | ThreadDeath fatal) { + throw fatal; + } catch (Throwable unreadableCause) { + return; + } + if (cause == null) return; + } + } + private static void fireOnInvocationEnd( PluginRunner pluginRunner, ExecutionManager executionManager, @@ -459,7 +518,7 @@ private static void fireOnInvocationEnd( error = fatal; executionResult = null; } - try (var ignored = restoreMdcOnClose()) { + try (var ignored = preserveEndMdc()) { pluginRunner.onInvocationEnd(new InvocationEndInfo( requestId, executionArn, From 25319491aaf2e359ec91d0da816d3da2338a73ef Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 20:20:07 +0000 Subject: [PATCH 15/17] fix: report worker MDC fatals before end dispatch --- .../durable/otel/MdcFailureBoundaryTest.java | 259 ++++++++++++++++++ .../durable/execution/DurableExecutor.java | 22 +- 2 files changed, 277 insertions(+), 4 deletions(-) diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFailureBoundaryTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFailureBoundaryTest.java index 7082945d6..16ebb3f16 100644 --- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFailureBoundaryTest.java +++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/MdcFailureBoundaryTest.java @@ -14,12 +14,15 @@ import java.util.concurrent.AbstractExecutorService; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -31,7 +34,9 @@ import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationEndInfo; import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.plugin.InvocationStatus; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.util.ExceptionHelper; class MdcFailureBoundaryTest { @Test @@ -76,6 +81,260 @@ void workerCaptureFailureDoesNotEndAnInvocationThatNeverStarted() throws Excepti } } + @SuppressWarnings("removal") + @ParameterizedTest + @CsvSource({"false,false", "true,false", "false,true", "true,true"}) + void workerRestoreFatalBeforeEndDispatchIsObservedByTheCaller(boolean direct, boolean threadDeath) + throws Exception { + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("outer worker restore"); + var observed = runOuterFailure(direct, null, fatal); + assertSame(fatal, observed.failure()); + assertEquals(1, observed.ends()); + assertEquals(InvocationStatus.RETRYING, observed.end().invocationStatus()); + assertSame(fatal, observed.end().executionError()); + } + + @ParameterizedTest + @CsvSource({"false,false", "true,false", "false,true", "true,true"}) + void failedBodyRetainsPrimaryFailureAndSuppressedRestoration(boolean direct, boolean bodyFatal) throws Exception { + Throwable primary = bodyFatal ? new InternalError("primary body") : new IllegalStateException("primary body"); + var secondary = new InternalError("suppressed restoration"); + var observed = runOuterFailure(direct, primary, secondary); + if (bodyFatal) assertSame(primary, observed.failure()); + else { + assertNull(observed.failure()); + assertEquals(ExecutionStatus.FAILED, observed.status()); + assertEquals("primary body", observed.errorMessage()); + } + assertEquals(1, observed.ends()); + assertSame(primary, observed.end().executionError()); + assertEquals( + bodyFatal ? InvocationStatus.RETRYING : InvocationStatus.FAILED, + observed.end().invocationStatus()); + assertTrue(Arrays.asList(primary.getSuppressed()).contains(secondary)); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void ordinaryWorkerRestorationFailureRetainsSuccessfulOutcome(boolean direct) throws Exception { + var observed = runOuterFailure(direct, null, new IllegalStateException("adapter refused restoration")); + assertNull(observed.failure()); + assertEquals(ExecutionStatus.SUCCEEDED, observed.status()); + assertEquals(1, observed.ends()); + assertEquals(InvocationStatus.SUCCEEDED, observed.end().invocationStatus()); + } + + private static OuterOutcome runOuterFailure(boolean direct, Throwable bodyFailure, Throwable restoreFailure) + throws Exception { + var original = MDC.getMDCAdapter(); + var callerBefore = MDC.getCopyOfContextMap(); + var injected = new AtomicBoolean(); + var ends = new AtomicInteger(); + var endInfo = new AtomicReference(); + ExecutorService executor = direct ? new InlineExecutor() : new CompletingWorker(); + DurableExecutionPlugin plugin = new DurableExecutionPlugin() { + @Override + public void onInvocationEnd(InvocationEndInfo info) { + ends.incrementAndGet(); + endInfo.set(info); + } + }; + try { + replaceAdapter(proxy(original, name -> { + if (isOuterRestore(name) && injected.compareAndSet(false, true)) throw restoreFailure; + return null; + })); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + if (bodyFailure != null) ExceptionHelper.sneakyThrow(bodyFailure); + return "done"; + }, + DurableConfig.builder() + .withExecutorService(executor) + .withPlugins(info -> plugin) + .build()); + Error observed = null; + ExecutionStatus status = null; + String errorMessage = null; + try { + var result = runner.run("input"); + status = result.getStatus(); + errorMessage = + result.getError().map(error -> error.errorMessage()).orElse(null); + } catch (Error failure) { + observed = failure; + } + assertTrue(injected.get(), "The actual outer worker restoration must reach the injected failure"); + return new OuterOutcome(observed, status, errorMessage, endInfo.get(), ends.get()); + } finally { + try { + executor.shutdownNow(); + assertTrue(executor.awaitTermination(3, TimeUnit.SECONDS)); + } finally { + replaceAdapter(original); + if (callerBefore == null) MDC.clear(); + else MDC.setContextMap(callerBefore); + } + } + } + + private record OuterOutcome( + Error failure, ExecutionStatus status, String errorMessage, InvocationEndInfo end, int ends) {} + + @SuppressWarnings("removal") + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void postDispatchWorkerFatalEscapesOwnerWithoutRepeatingEnd(boolean threadDeath) throws Exception { + var original = MDC.getMDCAdapter(); + var callerBefore = MDC.getCopyOfContextMap(); + var startEntered = new CountDownLatch(1); + var releaseStart = new CountDownLatch(1); + var restoreEntered = new CountDownLatch(1); + var releaseRestore = new CountDownLatch(1); + var ownerObserved = new CountDownLatch(1); + var ownerFailure = new AtomicReference(); + var callerThread = new AtomicReference(); + var ended = new AtomicInteger(); + var injected = new AtomicBoolean(); + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("late outer restore"); + var worker = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "late-mdc-owner"); + thread.setUncaughtExceptionHandler((owner, failure) -> { + ownerFailure.set(failure); + ownerObserved.countDown(); + }); + return thread; + }); + var caller = Executors.newSingleThreadExecutor(task -> { + var thread = new Thread(task, "late-mdc-caller"); + callerThread.set(thread); + return thread; + }); + DurableExecutionPlugin plugin = new DurableExecutionPlugin() { + @Override + public void onInvocationStart(InvocationInfo info) { + startEntered.countDown(); + awaitGate(releaseStart); + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + assertEquals(InvocationStatus.SUCCEEDED, info.invocationStatus()); + ended.incrementAndGet(); + } + }; + try { + replaceAdapter(proxy(original, name -> { + if (isOuterRestore(name) && injected.compareAndSet(false, true)) { + restoreEntered.countDown(); + awaitGate(releaseRestore); + throw fatal; + } + return null; + })); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> "done", + DurableConfig.builder() + .withExecutorService(worker) + .withPlugins(info -> plugin) + .build()); + var result = caller.submit(() -> runner.run("input")); + awaitGate(startEntered); + awaitFinalizerJoin(callerThread.get()); + releaseStart.countDown(); + awaitGate(restoreEntered); + assertEquals( + ExecutionStatus.SUCCEEDED, result.get(2, TimeUnit.SECONDS).getStatus()); + assertEquals(1, ended.get()); + releaseRestore.countDown(); + awaitGate(ownerObserved); + assertSame(fatal, ownerFailure.get()); + assertEquals(1, ended.get()); + } finally { + releaseStart.countDown(); + releaseRestore.countDown(); + try { + caller.shutdownNow(); + worker.shutdownNow(); + assertTrue(caller.awaitTermination(3, TimeUnit.SECONDS)); + assertTrue(worker.awaitTermination(3, TimeUnit.SECONDS)); + } finally { + replaceAdapter(original); + if (callerBefore == null) MDC.clear(); + else MDC.setContextMap(callerBefore); + } + } + } + + private static boolean isOuterRestore(String method) { + if (!method.equals("clear") && !method.equals("setContextMap")) return false; + var frames = Arrays.stream(Thread.currentThread().getStackTrace()).toList(); + return frames.stream() + .anyMatch(frame -> frame.getClassName().endsWith(".DurableExecutor") + && frame.getMethodName().startsWith("lambda$restoreMdcOnClose")) + && frames.stream() + .noneMatch(frame -> frame.getClassName().endsWith(".DurableExecutor") + && frame.getMethodName().equals("fireOnInvocationEnd")); + } + + private static void awaitGate(CountDownLatch gate) { + try { + assertTrue(gate.await(5, TimeUnit.SECONDS), "Controlled MDC gate was not released"); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError(failure); + } + } + + private static void awaitFinalizerJoin(Thread caller) { + var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (System.nanoTime() < deadline) { + if (caller.getState() == Thread.State.WAITING + && Arrays.stream(caller.getStackTrace()) + .anyMatch(frame -> frame.getClassName().endsWith(".DurableExecutor") + && frame.getMethodName().equals("finalizeAfterHandlerScopes"))) return; + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + } + fail("The invocation caller did not attach its finalizer before handler completion"); + } + + private static final class InlineExecutor extends AbstractExecutorService { + private boolean shutdown; + + @Override + public void execute(Runnable command) { + command.run(); + } + + @Override + public void shutdown() { + shutdown = true; + } + + @Override + public List shutdownNow() { + shutdown = true; + return List.of(); + } + + @Override + public boolean isShutdown() { + return shutdown; + } + + @Override + public boolean isTerminated() { + return shutdown; + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) { + return shutdown; + } + } + @ParameterizedTest @CsvSource({ "getCopyOfContextMap,false", diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index db086c292..d20fe02a1 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -18,6 +18,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -141,7 +142,8 @@ public static DurableExecutionOutput execute( } }, config.getExecutorService(), - !pluginRunner.isEmpty()); // Get executor from config for running user code + !pluginRunner.isEmpty(), + pluginRunner.isEmpty() ? null : executionManager::recordHandlerScopeFatal); // Execute the handlerFuture in ExecutionManager. If it completes successfully, the output of user function // will be returned. Otherwise, it will complete exceptionally with a SuspendExecutionException or a @@ -294,7 +296,7 @@ public static DurableExecutionOutput execute( /** Completes the observation future without absorbing fatal failures on an executor worker. */ private static CompletableFuture supplyAsync( - Supplier task, Executor executor, boolean preservePluginMdc) { + Supplier task, Executor executor, boolean preservePluginMdc, Consumer onPostCompletionFatal) { var result = new CompletableFuture(); var caller = Thread.currentThread(); Runnable work = (Runnable & CompletableFuture.AsynchronousCompletionTask) () -> { @@ -318,11 +320,23 @@ private static CompletableFuture supplyAsync( } } } catch (Throwable failure) { - result.completeExceptionally(failure); + var settled = result.completeExceptionally(failure); + Throwable normalizedFailure = null; + if (preservePluginMdc && !settled && !result.isCompletedExceptionally()) { + // Worker restoration follows completion callbacks. Report a fatal that follows a successful + // result through the existing owner-cleanup boundary: before end dispatch it changes the + // invocation outcome; after dispatch it only escapes the owner. An already-failed task retains + // its primary failure and normal try-with-resources suppression. + normalizedFailure = ExceptionHelper.unwrapAsyncFailure(failure); + if (isFatal(normalizedFailure)) onPostCompletionFatal.accept((Error) normalizedFailure); + } // A direct executor is already on the invocation caller; its fatal result is rethrown below after // finalization. On an asynchronous executor it must also escape the runnable so the worker terminates. if (Thread.currentThread() != caller) { - rethrowFatal(ExceptionHelper.unwrapAsyncFailure(failure)); + rethrowFatal( + normalizedFailure != null + ? normalizedFailure + : ExceptionHelper.unwrapAsyncFailure(failure)); } } }; From 7b19a27584b635c57aca449fbec03de584e30aa3 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 20:45:33 +0000 Subject: [PATCH 16/17] test: verify parallel tolerance without racing completion assumptions --- .../ParallelFailureToleranceExampleTest.java | 125 ++++++++++++++++-- 1 file changed, 115 insertions(+), 10 deletions(-) diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/parallel/ParallelFailureToleranceExampleTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/parallel/ParallelFailureToleranceExampleTest.java index f1518e2ac..c3425f295 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/parallel/ParallelFailureToleranceExampleTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/parallel/ParallelFailureToleranceExampleTest.java @@ -5,8 +5,27 @@ import static org.junit.jupiter.api.Assertions.*; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.model.ConcurrencyCompletionStatus; import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.model.ParallelResult; +import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.plugin.OperationEndInfo; +import software.amazon.lambda.durable.plugin.UserFunctionEndInfo; +import software.amazon.lambda.durable.plugin.UserFunctionStartInfo; +import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; class ParallelFailureToleranceExampleTest { @@ -41,19 +60,105 @@ void succeedsWhenAllBranchesSucceed() { assertEquals(3, output.succeeded()); } - @Test - void failsWhenFailuresExceedTolerance() { + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void failsWhenFailuresExceedTolerance(boolean holdHealthyBranch) throws Exception { var handler = new ParallelFailureToleranceExample(); - var runner = LocalDurableTestRunner.create(ParallelFailureToleranceExample.Input.class, handler); + var probe = new CompletionProbe(holdHealthyBranch); + var config = + DurableConfig.builder().withPlugins(info -> probe.newPlugin()).build(); + var runner = LocalDurableTestRunner.create(ParallelFailureToleranceExample.Input.class, handler) + .withDurableConfig(config); + var caller = Executors.newSingleThreadExecutor(); + try { + var input = new ParallelFailureToleranceExample.Input(List.of("svc-a", "bad-svc-b", "bad-svc-c"), 1, 2); + var invocation = caller.submit(() -> runner.runUntilComplete(input)); + if (holdHealthyBranch) { + await(probe.parallelStored); + probe.releaseHealthy.countDown(); + } + var result = invocation.get(5, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + var output = result.getResult(ParallelFailureToleranceExample.Output.class); + assertEquals(2, output.failed()); - // 2 bad services, toleratedFailureCount=1 — second failure exceeds tolerance - var input = new ParallelFailureToleranceExample.Input(List.of("svc-a", "bad-svc-b", "bad-svc-c"), 1, 2); - var result = runner.runUntilComplete(input); + var operation = result.getOperation("call-services"); + assertEquals(OperationStatus.SUCCEEDED, operation.getStatus()); + var stored = new JacksonSerDes() + .deserialize(operation.getContextDetails().result(), TypeToken.get(ParallelResult.class)); + assertEquals(ConcurrencyCompletionStatus.FAILURE_TOLERANCE_EXCEEDED, stored.completionStatus()); + assertFalse(stored.completionStatus().isSucceeded()); + assertEquals(3, stored.size()); + assertEquals( + List.of(ParallelResult.Status.FAILED, ParallelResult.Status.FAILED), + stored.statuses().subList(1, 3)); + assertTrue(List.of(ParallelResult.Status.SUCCEEDED, ParallelResult.Status.SKIPPED) + .contains(stored.statuses().get(0))); + assertEquals(output.succeeded(), stored.succeeded()); + assertEquals(output.failed(), stored.failed()); + assertEquals(1 - output.succeeded(), stored.skipped()); + if (holdHealthyBranch) assertEquals(0, output.succeeded()); - assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + var completedCalls = result.getOperations().stream() + .filter(op -> op.getType() == OperationType.STEP && op.isCompleted()) + .collect(Collectors.toMap(op -> op.getName(), op -> probe.completedCalls.get(op.getName()))); + assertEquals(1, completedCalls.get("invoke-bad-svc-b")); + assertEquals(1, completedCalls.get("invoke-bad-svc-c")); + var replay = runner.run(input); + assertEquals(ExecutionStatus.SUCCEEDED, replay.getStatus()); + assertEquals(output, replay.getResult(ParallelFailureToleranceExample.Output.class)); + completedCalls.forEach((name, count) -> assertEquals( + count, + probe.completedCalls.get(name), + "Completed step bodies must not run again on replay: " + name)); + } finally { + probe.releaseHealthy.countDown(); + caller.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } - var output = result.getResult(ParallelFailureToleranceExample.Output.class); - assertEquals(2, output.failed()); - assertEquals(1, output.succeeded()); + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(5, TimeUnit.SECONDS), "Controlled branch scheduling was not released"); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError(failure); + } + } + + private static final class CompletionProbe { + final boolean holdHealthy; + final CountDownLatch healthyStarted = new CountDownLatch(1); + final CountDownLatch releaseHealthy = new CountDownLatch(1); + final CountDownLatch parallelStored = new CountDownLatch(1); + final Map completedCalls = new ConcurrentHashMap<>(); + + CompletionProbe(boolean holdHealthy) { + this.holdHealthy = holdHealthy; + } + + DurableExecutionPlugin newPlugin() { + return new DurableExecutionPlugin() { + @Override + public void onUserFunctionStart(UserFunctionStartInfo info) { + if (!holdHealthy || !"STEP".equals(info.type())) return; + if ("invoke-svc-a".equals(info.name())) { + healthyStarted.countDown(); + await(releaseHealthy); + } else if (info.name().startsWith("invoke-bad-")) await(healthyStarted); + } + + @Override + public void onUserFunctionEnd(UserFunctionEndInfo info) { + if ("STEP".equals(info.type())) completedCalls.merge(info.name(), 1, Integer::sum); + } + + @Override + public void onOperationEnd(OperationEndInfo info) { + if ("call-services".equals(info.name())) parallelStored.countDown(); + } + }; + } } } From fc75e95f6bedd3105c30930be7c91cb5db03fe40 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 7 Oct 2026 21:57:32 +0000 Subject: [PATCH 17/17] fix: bound peer cleanup for selected invocation fatals --- .../InvocationFatalPeerCleanupTest.java | 176 ++++++++++++++++++ .../durable/execution/DurableExecutor.java | 2 + .../durable/execution/ExecutionManager.java | 7 +- 3 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/InvocationFatalPeerCleanupTest.java diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/InvocationFatalPeerCleanupTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/InvocationFatalPeerCleanupTest.java new file mode 100644 index 000000000..55b1cf7ce --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/InvocationFatalPeerCleanupTest.java @@ -0,0 +1,176 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.plugin.*; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.testing.TestResult; +import software.amazon.lambda.durable.util.ExceptionHelper; + +class InvocationFatalPeerCleanupTest { + static Stream fatalCases() { + return Stream.of("handler", "output") + .flatMap(stage -> Stream.of(false, true) + .flatMap(plugins -> Stream.of(false, true) + .flatMap(ignores -> Stream.of(false, true) + .map(threadDeath -> Arguments.of(stage, plugins, ignores, threadDeath))))); + } + + @SuppressWarnings("removal") + @ParameterizedTest + @MethodSource("fatalCases") + void selectedFatalBoundsPeerCleanupAndFinalizesOnce( + String stage, boolean plugins, boolean ignoresInterrupt, boolean threadDeath) throws Exception { + Error fatal = threadDeath ? new ThreadDeath() : new InternalError("invocation fatal"); + try (var run = new Run(stage, plugins, ignoresInterrupt, fatal)) { + var response = run.start(); + await(run.peerEntered); + var failure = assertThrows(ExecutionException.class, () -> response.get(2, TimeUnit.SECONDS)); + assertSame(fatal, ExceptionHelper.unwrapAsyncFailure(failure)); + assertTrue(run.interrupted.get(), "selected fatal requests cooperative cancellation"); + assertEquals(!ignoresInterrupt, run.peerExited.get()); + if (plugins) { + assertEquals(1, run.ends.get()); + assertEquals(InvocationStatus.RETRYING, run.end.get().invocationStatus()); + assertSame(fatal, run.end.get().executionError()); + assertEquals(!ignoresInterrupt, run.peerCleanedAtEnd.get()); + assertFalse(run.wrongOwner.get()); + } + run.releasePeer.countDown(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void ordinaryRootFailureKeepsExistingPeerWaitAndDoesNotInterrupt(boolean plugins) throws Exception { + try (var run = new Run("handler", plugins, false, new IllegalStateException("ordinary root failure"))) { + var response = run.start(); + await(run.peerEntered); + assertThrows(TimeoutException.class, () -> response.get(100, TimeUnit.MILLISECONDS)); + assertFalse(run.interrupted.get()); + run.releasePeer.countDown(); + var result = response.get(2, TimeUnit.SECONDS); + assertEquals(ExecutionStatus.FAILED, result.getStatus()); + assertEquals( + "ordinary root failure", result.getError().orElseThrow().errorMessage()); + if (plugins) assertEquals(1, run.ends.get()); + } + } + + private static final class Run implements AutoCloseable { + final CountDownLatch peerEntered = new CountDownLatch(1); + final CountDownLatch releasePeer = new CountDownLatch(1); + final AtomicBoolean interrupted = new AtomicBoolean(); + final AtomicBoolean peerExited = new AtomicBoolean(); + final AtomicBoolean peerCleanedAtEnd = new AtomicBoolean(); + final AtomicBoolean wrongOwner = new AtomicBoolean(); + final AtomicReference peerOwner = new AtomicReference<>(); + final AtomicInteger peerEnds = new AtomicInteger(); + final AtomicInteger ends = new AtomicInteger(); + final AtomicReference end = new AtomicReference<>(); + final ExecutorService workers = Executors.newCachedThreadPool(task -> { + var thread = new Thread(task, "invocation-fatal-peer"); + thread.setDaemon(true); + thread.setUncaughtExceptionHandler((owner, failure) -> {}); + return thread; + }); + final ExecutorService caller = Executors.newSingleThreadExecutor(); + final LocalDurableTestRunner runner; + + Run(String stage, boolean plugins, boolean ignoresInterrupt, Throwable failure) { + SerDes serde = new SerDes() { + private final JacksonSerDes delegate = new JacksonSerDes(); + + public String serialize(Object value) { + if (stage.equals("output") && "root-result".equals(value)) ExceptionHelper.sneakyThrow(failure); + return delegate.serialize(value); + } + + public T deserialize(String value, TypeToken type) { + return delegate.deserialize(value, type); + } + }; + var config = DurableConfig.builder() + .withExecutorService(workers) + .withSerDes(serde) + .withCheckpointDelay(Duration.ZERO); + if (plugins) + config.withPlugins(info -> new DurableExecutionPlugin() { + @Override + public void onUserFunctionEnd(UserFunctionEndInfo info) { + if ("peer".equals(info.name())) { + wrongOwner.set(Thread.currentThread() != peerOwner.get()); + peerEnds.incrementAndGet(); + } + } + + @Override + public void onInvocationEnd(InvocationEndInfo info) { + end.set(info); + peerCleanedAtEnd.set(peerEnds.get() == 1); + ends.incrementAndGet(); + } + }); + runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + context.stepAsync("peer", String.class, step -> { + peerOwner.set(Thread.currentThread()); + peerEntered.countDown(); + try { + while (releasePeer.getCount() != 0) { + try { + releasePeer.await(); + } catch (InterruptedException stop) { + interrupted.set(true); + if (!ignoresInterrupt) return "interrupted"; + } + } + return "released"; + } finally { + peerExited.set(true); + } + }); + await(peerEntered); + if (stage.equals("handler")) ExceptionHelper.sneakyThrow(failure); + return "root-result"; + }, + config.build()); + } + + Future> start() { + return caller.submit(() -> runner.run("input")); + } + + @Override + public void close() throws Exception { + releasePeer.countDown(); + caller.shutdown(); + workers.shutdown(); + assertTrue(caller.awaitTermination(3, TimeUnit.SECONDS)); + assertTrue(workers.awaitTermination(3, TimeUnit.SECONDS)); + } + } + + private static void await(CountDownLatch latch) { + try { + assertTrue(latch.await(3, TimeUnit.SECONDS), "Peer did not enter"); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError(failure); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index d20fe02a1..c2a4223b0 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -517,6 +517,8 @@ private static void fireOnInvocationEnd( Throwable error, Object executionInput, Object executionResult) { + // The selected failure can originate in user code or result delivery, without a plugin callback reporting it. + if (isFatal(error)) executionManager.recordInvocationFatal((Error) error); if (pluginRunner.isEmpty()) { return; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index c952dfd31..57ac0bf6e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -725,7 +725,7 @@ private synchronized long fatalCleanupDeadline(long current) { return fatalOperationCleanupDeadline; } - /** Gives cooperative operation owners their existing fatal cleanup allowance before plugin finalization. */ + /** Gives cooperative operation owners their shared fatal cleanup allowance before invocation finalization. */ void awaitFatalOperationCleanup() { if (pluginFatal.get() == null) return; registeredOperations.values().forEach(BaseDurableOperation::interruptRunningUserHandler); @@ -776,6 +776,11 @@ public boolean isExecutionCompletedExceptionally() { return executionExceptionFuture.isCompletedExceptionally(); } + /** Applies the existing shutdown budget to a root or result-delivery fatal already selected by the invocation. */ + void recordInvocationFatal(Error fatal) { + failFromPlugin(fatal); + } + private void failFromPlugin(Error fatal) { var original = recordPluginFatal(fatal, false); executionExceptionFuture.completeExceptionally(original);