completedCalls = new ConcurrentHashMap<>();
+
+ CompletionProbe(boolean holdHealthy) {
+ this.holdHealthy = holdHealthy;
+ }
+
+ DurableExecutionPlugin newPlugin() {
+ return new DurableExecutionPlugin() {
+ @Override
+ public void onUserFunctionStart(UserFunctionStartInfo info) {
+ if (!holdHealthy || !"STEP".equals(info.type())) return;
+ if ("invoke-svc-a".equals(info.name())) {
+ healthyStarted.countDown();
+ await(releaseHealthy);
+ } else if (info.name().startsWith("invoke-bad-")) await(healthyStarted);
+ }
+
+ @Override
+ public void onUserFunctionEnd(UserFunctionEndInfo info) {
+ if ("STEP".equals(info.type())) completedCalls.merge(info.name(), 1, Integer::sum);
+ }
+
+ @Override
+ public void onOperationEnd(OperationEndInfo info) {
+ if ("call-services".equals(info.name())) parallelStored.countDown();
+ }
+ };
+ }
}
}
diff --git a/insight-plugin/README.md b/insight-plugin/README.md
index 75d4e5132..694cf0df9 100644
--- a/insight-plugin/README.md
+++ b/insight-plugin/README.md
@@ -192,24 +192,29 @@ one event per record to `{logStreamPrefix}YYYY/MM/DD` (IAM: `logs:CreateLogStrea
logs the failure rather than silently dropping it or failing the execution.
- **`includeErrors`** gates **both** the execution-level error and each operation-level error; with
`includeErrors(false)` neither is emitted, so a sensitive failure message never reaches a record.
-- **Plugin failures never disrupt execution.** Every plugin-owned boundary — record construction,
- input snapshotting, transforms, truncation, and each exporter's render/export/flush (including a
- `NoClassDefFoundError` from an optional exporter's absent SDK) — is guarded against any `Throwable`
- and logged, so one failing exporter cannot block the others and no plugin fault propagates into
- the durable execution.
+- **Non-fatal plugin failures are isolated.** Record construction, snapshotting, transforms,
+ truncation and exporter render/export/flush contain and log ordinary failures, including
+ `AssertionError` and optional-dependency `NoClassDefFoundError`. `VirtualMachineError` and
+ `ThreadDeath` propagate as their original error objects, including through future, reflection
+ and Jackson transport wrappers. Arbitrary business-exception causes are not reclassified.
+ A fatal failure stops the shared scheduler: queued records are released and drain/flush waiters
+ fail, including when another exporter is blocked. Subsequent scheduler calls rethrow that fatal;
+ a finished fatal pump cannot later be mistaken for a successful drain or flush. Already-running
+ customer exporter code cannot be forcibly stopped, but it does not delay fatal delivery.
- **Per-exporter size truncation** (`Truncation`) drops, in order: operation results oldest-first,
then whole operations oldest-first, then execution input, then output — setting `truncated`,
`droppedOperations`, `droppedInput`, `droppedOutput` as applicable. The size is measured against
the exact shape each exporter emits (its `render`).
- **Exporter isolation.** Every exporter receives its own copy of each record, truncated to its own
- limit; a failing or slow exporter is logged and never blocks the others or the execution.
+ limit; ordinary exporter failures are logged without preventing the other exporters from running.
- **Export scheduling.** Exporter I/O never runs on the SDK threads that deliver plugin hooks.
Records are handed to a background worker that exports at most one record at a time; each
record is a complete snapshot of its execution, so while an export is in flight newer updates for
- the same execution coalesce into that execution's pending slot and only the latest is exported
- next; records of different executions never displace each other. At invocation end the plugin waits
- for the queue to drain and then flushes every exporter once, so the final record is always
- delivered before the invocation returns.
+ the same invocation coalesce into that invocation's pending slot and only the latest is exported
+ next; records of different invocations never displace each other. At invocation end the plugin waits
+ for its latest record to reach every exporter, then requests a flush through the same worker.
+ Concurrent flush requests may share one flush, and the final record is delivered before the
+ invocation returns.
## Conformance
diff --git a/insight-plugin/pom.xml b/insight-plugin/pom.xml
index c2cbe8df5..cc949030f 100644
--- a/insight-plugin/pom.xml
+++ b/insight-plugin/pom.xml
@@ -7,7 +7,7 @@
software.amazon.lambda.durable
aws-durable-execution-sdk-java-parent
- 2.2.2-SNAPSHOT
+ 3.0.0-SNAPSHOT
aws-durable-execution-sdk-java-plugin-insight
diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/ContentConfig.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/ContentConfig.java
index 3a8228ec9..0ecc1e38b 100644
--- a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/ContentConfig.java
+++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/ContentConfig.java
@@ -21,8 +21,8 @@
* detached, JSON-compatible copy of the value, never the SDK's original Java object: a POJO is presented as a
* {@code Map}, a list as a {@code List}, and a Java-time type as its JSON representation (for example an
* {@code Instant} arrives as an ISO-8601 {@code String}). Mutating the argument is therefore safe — it cannot corrupt
- * the cached input snapshot or any later emission — and a transform that throws omits the field (the failure is logged)
- * rather than failing the execution.
+ * the cached input snapshot or any later emission — and a non-fatal transform failure omits the field and is logged.
+ * VM/thread-termination errors propagate.
*/
@Experimental
public final class ContentConfig {
diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/ExportScheduler.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/ExportScheduler.java
index 84e8be3c2..9731683a9 100644
--- a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/ExportScheduler.java
+++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/ExportScheduler.java
@@ -2,37 +2,82 @@
// SPDX-License-Identifier: Apache-2.0
package software.amazon.lambda.durable.insight;
+import java.util.ArrayDeque;
import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
+import java.util.Deque;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
+import software.amazon.lambda.durable.insight.internal.FatalErrors;
/**
- * Serializes record exports so that, at most, one export runs at a time.
+ * Serializes record exports so that, at most, one export runs at a time, while keeping the records of concurrently
+ * running executions independent.
*
- * Each {@link WorkflowInsightRecord} is a complete snapshot of its execution, so a newer record for the
- * same execution fully supersedes any record of that execution still waiting to be exported. While an export is in
- * flight, additional updates are coalesced into a single pending slot per execution — intermediate records are dropped
- * because the latest one already contains all of their information. Records of different executions never displace each
- * other: one execution's snapshot carries none of another's information, so each execution keeps its own slot and the
- * slots are served in the order the executions first became pending. This prevents overlapping {@code export()} calls
- * when updates arrive faster than the exporters can keep up, and it keeps exporter I/O off the SDK threads that deliver
- * plugin hooks.
+ *
One scheduler serves the whole execution environment — the exporters it fans out to belong to the environment, and
+ * an environment can host several durable executions at the same time (Lambda Managed Instances makes that routine).
+ * The work, by contrast, is held on the invocation it belongs to: each {@link InsightPlugin} instance is one
+ * invocation's plugin and owns that invocation's latest-record slot, drain signal, mid-export marker and drain-waiter
+ * count. Coalescing happens only within one invocation, because the slot belongs to it.
*
- *
Exports are otherwise fire-and-forget; {@link #drain()} is called before the invocation returns to guarantee the
- * final record is delivered.
+ *
That shape is not a tidying-up. The same facts once lived in five structures keyed by execution ARN — the plugin's
+ * state map plus this class's {@code pending}, {@code settled}, {@code exporting} and {@code drainWaiters} — and two of
+ * them could disagree about one execution. They did: "nothing queued for this ARN" was read as "nothing outstanding",
+ * which is equally true of a record already taken and being exported, so an exiting pump completed another execution's
+ * drain signal mid-export and that invocation returned before its record was delivered. Every fact now exists exactly
+ * once, as a field of the one object the SDK gave that invocation, so the disagreement has no place to happen — and
+ * since the SDK creates that object and drops it, the scheduler has no execution registry to keep in step with
+ * anything.
+ *
+ *
Each {@link WorkflowInsightRecord} is a complete snapshot of its execution, so a newer record from the same
+ * invocation fully supersedes one still waiting to be exported. While an export is in flight, additional updates from
+ * that invocation are coalesced into its slot — intermediate records are dropped because the latest one already
+ * contains all of their information. A record from a different invocation never displaces another's record.
+ *
+ *
The slot takes whichever record is handed to it last and compares nothing, so "newer" has to be established before
+ * the hand-off. Customer code runs while a record is being built and can re-enter a hook of the same invocation, which
+ * builds and hands over a newer record first; the build it re-entered from then hands over an older snapshot last.
+ * {@code InsightPlugin}'s build revision identifies each build and {@link #scheduleIfNotSuperseded} drops a record
+ * whose build has been overtaken, so the slot only ever advances.
+ *
+ *
A single pump exports the queued records one at a time, in the order the invocations first queued work
+ * ({@link #queue}, which is ordering only — membership in it is the same fact as "this invocation has a record",
+ * written in one place), so exporters still never see two exports at once and each record keeps its per-exporter
+ * fan-out. {@link #flush()} requests are served by that same pump, between records, so an exporter never sees a
+ * {@code flush()} overlap an {@code export()} either. Requests are served as a batch — the cadence is at most one flush
+ * per requesting invocation end, not exactly one — and a flush is preceded by the queued records a drain is waiting
+ * for, so a burst of invocation ends is covered by one flush rather than one each. A request made while a flush runs
+ * waits for the next turn. Exports are otherwise fire-and-forget; {@link #drain(InsightPlugin)} is called before an
+ * invocation returns and waits for that invocation's own latest record to reach every exporter.
+ *
+ *
Because there is one pump, that wait can also cover records other invocations had already queued ahead of this
+ * one: a drain is not isolated from the queue's head-of-line cost. What per-invocation ownership guarantees is that
+ * another invocation's record can never displace this one — records coalesce only within their own invocation,
+ * and a drain cannot return until its own latest record has reached every exporter.
*/
final class ExportScheduler {
private static final AtomicInteger THREAD_NUMBER = new AtomicInteger();
+ /**
+ * Upper bound on the passes {@link #drainAll()} makes over the outstanding invocations. Only reached if new work
+ * keeps arriving for as long as the drain runs; a normal drain settles in two passes.
+ */
+ private static final int MAX_DRAIN_ALL_PASSES = 1_000;
+
+ /** How long one {@link #drainAll()} pass waits for a running pump before taking another pass. */
+ private static final long PUMP_WAIT_MILLIS = 50;
+
/** Shared for the process lifetime; idle daemon workers are reclaimed, so nothing keeps the runtime alive. */
private static final ExecutorService WORKERS = Executors.newCachedThreadPool(runnable -> {
var thread = new Thread(runnable, "workflow-insight-export-" + THREAD_NUMBER.incrementAndGet());
@@ -48,11 +93,68 @@ final class ExportScheduler {
/** Completes when the current pump finishes; {@code null} while idle. Guarded by {@code this}. */
private CompletableFuture inFlight;
+ // A fatal worker failure is terminal for this shared scheduler. Keep it observable even
+ // after its pump/owner signals have been released, and wake fan-out joins immediately.
+ private volatile Error fatalFailure;
+ private final CompletableFuture fatalSignal = new CompletableFuture<>();
+ private InsightPlugin activeExecution; // guarded by this monitor
+
+ /**
+ * The thread serving the pump right now, or {@code null} while no pump is running. Deliberately not
+ * guarded by {@code this}: it is read by {@link #flush()} and the drains before they touch anything else, and a
+ * lock acquisition there would put the pump's own monitor on the path of every invocation end.
+ *
+ * Written only by the thread that enters {@link #pump} — a worker, or the caller's own thread on the
+ * rejected-worker fallback, which is a case where the calling thread genuinely is the pump — and cleared
+ * by that same thread on the way out, only if it is still the recorded one. A compare-and-set on the way out rather
+ * than a blind clear: if some anomaly ever did leave two pumps running, the one that finishes first must not clear
+ * the other, and must not leave a stale thread behind that a later, legitimate {@code flush()} from that same
+ * thread would be mistaken for.
+ */
+ private final AtomicReference pumpThread = new AtomicReference<>();
+
+ /**
+ * Marks the thread currently running one exporter's share of a fan-out for this scheduler, so that a
+ * {@code flush()} or {@code drain()} re-entered from an exporter callback can tell that the pump is waiting for it.
+ *
+ * With a single exporter the fan-out runs on the pump thread and {@link #pumpThread} already recognizes it. With
+ * two or more, {@link #forEachExporterSettled} submits one task per exporter and the pump then joins them all, so
+ * the callback runs on a thread that is not the pump but that the pump cannot outlive: a wait for the pump issued
+ * from there is the same wait-for cycle, two threads wide instead of one. The pump parks in the join, so it never
+ * reaches the point in its loop that would complete the future the worker is parked on.
+ *
+ *
An instance field rather than a static: a fan-out worker of one scheduler is not pump-dependent on any other
+ * scheduler, and refusing its waits there would be a false positive. Set and cleared around each callback by the
+ * thread that runs it, restoring whatever was there before rather than blindly removing, so a callback that the
+ * pump ran inline (the rejected-worker fallback, where the fan-out thread is the pump) cannot clear a mark
+ * an enclosing frame still needs.
+ */
+ private final ThreadLocal exporterFanOutThread = new ThreadLocal<>();
+
+ /**
+ * The invocations with a record no pump has picked up yet, in the order they first queued work. Ordering only — the
+ * record itself lives on the invocation's plugin instance. Guarded by {@code this}.
+ *
+ * This is the only collection of per-invocation objects the scheduler has, and it holds an instance for exactly
+ * as long as that instance has a record waiting: nothing here has to be cleaned up at an invocation boundary, and
+ * an instance the SDK has dropped is unreachable from the scheduler the moment its last record is taken.
+ *
+ *
Invariant, and the only thing that could still be said twice: an invocation is in here exactly while its
+ * {@link InsightPlugin#record} is non-null. Every record moves through {@link #queueRecord}, {@link #takeRecord} or
+ * {@link #dropRecord}, which write both halves together, and a set makes a double entry impossible by construction.
+ */
+ private final Set queue = new LinkedHashSet<>();
+
/**
- * The latest record of each execution not yet picked up by the pump, keyed by execution ARN and served in the order
- * the executions first became pending. Guarded by {@code this}.
+ * One entry per outstanding {@link #flush()} request, in request order, completed when a {@code flush()} that
+ * started after that request was enqueued has reached every exporter. Guarded by {@code this}.
+ *
+ * A queue of requests rather than a single flag: the pump takes the requests that are queued when its turn
+ * begins and satisfies all of them with one flush, so concurrent invocation ends share a flush; a request enqueued
+ * while that flush runs stays in the queue for the next turn, because a flush already in progress cannot be shown
+ * to have seen the new requester's records.
*/
- private final LinkedHashMap pending = new LinkedHashMap<>();
+ private final Deque> flushRequests = new ArrayDeque<>();
ExportScheduler(
List exporters,
@@ -72,53 +174,214 @@ final class ExportScheduler {
this.executor = executor;
}
+ // --- Scheduling. ---
+
/**
- * Queues the latest record of its execution for export. If an export is already running, the record is held in that
- * execution's pending slot (replacing any earlier pending record of the same execution) and exported once the
- * in-flight export completes.
+ * Queues the latest record of one invocation for export, with no ordering check. If an export is already running,
+ * the record is held in that invocation's own slot (replacing only an earlier record of the same
+ * invocation) and exported once the pump reaches it.
+ *
+ * The slot takes whichever record is handed over last and does not compare record ages, so this is the right
+ * entry point only for a record that cannot be superseded. The plugin's RUNNING records go through
+ * {@link #scheduleIfNotSuperseded} and its final record through {@link #closeAndSchedule}; both add the ordering
+ * checks this one omits.
*/
- void schedule(WorkflowInsightRecord record) {
+ void schedule(InsightPlugin execution, WorkflowInsightRecord record) {
CompletableFuture handle;
synchronized (this) {
- // put() on an existing key keeps its position, so a chatty execution cannot jump ahead of a quieter one.
- pending.put(executionKey(record), record);
- if (inFlight != null) {
- return;
+ queueRecord(execution, record);
+ handle = claimPumpIfIdle();
+ }
+ startPump(handle);
+ }
+
+ /**
+ * Schedules a non-terminal record unless it has been superseded, which is two separate facts.
+ *
+ * The invocation may already have ended. No RUNNING snapshot may follow the final record, so
+ * {@link InsightPlugin#closed} rejects it.
+ *
+ *
A newer build of this same invocation may already have started. Customer code runs inside a build — the
+ * content transforms, an operation result transform, a serializer for a customer type — and can re-enter a hook, so
+ * the build that hands its record over last is not necessarily the build that started last. Without the revision
+ * check the slot would take that older snapshot and the newer one would be lost, or, if a pump had already taken
+ * the newer one, an exporter would see the older snapshot after the newer one.
+ *
+ *
The superseded record is dropped rather than queued. Nothing is lost: a record is a complete snapshot of one
+ * execution, so the record that superseded it carries everything it carries. That is the same property that makes
+ * the slot's coalescing sound.
+ *
+ *
Both checks and the hand-off are one critical section, on the monitor that owns both fields, so a record
+ * cannot pass the checks and then be queued after the record that supersedes it.
+ *
+ * @param buildRevision the revision the caller took before it started building this record
+ * @return whether the record was queued
+ */
+ boolean scheduleIfNotSuperseded(InsightPlugin execution, WorkflowInsightRecord record, long buildRevision) {
+ CompletableFuture handle;
+ synchronized (this) {
+ if (execution.closed || !execution.isNewestBuild(buildRevision)) {
+ return false;
}
- handle = new CompletableFuture<>();
- inFlight = handle;
+ queueRecord(execution, record);
+ handle = claimPumpIfIdle();
+ }
+ startPump(handle);
+ return true;
+ }
+
+ /**
+ * Marks the invocation ended and, when given a record, schedules it as the last one for that invocation.
+ *
+ * The final record is queued without the build-revision check {@link #scheduleIfNotSuperseded} makes. Customer
+ * code running inside the final record's build can start a newer RUNNING build, which would leave the final
+ * record's revision stale, and a checked hand-off would then drop it and leave a RUNNING snapshot as the
+ * execution's last exported state. Exempting it cannot let a stale record win, because {@code closed} is set in
+ * this same critical section and every RUNNING record handed over afterwards is rejected.
+ */
+ void closeAndSchedule(InsightPlugin execution, WorkflowInsightRecord finalRecord) {
+ if (finalRecord == null && execution.closed) {
+ // The idempotent second call from the hook's `finally`. A volatile read, so the common case of an
+ // invocation end that already scheduled its final record does not take the lock again.
+ return;
+ }
+ CompletableFuture handle = null;
+ synchronized (this) {
+ execution.closed = true;
+ if (finalRecord != null) {
+ queueRecord(execution, finalRecord);
+ handle = claimPumpIfIdle();
+ }
+ }
+ startPump(handle);
+ }
+
+ /**
+ * Claims the pump for the caller when none is running, returning the handle to run with, or {@code null} when a
+ * pump already owns the scheduler and will pick the work up. Caller holds the lock.
+ */
+ private CompletableFuture claimPumpIfIdle() {
+ if (inFlight != null) {
+ return null;
+ }
+ CompletableFuture handle = new CompletableFuture<>();
+ inFlight = handle;
+ return handle;
+ }
+
+ /** Starts a claimed pump on a worker; a no-op when the caller claimed nothing. */
+ private void startPump(CompletableFuture handle) {
+ if (handle == null) {
+ return;
}
try {
executor.execute(() -> pump(handle));
} catch (Throwable t) {
- // No worker could be started. Keep the pending record and return to idle so a later schedule() retries,
- // and drain() runs whatever is still pending on the calling thread before the invocation returns. Complete
- // the handle too: a drain() that already observed it must wake up and take that inline path.
+ rejectFatal(t);
+ // No worker could be started. Keep the queued record and return to idle so a later schedule() retries, and
+ // a drain runs whatever is still queued on the calling thread before the invocation returns. Complete the
+ // handle too: a drain that already observed it must wake up and take that inline path.
synchronized (this) {
if (inFlight == handle) {
inFlight = null;
}
}
- handle.complete(null);
+ completeSignal(handle);
reportFailure(t);
}
}
+ // --- The record slot: the two halves of "this invocation has a record queued", always written together. ---
+
+ /** Puts this invocation's latest record in its slot and makes sure it has a drain signal. Caller holds the lock. */
+ private void queueRecord(InsightPlugin execution, WorkflowInsightRecord record) {
+ throwIfFailed();
+ execution.record = record;
+ queue.add(execution);
+ if (execution.settled == null) {
+ execution.settled = new CompletableFuture<>();
+ }
+ }
+
+ /**
+ * Takes this invocation's queued record for export and marks it mid-export. Caller holds the lock and has checked
+ * that a record is there.
+ *
+ * The marking is not a separate step in a separate structure: leaving the slot and becoming "inside the
+ * exporters" are one write of one object, so no reader can see the invocation between the two and conclude it has
+ * nothing outstanding.
+ */
+ private WorkflowInsightRecord takeRecord(InsightPlugin execution) {
+ WorkflowInsightRecord record = execution.record;
+ execution.record = null;
+ queue.remove(execution);
+ execution.exporting = true;
+ activeExecution = execution;
+ return record;
+ }
+
+ /** Drops this invocation's queued record without exporting it. Caller holds the lock. */
+ private void dropRecord(InsightPlugin execution) {
+ execution.record = null;
+ queue.remove(execution);
+ }
+
+ // --- Draining. ---
+
/**
- * Waits for any in-flight and pending exports to complete. Safe to call when idle. Used before the invocation
- * returns to guarantee the final record is delivered.
+ * Waits until the latest record of one invocation has been handed to every exporter. Safe to call when that
+ * invocation has nothing outstanding. Used before the invocation returns to guarantee the final record is
+ * delivered.
+ *
+ *
The wait is for that invocation's own latest record. Another invocation's record can never displace it, so
+ * this always returns having delivered this invocation's latest snapshot; but since one pump exports serially, the
+ * wait can also cover records other invocations had already queued ahead of it.
+ *
+ *
While this waits, the invocation counts a drain waiter — on the instance itself, so the count cannot come to
+ * describe a different one. It tells the pump that this record gates an invocation return, so the pump exports it
+ * before spending a flush fan-out. See {@link #exportRecordsADrainIsWaitingFor}.
+ *
+ *
Called from the pump thread itself, or from an exporter fan-out worker that pump is waiting for, the wait is
+ * refused and reported instead of made: see {@link #refuseWaitThatWouldBlockThePump}.
*/
- void drain() {
+ void drain(InsightPlugin execution) {
+ throwIfFailed();
+ // Re-entered from a thread the pump's progress depends on: waiting here would park on a signal only that pump
+ // can settle. Refuse and return; the record stays queued and that same pump exports it once it resumes its
+ // loop.
+ if (refuseWaitThatWouldBlockThePump("drain(execution)")) {
+ return;
+ }
+ synchronized (this) {
+ throwIfFailed();
+ if (execution.settled == null) {
+ return;
+ }
+ execution.drainWaiters++;
+ }
+ try {
+ drainUntilSettled(execution);
+ } finally {
+ synchronized (this) {
+ execution.drainWaiters--;
+ }
+ }
+ }
+
+ private void drainUntilSettled(InsightPlugin execution) {
while (true) {
+ CompletableFuture signal;
CompletableFuture handle;
boolean runInline = false;
synchronized (this) {
+ throwIfFailed();
+ signal = execution.settled;
+ if (signal == null) {
+ return;
+ }
handle = inFlight;
if (handle == null) {
- if (pending.isEmpty()) {
- return;
- }
- // Records are pending with no pump running (a worker could not be started): export them here.
+ // A record is outstanding with no pump running (a worker could not be started): export it here.
handle = new CompletableFuture<>();
inFlight = handle;
runInline = true;
@@ -126,52 +389,447 @@ void drain() {
}
if (runInline) {
pump(handle);
- } else {
- handle.join();
+ if (nothingCanSettle(execution, signal)) {
+ // The pump this thread just ran found no record for this invocation and left none inside the
+ // exporters, yet the signal survives: no later step can complete it, so waiting again would only
+ // start empty pumps forever. Release it here and report — the pump's own exit does not sweep for
+ // this any more, because it has no registry of invocations to sweep and does not need one: the
+ // thread that would be stranded is this one, and it holds the instance.
+ abandon(execution);
+ reportFailure(new IllegalStateException(
+ "a drain signal survived a pump that had nothing to export for it; the drain was released"
+ + " rather than waiting for work nobody will do"));
+ return;
+ }
+ continue;
+ }
+ // Wake either when this invocation's record has been exported or when the current pump ends — the pump may
+ // have ended without taking this record (a rejected worker), in which case the loop re-evaluates and
+ // exports it inline.
+ try {
+ CompletableFuture.anyOf(signal, handle).join();
+ } catch (Throwable t) {
+ // Never spin on an unexpected wait failure, and never let it escape into the execution. Abandon this
+ // invocation's outstanding record instead of leaving it queued: WORKERS is a static, process-wide pool,
+ // so a record left queued here would be exported later by some unrelated execution's pump — out of
+ // order, and after this invocation has already returned. Completing the signal also releases any other
+ // drain waiting on the same invocation rather than stranding it behind work nobody will do.
+ abandon(execution);
+ reportFailure(t);
+ return;
}
}
}
+ /**
+ * True when this invocation still holds the same drain signal but has no queued record and none inside the
+ * exporters, so nothing that could complete the signal is left. No ordinary path produces that; this is the
+ * liveness backstop for the unwinds that are hard to enumerate exhaustively.
+ */
+ private synchronized boolean nothingCanSettle(InsightPlugin execution, CompletableFuture signal) {
+ return execution.settled == signal && execution.record == null && !execution.exporting;
+ }
+
+ /**
+ * Waits for every outstanding record. Test seam for an environment-wide drain; the per-invocation path uses
+ * {@link #drain(InsightPlugin)}.
+ *
+ * Returns once nothing is queued and no pump owns the scheduler, which is exactly "every record scheduled so far
+ * has reached the exporters": a pump only returns to idle with its queue empty and the record it took settled.
+ *
+ *
Bounded by the number of passes, not by the set of invocations seen: an invocation that queues new work after
+ * it was already drained must still be waited for (dropping it would silently weaken every assertion made after
+ * this returns), while a producer that never stops cannot keep this spinning forever.
+ */
+ void drainAll() {
+ throwIfFailed();
+ // Every pass below is a drain, and each one would be refused; without this the loop spends all of its passes
+ // reporting the same refusal.
+ if (refuseWaitThatWouldBlockThePump("drainAll()")) {
+ return;
+ }
+ for (int pass = 0; pass < MAX_DRAIN_ALL_PASSES; pass++) {
+ List outstanding;
+ CompletableFuture handle;
+ synchronized (this) {
+ throwIfFailed();
+ outstanding = new ArrayList<>(queue);
+ handle = inFlight;
+ if (outstanding.isEmpty() && handle == null) {
+ return;
+ }
+ }
+ for (InsightPlugin execution : outstanding) {
+ drain(execution);
+ }
+ if (outstanding.isEmpty()) {
+ // Nothing is queued for anyone, but a pump still owns the scheduler: it may be inside an exporter with
+ // a
+ // record whose only reference is its own local variable, and with no registry of invocations there is
+ // no
+ // way to name that record and drain it. Waiting for the pump itself covers it — in bounded steps, so a
+ // producer that keeps the pump permanently busy cannot make this unbounded, and so the wait is a real
+ // wait rather than a re-poll.
+ try {
+ handle.get(PUMP_WAIT_MILLIS, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException e) {
+ // Still running; take another pass.
+ } catch (Throwable t) {
+ reportFailure(t);
+ return;
+ }
+ }
+ }
+ }
+
+ /**
+ * Test seam: how many invocations the scheduler still holds a reference to.
+ *
+ * {@link #queue} is the only collection of per-invocation objects the scheduler has, so this is the whole of the
+ * per-invocation state the environment retains. Zero means the environment — which outlives every invocation —
+ * holds nothing belonging to any invocation it has served.
+ */
+ synchronized int retainedInvocationCount() {
+ return queue.size();
+ }
+
+ /** Test seam: whether the scheduler still holds a reference to one particular invocation. */
+ synchronized boolean retains(InsightPlugin execution) {
+ return queue.contains(execution);
+ }
+
+ /** Gives up one invocation's outstanding work: drops its queued record and releases every drain waiting on it. */
+ private void abandon(InsightPlugin execution) {
+ CompletableFuture signal;
+ synchronized (this) {
+ dropRecord(execution);
+ execution.exporting = false;
+ signal = execution.settled;
+ execution.settled = null;
+ }
+ if (signal != null) {
+ completeSignal(signal);
+ }
+ }
+
+ // --- The pump. ---
+
private void pump(CompletableFuture handle) {
+ // Recorded for as long as this thread serves the pump — a worker, or a caller pumping inline after a rejected
+ // worker — so that a flush() or drain() re-entered from anything the fan-out calls synchronously can tell that
+ // it is asking itself. One atomic write per pump, and no lock: see the field.
+ Thread self = Thread.currentThread();
+ pumpThread.set(self);
+ // The invocation this pump has taken a record from and not settled yet. Only this pump may release it, so an
+ // abnormal unwind cannot strand a drain, and no other pump can mistake it for orphaned work.
+ InsightPlugin taken = null;
+ // Likewise for the flush requests this pump has taken out of the queue and not completed yet.
+ List> takenFlushes = null;
try {
- // Serve pending executions in order until none is left. Taking a record and returning to idle both happen
- // under the lock, so an update scheduled at any point is either exported by this pump or starts the next
- // one — never lost.
+ // One record, then every flush request queued at that moment, alternating. Taking the record and returning
+ // to idle both happen under the lock, so a record scheduled at any point is either exported by this pump or
+ // starts the next one — never lost, and never displaced by another invocation's record. A flush therefore
+ // waits at most one fan-out (it cannot be starved by a queue that never runs dry) and still never overlaps
+ // an export, because this loop runs them one after the other.
+ //
+ // A loop, deliberately, not a pump that re-enters itself to pick up the next item: written that way, one
+ // frame per queued item accumulates until the stack overflows, and the rest of the queue is dropped.
while (true) {
- WorkflowInsightRecord record;
+ InsightPlugin next = null;
+ WorkflowInsightRecord record = null;
synchronized (this) {
- Iterator head = pending.values().iterator();
- if (!head.hasNext()) {
- inFlight = null;
+ throwIfFailed();
+ if (queue.isEmpty() && flushRequests.isEmpty()) {
+ if (inFlight == handle) {
+ inFlight = null;
+ }
return;
}
- record = head.next();
- head.remove();
+ if (!queue.isEmpty()) {
+ next = queue.iterator().next();
+ // Leaving the slot and being marked mid-export are one write of one object, so the invocation
+ // is
+ // never momentarily indistinguishable from one with nothing outstanding.
+ record = takeRecord(next);
+ }
+ }
+ if (next != null) {
+ taken = next;
+ try {
+ exportToAll(record);
+ } finally {
+ signalSettled(next);
+ taken = null;
+ }
+ }
+ // Taken only now that the fan-out above has settled, and taken as a batch: every request queued at this
+ // instant is satisfied by the single flush below, so invocation ends that ask together cost one flush
+ // rather than one each. Sound because each requester drained its own record before asking, so a flush
+ // that starts after the request was enqueued already has that record in the exporter's buffer.
+ //
+ // Emptying the queue here — rather than after the flush — is what keeps a request that arrives while
+ // that flush runs out of this batch: it lands in the now-empty queue and is served by the next turn,
+ // never credited to a flush that was already in progress when it was made.
+ synchronized (this) {
+ if (!flushRequests.isEmpty()) {
+ takenFlushes = new ArrayList<>(flushRequests);
+ flushRequests.clear();
+ }
+ }
+ if (takenFlushes != null) {
+ // Before spending the fan-out: export the queued records other invocations are still waiting on.
+ // Those ends cannot have asked for their flush yet — they are inside a drain — so without this the
+ // pump staggers them one record per turn, with a whole flush in between, and each pays for its own
+ // flush however aggressively the queue is coalesced.
+ exportRecordsADrainIsWaitingFor();
+ // Re-take: the ends released above ask for their flush now, and one flush covers all of them since
+ // it starts after every one of those records reached the exporters.
+ synchronized (this) {
+ if (!flushRequests.isEmpty()) {
+ takenFlushes.addAll(flushRequests);
+ flushRequests.clear();
+ }
+ }
+ try {
+ flushEveryExporter();
+ } finally {
+ // In `finally`: a Throwable from a customer's flush() — an Error, not just an exception — must
+ // never leave the invocations waiting on these requests parked forever.
+ completeAll(takenFlushes);
+ takenFlushes = null;
+ }
}
- exportToAll(record);
}
+ } catch (Throwable failure) {
+ rejectFatal(failure);
+ throw failure;
} finally {
+ try {
+ // Before anything else, and before the handle below: whoever waits on these requests must be released
+ // even
+ // if this pump is unwinding for a reason none of the guards above anticipated.
+ if (takenFlushes != null) {
+ completeAll(takenFlushes);
+ }
+ CompletableFuture orphaned = null;
+ synchronized (this) {
+ if (inFlight == handle) {
+ inFlight = null;
+ }
+ if (taken != null) {
+ // Unwinding with a record still marked as being exported: this pump will never settle it.
+ // Release
+ // the marker and, unless a newer record for the same invocation is queued for a later pump to
+ // export, complete the drain waiting on it — the instance is right here, so no sweep over other
+ // invocations is needed to find it.
+ taken.exporting = false;
+ if (taken.record == null) {
+ orphaned = taken.settled;
+ taken.settled = null;
+ }
+ }
+ }
+ if (orphaned != null) {
+ completeSignal(orphaned);
+ }
+ completeSignal(handle);
+ // Last, because everything above is still this pump's work and a flush() re-entered from any of it
+ // would
+ // still have nobody to serve it. Conditional: a pump that recorded itself since must not be cleared
+ // here.
+ } finally {
+ pumpThread.compareAndSet(self, null);
+ }
+ }
+ }
+
+ /**
+ * Exports the queued records that a drain is waiting for, one at a time, and returns once they have all reached the
+ * exporters. Called by the pump immediately before a flush.
+ *
+ * Those records are the last records of invocations that cannot return until they are exported, and their ends
+ * cannot ask for their flush until then. Exporting them first is therefore what lets one flush serve a whole burst
+ * of invocation ends: without it the pump interleaves one record and one flush fan-out, and each end pays for a
+ * flush of its own even though every request is coalesced.
+ *
+ *
Bounded by the snapshot taken under the lock, so a producer that keeps scheduling for an invocation someone is
+ * draining cannot hold a flush back indefinitely — and records nobody waits for are not exported here at all, so a
+ * stream of {@code ON_CHANGE} snapshots still cannot starve a flush: it waits at most one ordinary fan-out plus
+ * this pass over the invocations whose return is already blocked on their own record.
+ */
+ private void exportRecordsADrainIsWaitingFor() {
+ List awaited = null;
+ synchronized (this) {
+ for (InsightPlugin execution : queue) {
+ if (execution.drainWaiters > 0) {
+ if (awaited == null) {
+ awaited = new ArrayList<>();
+ }
+ awaited.add(execution);
+ }
+ }
+ }
+ if (awaited == null) {
+ return;
+ }
+ for (InsightPlugin execution : awaited) {
+ WorkflowInsightRecord record;
synchronized (this) {
- if (inFlight == handle) {
- inFlight = null;
+ record = execution.record == null ? null : takeRecord(execution);
+ }
+ if (record == null) {
+ continue;
+ }
+ try {
+ exportToAll(record);
+ } finally {
+ try {
+ signalSettled(execution);
+ } catch (Throwable t) {
+ // Nothing here is expected to throw, but a record left marked as being exported would strand the
+ // drain that is waiting for it, so release it and that drain rather than leave the invocation
+ // parked.
+ abandon(execution);
+ reportFailure(t);
}
}
- handle.complete(null);
}
}
- private static String executionKey(WorkflowInsightRecord record) {
- return record.executionArn() != null ? record.executionArn() : "";
+ /** Completes every taken flush request; one that cannot be completed must not stop the rest from being. */
+ private void completeAll(List> requests) {
+ for (CompletableFuture request : requests) {
+ try {
+ completeSignal(request);
+ } catch (Throwable t) {
+ reportFailure(t);
+ }
+ }
+ }
+
+ /**
+ * Completes one invocation's drain signal now that its record has been exported, unless a newer record from the
+ * same invocation arrived meanwhile — that one settles the signal instead, so a drain always waits for the latest.
+ */
+ private void signalSettled(InsightPlugin execution) {
+ CompletableFuture signal;
+ synchronized (this) {
+ if (activeExecution == execution) activeExecution = null;
+ if (execution.record != null) {
+ // A newer record is queued for the same invocation. Leave it marked as being exported: it is still
+ // outstanding, and the export of that newer record settles the signal.
+ return;
+ }
+ execution.exporting = false;
+ signal = execution.settled;
+ execution.settled = null;
+ }
+ if (signal != null) {
+ completeSignal(signal);
+ }
+ }
+
+ // --- Flushing. ---
+
+ /**
+ * Flushes every exporter, serialized against exports: the request is queued and served by the pump between records,
+ * so an exporter never sees {@code flush()} overlap {@code export()} — not even an export belonging to a different
+ * execution running in the same environment. Returns once a flush that started after this request was enqueued has
+ * reached every exporter.
+ *
+ * Requests are coalesced: the pump takes every request queued at the start of its turn, exports any queued
+ * record a drain is still waiting for, re-takes the requests those ends make as they are released, and satisfies
+ * them all with one flush. Invocation ends that overlap therefore share a flush instead of paying for one fan-out
+ * each. That is sound because a caller drains its own record before asking, so a flush that starts after
+ * the request was enqueued has that record in the exporter's buffer. A request enqueued while a flush is already
+ * running is never satisfied by it — it waits for the next turn.
+ *
+ *
A queue that never runs dry cannot starve a request either: the pump alternates one record and one batch of
+ * requests, so a flush waits at most one export fan-out.
+ *
+ *
Called from the pump thread itself — or from an exporter fan-out worker that pump is waiting for, which is
+ * what a callback re-entering the scheduler does when two or more exporters are configured — the request is refused
+ * and reported instead of made: see {@link #refuseWaitThatWouldBlockThePump}.
+ */
+ void flush() {
+ throwIfFailed();
+ // Re-entered from a thread the pump's progress depends on: the pump is the only thread that could serve the
+ // request, and it cannot while this caller has not returned. Refuse rather than enqueue a request nobody
+ // serves.
+ if (refuseWaitThatWouldBlockThePump("flush()")) {
+ return;
+ }
+ CompletableFuture request = new CompletableFuture<>();
+ synchronized (this) {
+ throwIfFailed();
+ flushRequests.add(request);
+ }
+ while (true) {
+ CompletableFuture handle;
+ boolean startPump = false;
+ synchronized (this) {
+ throwIfFailed();
+ if (request.isDone()) {
+ return;
+ }
+ handle = inFlight;
+ if (handle == null) {
+ if (!flushRequests.contains(request)) {
+ // Liveness backstop: a pump took this request and unwound without serving it, which its
+ // `finally` is there to prevent. The request is no longer in the queue, so no future pump can
+ // find it — release the caller here instead of spinning up pumps that have nothing to do.
+ break;
+ }
+ // No pump is running (a worker could not be started earlier, or the pump went idle between the add
+ // above and this check): start one.
+ handle = new CompletableFuture<>();
+ inFlight = handle;
+ startPump = true;
+ }
+ }
+ if (startPump) {
+ CompletableFuture started = handle;
+ try {
+ executor.execute(() -> pump(started));
+ } catch (Throwable t) {
+ // No worker could be started. Serve the request on the calling thread, exactly as a drain exports a
+ // queued record inline: this pump owns `inFlight`, so no export can run beside it.
+ reportFailure(t);
+ pump(started);
+ continue;
+ }
+ }
+ // Wake either when this request has been served or when the current pump ends — a pump can end without
+ // serving it (a rejected worker), in which case the loop starts another one.
+ try {
+ CompletableFuture.anyOf(request, handle).join();
+ } catch (Throwable t) {
+ // Never spin on an unexpected wait failure, and never let it escape into the execution. Drop the
+ // request rather than leaving it queued for some later, unrelated invocation's pump to serve.
+ synchronized (this) {
+ flushRequests.remove(request);
+ }
+ reportFailure(t);
+ break;
+ }
+ }
+ completeSignal(request);
}
/**
* Flushes every exporter, each on its own worker, and waits for all of them to settle. A slow or failing flush on
- * one exporter never delays or fails the others.
+ * one exporter never delays or fails the others. Environment-wide, like the exporters themselves.
+ *
+ * Private and called only from the pump: routing every flush through the pump is what keeps a {@code flush()}
+ * from overlapping an {@code export()}, so this must not be reachable from outside. The per-exporter fan-out below
+ * is parallelism within one flush, not concurrency with an export.
*/
- void flushAll() {
+ private void flushEveryExporter() {
forEachExporterSettled(InsightExporter::flush);
}
+ // --- Exporting. ---
+
/**
* Exports one record to every exporter, each on its own worker, and waits for all of them to settle. One failing or
* slow exporter never blocks or fails the others, and an export error never propagates into the execution.
@@ -183,26 +841,52 @@ private void exportToAll(WorkflowInsightRecord record) {
/** Runs the action for every exporter concurrently and returns once all have settled, reporting each failure. */
private void forEachExporterSettled(Consumer action) {
if (exporters.size() == 1) {
+ // On the pump thread itself, which the pump-thread check already refuses waits from.
runSafely(() -> action.accept(exporters.get(0)));
return;
}
- List> settled = new ArrayList<>(exporters.size());
+ List> settledExporters = new ArrayList<>(exporters.size());
for (InsightExporter exporter : exporters) {
- Runnable task = () -> runSafely(() -> action.accept(exporter));
+ throwIfFailed();
+ // Marked as a fan-out task: the pump joins every one of these below, so a wait for the pump issued from
+ // inside one must be refused exactly as one issued from the pump itself.
+ Runnable task = () -> runSafely(() -> runAsExporterFanOut(() -> action.accept(exporter)));
try {
- settled.add(CompletableFuture.runAsync(task, executor));
+ settledExporters.add(CompletableFuture.runAsync(task, executor));
} catch (Throwable t) {
reportFailure(t);
task.run();
}
}
- for (CompletableFuture task : settled) {
- runSafely(task::join);
+ for (CompletableFuture task : settledExporters) {
+ // A fatal on another worker must reach the invocation even if this peer is blocked.
+ runSafely(() -> CompletableFuture.anyOf(task, fatalSignal).join());
+ }
+ }
+
+ /**
+ * Runs one exporter's share of a fan-out with this thread marked pump-dependent, restoring the previous mark on the
+ * way out. The mark is what makes {@link #refuseWaitThatWouldBlockThePump} recognize a fan-out worker.
+ */
+ private void runAsExporterFanOut(Runnable action) {
+ Boolean previous = exporterFanOutThread.get();
+ exporterFanOutThread.set(Boolean.TRUE);
+ try {
+ action.run();
+ } finally {
+ if (previous == null) {
+ // Removed rather than set back to null: these run on a shared, process-wide pool, so a thread must not
+ // keep an entry for this scheduler after its task ends.
+ exporterFanOutThread.remove();
+ } else {
+ exporterFanOutThread.set(previous);
+ }
}
}
private void runSafely(Runnable action) {
try {
+ throwIfFailed();
action.run();
} catch (Throwable t) {
reportFailure(t);
@@ -210,10 +894,102 @@ private void runSafely(Runnable action) {
}
private void reportFailure(Throwable t) {
+ rejectFatal(t);
try {
failureHandler.accept(t);
} catch (Throwable ignored) {
- // A scheduler diagnostic must never disrupt durable execution.
+ rejectFatal(ignored);
+ // Ordinary diagnostic failures remain isolated.
+ }
+ }
+
+ void throwIfFailed() {
+ Error failure = fatalFailure;
+ if (failure != null) throw failure;
+ }
+
+ private void rejectFatal(Throwable failure) {
+ Error fatal = FatalErrors.find(failure);
+ if (fatal != null) {
+ fail(fatal);
+ throw fatal;
+ }
+ }
+
+ private void completeSignal(CompletableFuture signal) {
+ Error failure = fatalFailure;
+ if (failure == null) signal.complete(null);
+ else signal.completeExceptionally(failure);
+ }
+
+ /** Releases all owners and waiters without waiting for uncooperative exporter code. */
+ synchronized void fail(Error failure) {
+ if (fatalFailure != null) return;
+ fatalFailure = failure;
+ CompletableFuture handle = inFlight;
+ inFlight = null;
+ fatalSignal.completeExceptionally(failure);
+ if (handle != null) handle.completeExceptionally(failure);
+ if (activeExecution != null) {
+ failOwner(activeExecution, failure);
+ activeExecution = null;
+ }
+ while (!queue.isEmpty()) failOwner(queue.iterator().next(), failure);
+ CompletableFuture request;
+ while ((request = flushRequests.poll()) != null) request.completeExceptionally(failure);
+ }
+
+ private void failOwner(InsightPlugin execution, Error failure) {
+ dropRecord(execution);
+ execution.exporting = false;
+ CompletableFuture signal = execution.settled;
+ execution.settled = null;
+ if (signal != null) signal.completeExceptionally(failure);
+ }
+
+ /**
+ * Reports and refuses a wait for the pump that was issued from a thread the pump's own progress depends on. Returns
+ * whether the caller is such a thread; when it is, the failure has already been reported and the caller must return
+ * without waiting.
+ *
+ * Invariant: the thread that waits for the pump is never a thread the pump waits for. {@link #flush()} waits for
+ * a request only a pump can complete, and a drain waits for a signal only a pump can complete or for the running
+ * pump's own handle. All three are satisfied by the pump between records.
+ *
+ *
Two threads qualify. The pump thread itself: a wait issued from there is a wait-for cycle one thread wide —
+ * the pump parks on the future it would itself have completed, so it never reaches the point in its loop that
+ * completes it, and no other thread may take over because {@code inFlight} is this pump's. And an exporter fan-out
+ * worker: with two or more exporters the pump submits one task per exporter and joins them all, so a wait issued
+ * from a callback running on one of those workers is the same cycle two threads wide — the worker parks on a future
+ * only the pump can complete, and the pump is parked in the join waiting for that worker. Neither is a monitor
+ * deadlock, so the JVM's deadlock detection cannot see either one, and the invocation simply never returns.
+ *
+ *
Reachable through anything a fan-out calls synchronously: with a single exporter the fan-out runs on the pump
+ * thread, so a customer exporter's {@code export()} or {@code flush()} that asks the scheduler for a flush, or a
+ * non-conforming {@code exportOne}, is enough; with several it runs on a worker instead, and the same call is
+ * refused for the same reason. A conforming production {@code exportOne} does not re-enter the scheduler, so this
+ * is hardening.
+ *
+ *
So the call fails fast instead: the plugin's failure handler is told — it logs — and the caller returns as it
+ * would from any other flush or drain, with nothing propagating into the execution. The queued work itself is not
+ * dropped by refusing a drain: the record stays in the invocation's slot, and the pump that is waiting for this
+ * caller exports it as soon as this caller returns and the fan-out it belongs to settles. Callers that are neither
+ * — every SDK hook thread — never enter this branch and behave exactly as before; the check is a volatile read plus
+ * a thread-local read, so no lock is added to that path.
+ */
+ private boolean refuseWaitThatWouldBlockThePump(String call) {
+ if (pumpThread.get() == Thread.currentThread()) {
+ reportFailure(new IllegalStateException(call
+ + " was called from the export pump thread, the only thread able to serve it; the call was refused"
+ + " rather than deadlocking the invocation"));
+ return true;
+ }
+ if (Boolean.TRUE.equals(exporterFanOutThread.get())) {
+ reportFailure(new IllegalStateException(call
+ + " was called from an exporter fan-out worker the export pump is waiting for, so the pump cannot"
+ + " serve it; the call was refused rather than deadlocking the invocation"));
+ return true;
}
+ return false;
}
}
diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightExporter.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightExporter.java
index 69ff39739..59f04767e 100644
--- a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightExporter.java
+++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightExporter.java
@@ -11,7 +11,34 @@ public interface InsightExporter {
/** Emits one record to the destination. */
void export(WorkflowInsightRecord record);
- /** Flushes any buffered records; no-op by default. */
+ /**
+ * Flushes any records this exporter has buffered. The default is a no-op; override it only if
+ * {@link #export(WorkflowInsightRecord)} buffers rather than emitting immediately.
+ *
+ *
Called at most once per sampled-in invocation end, after that invocation's own record — if it emitted one —
+ * has been handed to every exporter. An end that emits no record still flushes (a non-terminal suspend under
+ * {@code ON_COMPLETE}, a success under {@code ON_FAILURE}), so records buffered by that execution's earlier
+ * emissions are never left behind. Invocation ends that overlap may share a single flush: one flush is enough for
+ * all of them, because it starts only after each of their records has been handed to every exporter. An execution
+ * that is sampled out neither exports nor flushes.
+ *
+ *
Never called concurrently with {@link #export(WorkflowInsightRecord)} by the plugins one
+ * {@link WorkflowInsight#workflowInsight} factory creates. That factory owns the scheduler serializing them, so the
+ * guarantee is per factory rather than per environment: an exporter instance handed to two factories is served by
+ * two schedulers, which can call its {@code export} and {@code flush} at the same time. Build the factory once per
+ * handler — which is what a {@code DurableConfig} created once per handler does — and give each factory its own
+ * exporter instances if a single exporter cannot tolerate concurrent calls.
+ *
+ *
May cover records belonging to other executions running in the same environment, so it is not a per-execution
+ * barrier.
+ *
+ *
Must return promptly. No invocation whose end is waiting on this flush can return until it returns, and since
+ * overlapping ends may share one flush, a slow flush is billed to every one of those invocations — not only to the
+ * one that asked for it.
+ *
+ *
Non-fatal failures are reported and isolated without retries. Fatal VM/thread-termination errors propagate
+ * through the scheduler to the invocation caller; pending scheduler work is released.
+ */
default void flush() {}
/**
diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightPlugin.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightPlugin.java
new file mode 100644
index 000000000..922f675de
--- /dev/null
+++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightPlugin.java
@@ -0,0 +1,458 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.insight;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicLong;
+import software.amazon.lambda.durable.insight.internal.FatalErrors;
+import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.OperationChangeInfo;
+import software.amazon.lambda.durable.plugin.OperationChangeItemInfo;
+
+/**
+ * The Workflow Insight plugin instance for one Lambda invocation: both the state that invocation's records are built
+ * from and the slot the {@link ExportScheduler} exports them through.
+ *
+ *
The SDK creates one of these per invocation, from the {@link InvocationInfo} it is about to hand the first hook,
+ * and drops it when the invocation returns. So everything about an execution is a plain field here — the parsed ARN,
+ * the stable start time, the one-time sampling decision, the detached input snapshot, the latest queued record, the
+ * drain signal, the mid-export marker and the drain-waiter count. There is nothing to key by execution ARN and nothing
+ * to register or release: an instance is the registration, and its lifetime is the invocation's.
+ *
+ *
Two objects outlive the invocation and are shared by every instance the factory creates: the resolved
+ * {@link InsightSettings} and the {@link ExportScheduler}. The scheduler is shared on purpose — serializing exports is
+ * a property of the exporters, which belong to the environment, not to one invocation.
+ *
+ *
Ownership. Three groups of fields:
+ *
+ *
+ * - Identity — {@link #executionArn}, {@link #arn}, {@link #startTime}, {@link #sampledIn} — is taken from
+ * the {@link InvocationInfo} the factory receives and is {@code final}. It cannot be observed half-built, and
+ * there is no second invocation that could change it.
+ *
- The input snapshot — {@link #cachedInput} — is written by the thread that fires
+ * {@code onInvocationStart} and read by the operation-change and invocation-end threads of the same invocation,
+ * which the SDK does not promise are the same thread; {@code volatile} for that publication.
+ *
- The build revision — {@link #buildRevision} — counts the record builds this invocation has started, so
+ * that a build which was overtaken can be recognized at hand-off time and its record dropped. Atomic rather than
+ * {@code volatile}, because the case it exists for is two builds running at once. See the field.
+ *
- Scheduling state — {@link #record}, {@link #settled}, {@link #exporting}, {@link #drainWaiters} and
+ * {@link #closed} — is shared with the export pump and guarded by the monitor of {@link #scheduler}. One monitor
+ * for the whole environment, not one per invocation, so the {@code closed} check and the hand-off of a record are
+ * a single critical section and there is no lock ordering between instances to get wrong.
+ *
+ *
+ * {@link #closed} is additionally {@code volatile}: the hook threads read it without the lock as a fast pre-check.
+ * That read only ever skips work — the authoritative check is made under the monitor by
+ * {@link ExportScheduler#scheduleIfOpen}. It is written once, from false to true, and never back: an execution that
+ * suspends and resumes gets a new instance rather than a reset one.
+ */
+final class InsightPlugin implements DurableExecutionPlugin {
+
+ /** Resolved configuration, shared by every instance of the environment. */
+ private final InsightSettings settings;
+
+ /**
+ * Shared with every other instance: exports are serialized across the whole environment. Package-private because it
+ * is the monitor this instance's scheduling fields are guarded by, which the tests in this package hold when they
+ * read them.
+ */
+ final ExportScheduler scheduler;
+
+ // --- Identity, from the InvocationInfo the factory was called with. ---
+
+ /** The execution this instance observes. */
+ final String executionArn;
+
+ /** The parsed execution ARN, parsed once for every record this instance builds. */
+ final ArnParser arn;
+
+ /** Stable execution start time, from {@code InvocationInfo.executionStartTime()}. */
+ final Instant startTime;
+
+ /** The one-time sampling decision; deterministic in the ARN, so a resumed invocation decides the same way. */
+ final boolean sampledIn;
+
+ // --- The input snapshot. ---
+
+ /**
+ * Detached snapshot of the execution input, the single source of truth for {@code input} on every emission of this
+ * invocation. Written by {@code onInvocationStart}, read by every later build.
+ */
+ private volatile Object cachedInput;
+
+ // --- Build ordering. ---
+
+ /**
+ * Counts the record builds this invocation has started. The value a build takes identifies that build.
+ *
+ *
Customer code runs inside a build, on the hook thread: the input and output content transforms, an operation's
+ * result transform, and any Jackson serializer registered for a customer type. That code can call back into a hook
+ * of this same instance, and it runs before anything is scheduled, so a build can be overtaken by a newer build
+ * that starts and finishes inside it. Two hook threads for one invocation would produce the same overlap.
+ *
+ *
The scheduler's slot holds one record per invocation and takes whichever record is handed to it last, with no
+ * comparison of age. An overtaken build would therefore write its older snapshot over the newer one. Every build
+ * takes the next value here before it starts, and the scheduler queues the record only while that value is still
+ * the newest, so an overtaken build's record is dropped instead.
+ *
+ *
An {@link AtomicLong} rather than a {@code volatile long}: {@code ++} on a {@code volatile long} is a
+ * read-modify-write, so two concurrent builds can take the same value and each conclude its own record is the
+ * newest. That is the very case the check exists for, so a racy counter would guard nothing.
+ */
+ private final AtomicLong buildRevision = new AtomicLong();
+
+ // --- Scheduling state: guarded by the scheduler's monitor. ---
+
+ /**
+ * The latest record for this invocation that no pump has picked up yet, or {@code null} when none is queued.
+ *
+ *
A newer record replaces an older one here — each record is a complete snapshot, so the older one carries
+ * nothing the newer one lacks. That is the whole of coalescing: one slot, on the instance, which no other
+ * invocation can reach.
+ *
+ *
Which record is newer is decided by {@link #buildRevision}, not by the order the records reach this slot. The
+ * slot itself takes the last hand-off unconditionally, and the last hand-off is not the newest build when a build
+ * was overtaken by one that customer code started from inside it.
+ */
+ WorkflowInsightRecord record;
+
+ /**
+ * Completes once this invocation's latest record has been handed to every exporter; {@code null} when nothing is
+ * outstanding.
+ */
+ CompletableFuture settled;
+
+ /**
+ * Whether a pump has taken this invocation's record and is handing it to the exporters right now.
+ *
+ * Set and cleared in the same critical sections that move {@link #record}, so "no queued record" is never
+ * mistaken for "nothing outstanding" while the record is inside the exporters.
+ */
+ boolean exporting;
+
+ /**
+ * How many {@code drain} calls are waiting for this invocation right now.
+ *
+ *
A record with a waiter gates an invocation return, so the pump exports it before it spends a flush fan-out.
+ */
+ int drainWaiters;
+
+ /**
+ * Set once invocation end begins; never cleared. Guarded by the scheduler's monitor — the same monitor that queues
+ * the record, so the check and the hand-off are one critical section — and {@code volatile} for the hook-side
+ * pre-check.
+ *
+ *
A checkpoint that completes while the end record is being drained still delivers an operation-change hook to
+ * this same instance, and that RUNNING snapshot must not supersede the final record.
+ *
+ *
This orders RUNNING records against the final record; {@link #buildRevision} orders RUNNING records against
+ * each other. Neither covers the other's case. A boolean cannot say which of two RUNNING builds is newer, and the
+ * revision cannot reject a RUNNING record that follows the final one, because the final record is queued without a
+ * revision check. See {@link ExportScheduler#closeAndSchedule}.
+ */
+ volatile boolean closed;
+
+ /**
+ * Creates the instance that serves one invocation. Identity comes from {@code info} rather than from the first
+ * hook, so every field a record is keyed by exists before any hook can fire.
+ *
+ * @throws RuntimeException if the invocation has no usable execution ARN; the SDK contains that exactly as it
+ * contains a hook failure, by skipping this plugin for the invocation
+ */
+ InsightPlugin(InsightSettings settings, ExportScheduler scheduler, InvocationInfo info) {
+ this.settings = settings;
+ this.scheduler = scheduler;
+ this.executionArn = info.durableExecutionArn();
+ this.arn = ArnParser.parse(executionArn);
+ this.startTime = info.executionStartTime();
+ this.sampledIn = WorkflowInsight.shouldSample(executionArn, settings.samplingRate);
+ }
+
+ /** Test seam: waits until every scheduled record has been handed to the exporters. */
+ void drainExports() {
+ scheduler.drainAll();
+ }
+
+ @Override
+ public void onInvocationStart(InvocationInfo info) {
+ try {
+ scheduler.throwIfFailed();
+ if (!sampledIn) {
+ return;
+ }
+ // Detach the execution input from the live handler value immediately, before the user handler or any
+ // content transform can mutate it. This raw, detached snapshot is the single source of truth for input
+ // on every emission (start / change / end); each build hands transforms a separate defensive copy so a
+ // mutating transform cannot corrupt it. Guard the snapshot: a Throwable here (e.g. a payload whose
+ // serialization overflows the stack) must omit the captured input, never fail the user handler.
+ try {
+ cachedInput = Json.deepCopyContent(info.executionInput());
+ } catch (Throwable t) {
+ reportFailure("failed to snapshot execution input; omitting input", t);
+ cachedInput = null;
+ }
+ if (settings.emitMode == WorkflowInsightConfig.EmitMode.ON_CHANGE) {
+ // The revision is taken before the build, never after. Customer code runs inside buildRecord and can
+ // re-enter a hook of this instance, which builds a newer record; a revision read afterwards would
+ // already be that newer build's, and this older record would pass the check and overwrite it.
+ long revision = beginBuild();
+ scheduler.scheduleIfNotSuperseded(
+ this, buildRecord("RUNNING", info.operations(), null, cachedInput, null, null), revision);
+ }
+ } catch (Throwable t) {
+ reportFailure("onInvocationStart failed", t);
+ }
+ }
+
+ @Override
+ public void onOperationChange(OperationChangeInfo info) {
+ try {
+ scheduler.throwIfFailed();
+ if (settings.emitMode != WorkflowInsightConfig.EmitMode.ON_CHANGE || !sampledIn) {
+ return;
+ }
+ // Lock-free pre-check: this invocation's end may already have begun, in which case no RUNNING snapshot may
+ // follow the final record. The authoritative check is made again under the scheduler's lock below.
+ if (closed) {
+ return;
+ }
+ long revision = beginBuild();
+ scheduler.scheduleIfNotSuperseded(
+ this, buildRecord("RUNNING", info.operations(), null, cachedInput, null, null), revision);
+ } catch (Throwable t) {
+ reportFailure("onOperationChange failed", t);
+ }
+ }
+
+ // onInvocationEnd is the hook the SDK awaits, so it is where the export queue is drained before the invocation
+ // returns; this guarantees the final record (scheduled above the drain) is delivered. The drain and flush run
+ // in finally so they also cover the paths where record construction fails.
+ @Override
+ public void onInvocationEnd(InvocationEndInfo info) {
+ try {
+ scheduler.throwIfFailed();
+ String status = WorkflowInsight.mapStatus(info.invocationStatus());
+ boolean isTerminal = "SUCCEEDED".equals(status) || "FAILED".equals(status);
+ boolean isFailure = "FAILED".equals(status);
+ boolean shouldEmit;
+ switch (settings.emitMode) {
+ case ON_CHANGE:
+ shouldEmit = true;
+ break;
+ case ON_FAILURE:
+ shouldEmit = isFailure;
+ break;
+ case ON_COMPLETE:
+ default:
+ shouldEmit = isTerminal;
+ break;
+ }
+
+ WorkflowInsightRecord finalRecord = null;
+ if (sampledIn && shouldEmit) {
+ // No build revision is taken here. Customer code running inside this build can start a newer RUNNING
+ // build, which would make a revision taken here stale, and a checked hand-off would then drop the final
+ // record and leave a RUNNING snapshot as this execution's last exported state. The final record is
+ // instead ordered by `closed`, which closeAndSchedule sets in the same critical section that queues it.
+ finalRecord = buildRecord(
+ status,
+ info.operations(),
+ Instant.now(),
+ cachedInput,
+ info.executionResult(),
+ info.executionError());
+ }
+ // Close before the drain below: an operation-change hook arriving from a checkpoint that completes
+ // during the drain is rejected, so no RUNNING snapshot can follow (or replace) the final record.
+ scheduler.closeAndSchedule(this, finalRecord);
+ } catch (Throwable t) {
+ // Ordinary hook/export failures remain isolated. Fatal VM/thread termination
+ // marks the scheduler failed and escapes without further exporter work.
+ reportFailure("onInvocationEnd failed", t);
+ } finally {
+ // If record construction failed above, this instance is still open: close it so a late change hook cannot
+ // schedule into the drain. Idempotent when already closed.
+ scheduler.closeAndSchedule(this, null);
+ // Sampled-out invocations never schedule a record, so there is nothing to drain or flush. The drain needs
+ // no lookup: this instance is the thing whose record it waits for.
+ if (sampledIn) {
+ drainAndFlush();
+ }
+ // Nothing is released here. There is no per-execution entry to remove — this instance is the state, the SDK
+ // drops it when the invocation returns, and a suspended execution that resumes in the same container is
+ // served by a new instance built from the resume's own InvocationInfo (same stable start time, same
+ // deterministic sampling decision, its own input snapshot). A plugin failure therefore cannot turn into a
+ // state leak, because there is no place a leak could accumulate.
+ }
+ }
+
+ /**
+ * Waits for this invocation's scheduled record to reach the exporters, then flushes each exporter once. The wait is
+ * per invocation: another execution running in the same environment can never displace this record, so this always
+ * returns having delivered this invocation's latest snapshot. It is not insulated from the queue, though — one pump
+ * exports serially, so records another execution had already queued ahead of this one are exported first and this
+ * drain waits for them too.
+ *
+ *
The flush goes through the scheduler's queue and is served by that same pump, between records, so no exporter
+ * ever sees this invocation's {@code flush()} overlap another's {@code export()}. Invocation ends that overlap
+ * share one flush: the cadence the exporter contract promises is at most one flush per sampled-in invocation end,
+ * not exactly one.
+ */
+ private void drainAndFlush() {
+ try {
+ scheduler.drain(this);
+ } catch (Throwable t) {
+ reportFailure("failed to drain export scheduler", t);
+ }
+ try {
+ scheduler.flush();
+ } catch (Throwable t) {
+ reportFailure("exporter flush failed", t);
+ }
+ }
+
+ // --- Record building. ---
+
+ /** Marks a fatal hook/logging failure before the invocation's cleanup can schedule more work. */
+ private void reportFailure(String message, Throwable failure) {
+ try {
+ WorkflowInsight.logSafely(message, failure);
+ } catch (Throwable reportingFailure) {
+ Error fatal = FatalErrors.find(reportingFailure);
+ if (fatal != null) scheduler.fail(fatal);
+ throw reportingFailure;
+ }
+ }
+
+ /** Starts a record build and returns the revision that identifies it. */
+ private long beginBuild() {
+ return buildRevision.incrementAndGet();
+ }
+
+ /**
+ * Whether the identified build is still the newest one this invocation has started.
+ *
+ *
Read by the scheduler inside the critical section that queues the record, so a record that passes cannot be
+ * queued after a record that supersedes it. A build that starts after the check passes still supersedes this one:
+ * its record is handed over later and replaces this one in the slot, which is the order the slot should have.
+ */
+ boolean isNewestBuild(long revision) {
+ return buildRevision.get() == revision;
+ }
+
+ private WorkflowInsightRecord buildRecord(
+ String status,
+ Map operations,
+ Instant endTime,
+ Object input,
+ Object output,
+ Throwable error) {
+ ContentConfig content = settings.content;
+ WorkflowInsightRecord record = new WorkflowInsightRecord();
+ record.emittedAt = Instant.now().toString();
+ record.executionArn = executionArn;
+ record.executionName = WorkflowInsight.emptyToNull(arn.executionName());
+ record.functionName = arn.functionName();
+ record.functionQualifier = arn.qualifier();
+ record.region = arn.region();
+ record.accountId = arn.accountId();
+ record.status = status;
+ record.startTime = startTime != null ? startTime.toString() : null;
+ if (endTime != null) {
+ record.endTime = endTime.toString();
+ if (startTime != null) {
+ record.durationMs = endTime.toEpochMilli() - startTime.toEpochMilli();
+ }
+ }
+ record.input = WorkflowInsight.applyDataContent(
+ "input",
+ input,
+ content == null || content.includeInput(),
+ content == null ? null : content.inputTransform());
+ record.output = WorkflowInsight.applyDataContent(
+ "output",
+ output,
+ content == null || content.includeOutput(),
+ content == null ? null : content.outputTransform());
+ // Honor ContentConfig.includeErrors for the execution-level error exactly as for operation-level errors
+ // below: with includeErrors(false) no execution error is emitted, so a sensitive failure message never
+ // reaches a record. Without this gate the execution error leaked even when errors were disabled.
+ if (settings.includeErrors && error != null) {
+ record.error = WorkflowInsight.toErrorInfo(error);
+ }
+ record.operations = buildOperationRecords(operations);
+ return record;
+ }
+
+ private List buildOperationRecords(Map operations) {
+ List out = new ArrayList<>();
+ if (operations == null) {
+ return out;
+ }
+ // The hook contract supplies a map with no iteration-order guarantee (the core snapshot originates from a
+ // concurrent map). Sort by startTimestamp ascending (null timestamps last), then by a stable operation id
+ // tie-breaker, so the emitted operations array is deterministic and OperationsIndex's "latest occurrence"
+ // scalar fields reflect true chronological order rather than arbitrary map iteration order.
+ List items = new ArrayList<>(operations.values());
+ items.sort(Comparator.comparing(
+ OperationChangeItemInfo::startTimestamp, Comparator.nullsLast(Comparator.naturalOrder()))
+ .thenComparing(OperationChangeItemInfo::id, Comparator.nullsLast(Comparator.naturalOrder())));
+ for (OperationChangeItemInfo item : items) {
+ // The SDK core tracks the invocation/execution itself as a pseudo-entry of type EXECUTION; it is not a
+ // customer operation and the record already carries the execution status/timing at top level.
+ if ("EXECUTION".equals(item.type())) {
+ continue;
+ }
+ // Unnamed operations can't be targeted or keyed — excluded by default (matches JS `if (!op.name)`).
+ if (item.name() == null) {
+ continue;
+ }
+ // top-level detail drops anything nested under a context (parallel branches, map items, nested steps).
+ if (settings.topLevelOnly && item.parentId() != null) {
+ continue;
+ }
+ OperationOverride override = settings.overridesByName.get(item.name());
+ if (override != null && override.isExclude()) {
+ continue;
+ }
+ OperationRecord rec = new OperationRecord()
+ .id(item.id())
+ .name(item.name())
+ .type(item.type())
+ .subType(item.subType())
+ .parentId(item.parentId())
+ .status(item.status() != null ? item.status().toString() : "UNKNOWN")
+ .startTime(
+ item.startTimestamp() != null
+ ? item.startTimestamp().toString()
+ : null)
+ .endTime(item.endTimestamp() != null ? item.endTimestamp().toString() : null)
+ .attempt(item.attempt());
+ if (item.startTimestamp() != null && item.endTimestamp() != null) {
+ rec.durationMs(item.endTimestamp().toEpochMilli()
+ - item.startTimestamp().toEpochMilli());
+ }
+ if (settings.includeErrors && item.error() != null) {
+ rec.error(WorkflowInsight.toErrorInfo(item.error()));
+ }
+ // Results are omitted unless an override explicitly opts in via a transform (matches JS).
+ if (override != null && override.result() != null) {
+ rec.result(WorkflowInsight.applyResultOverride(override.result(), item.result()));
+ }
+ out.add(rec);
+ }
+ return out;
+ }
+
+ @Override
+ public String toString() {
+ return "InsightPlugin[" + executionArn + "]";
+ }
+}
diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightSettings.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightSettings.java
new file mode 100644
index 000000000..8729b8bec
--- /dev/null
+++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/InsightSettings.java
@@ -0,0 +1,59 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.insight;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import software.amazon.lambda.durable.insight.exporters.LambdaLogExporter;
+
+/**
+ * The plugin's configuration, resolved once and then immutable: everything a record's shape depends on that does not
+ * depend on which invocation is being observed.
+ *
+ * This belongs to the execution environment, not to an invocation. {@link WorkflowInsight#workflowInsight} resolves
+ * it once and the factory it returns hands the same instance to every {@link InsightPlugin} it creates, so resolving
+ * defaults, validating the sampling rate and indexing the operation overrides happen once per environment rather than
+ * once per invocation.
+ */
+final class InsightSettings {
+
+ /** Sampling rate, clamped to [0, 1]; the per-invocation decision is derived from it and the execution ARN. */
+ final double samplingRate;
+
+ final WorkflowInsightConfig.EmitMode emitMode;
+
+ /** True when nested operations (parallel branches, map items, nested steps) are dropped from the record. */
+ final boolean topLevelOnly;
+
+ final boolean includeErrors;
+
+ /** May be null, which means "every default": include input, output and errors, with no transforms. */
+ final ContentConfig content;
+
+ /** Operation overrides indexed by operation name, in declaration order. */
+ final Map overridesByName;
+
+ /** The configured exporters, or the default single {@link LambdaLogExporter} when none were configured. */
+ final List exporters;
+
+ InsightSettings(WorkflowInsightConfig config) {
+ this.samplingRate = WorkflowInsight.resolveSamplingRate(config.samplingRate());
+ this.emitMode = config.emitMode() != null ? config.emitMode() : WorkflowInsightConfig.EmitMode.ON_COMPLETE;
+ this.topLevelOnly = config.operationDetail() != WorkflowInsightConfig.OperationDetail.FULL_TREE;
+ this.content = config.content();
+ this.includeErrors = content == null || content.includeErrors();
+ Map overrides = new LinkedHashMap<>();
+ if (content != null) {
+ for (OperationOverride override : content.overrides()) {
+ overrides.put(override.operationName(), override);
+ }
+ }
+ // Unmodifiable wrapper rather than Map.copyOf: declaration order is preserved and an override with a null
+ // operation name is tolerated exactly as the mutable map tolerated it.
+ this.overridesByName = Collections.unmodifiableMap(overrides);
+ this.exporters =
+ config.exporters().isEmpty() ? List.of(new LambdaLogExporter()) : List.copyOf(config.exporters());
+ }
+}
diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/Json.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/Json.java
index dc72a7092..fd843df59 100644
--- a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/Json.java
+++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/Json.java
@@ -15,6 +15,7 @@
import java.util.List;
import java.util.Map;
import software.amazon.lambda.durable.annotations.Experimental;
+import software.amazon.lambda.durable.insight.internal.FatalErrors;
/** Minimal JSON helper for emitting insight records and measuring their serialized size. */
@Experimental
@@ -40,6 +41,7 @@ public static String stringify(Object value) {
try {
return MAPPER.writeValueAsString(value);
} catch (JsonProcessingException e) {
+ FatalErrors.rethrow(e);
throw new IllegalStateException("failed to serialize insight record", e);
}
}
@@ -51,6 +53,7 @@ public static String prettyStringify(Object value) {
try {
return MAPPER.writer(PRETTY_PRINTER).writeValueAsString(value);
} catch (JsonProcessingException e) {
+ FatalErrors.rethrow(e);
throw new IllegalStateException("failed to serialize insight record", e);
}
}
@@ -68,6 +71,7 @@ public static Integer byteSize(Object value) {
try {
return MAPPER.writeValueAsString(value).getBytes(StandardCharsets.UTF_8).length;
} catch (JsonProcessingException e) {
+ FatalErrors.rethrow(e);
return null;
}
}
@@ -107,6 +111,8 @@ static Object deepCopyContent(Object value) {
try {
return MAPPER.convertValue(value, Object.class);
} catch (IllegalArgumentException e) {
+ // convertValue transports Jackson serialization failures in IllegalArgumentException.
+ if (e.getCause() instanceof JsonProcessingException) FatalErrors.rethrow(e.getCause());
throw new IllegalStateException("failed to copy insight record content", e);
}
}
diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/WorkflowInsight.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/WorkflowInsight.java
index 531bbc475..ef50acaac 100644
--- a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/WorkflowInsight.java
+++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/WorkflowInsight.java
@@ -2,13 +2,7 @@
// SPDX-License-Identifier: Apache-2.0
package software.amazon.lambda.durable.insight;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import com.fasterxml.jackson.core.JsonProcessingException;
import java.util.function.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -16,12 +10,11 @@
import software.amazon.lambda.durable.annotations.Experimental;
import software.amazon.lambda.durable.exception.DurableOperationException;
import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException;
-import software.amazon.lambda.durable.insight.exporters.LambdaLogExporter;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
+import software.amazon.lambda.durable.insight.internal.FatalErrors;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
-import software.amazon.lambda.durable.plugin.OperationChangeInfo;
import software.amazon.lambda.durable.plugin.OperationChangeItemInfo;
/**
@@ -35,12 +28,18 @@
* {@link OperationChangeItemInfo#result()}; these are the fields PR #618 surfaced on the hook records, so
* {@code input}, {@code output}, and operation {@code result} are now populated exactly as in the JS plugin.
*
- * Per-execution state (keyed by execution ARN) holds only the stable start time, the parsed ARN, the one-time
- * sampling decision, and a detached snapshot of the execution input. State is removed on every {@code onInvocationEnd}
- * — including non-terminal PENDING/RETRYING suspends — so a suspended execution never leaks a retained entry for the
- * lifetime of a warm container. Nothing is lost across a resume: the next invocation recreates the same stable start
- * time from {@link InvocationInfo#executionStartTime()}, the same sampling decision deterministically from the ARN, and
- * the input snapshot from {@link InvocationInfo#executionInput()}.
+ *
{@link #workflowInsight} returns a {@link DurableExecutionPluginFactory}, so the SDK creates one
+ * {@link InsightPlugin} per Lambda invocation and drops it when the invocation returns. Everything about an execution —
+ * the stable start time, the parsed ARN, the one-time sampling decision, the detached input snapshot, the queued
+ * record, the drain signal — is therefore a plain field of that instance. Nothing is keyed by execution ARN, and there
+ * is no per-execution entry to remove at invocation end, so a suspended execution cannot leak one for the lifetime of a
+ * warm container. Nothing is lost across a resume either: the resume's own {@link InvocationInfo} carries the same
+ * stable start time, the sampling decision is deterministic in the ARN, and the input snapshot is taken again from
+ * {@link InvocationInfo#executionInput()}.
+ *
+ *
What belongs to the execution environment rather than to an invocation stays in the factory: the resolved
+ * {@link InsightSettings}, the exporters, and the {@link ExportScheduler} that serializes exports across every
+ * execution the environment hosts.
*/
@Experimental
public final class WorkflowInsight {
@@ -49,378 +48,51 @@ public final class WorkflowInsight {
private WorkflowInsight() {}
- /** Creates a Workflow Insight plugin from the given config. Mirrors the JS {@code workflowInsight(config)}. */
- public static DurableExecutionPlugin workflowInsight(WorkflowInsightConfig config) {
- return new InsightPlugin(config);
- }
-
- /** Per-execution state, keyed by execution ARN, to prevent warm-container bleed and handle resume. */
- private static final class ExecutionState {
- final Instant startTime;
- final ArnParser arn;
- final boolean sampledIn;
- volatile Object cachedInput;
-
- /**
- * Set once invocation end begins; guarded by {@code this}. A checkpoint that completes while the end record is
- * being drained still delivers an operation-change hook, and that RUNNING snapshot must not supersede the final
- * record.
- */
- boolean closed;
-
- ExecutionState(Instant startTime, ArnParser arn, boolean sampledIn) {
- this.startTime = startTime;
- this.arn = arn;
- this.sampledIn = sampledIn;
- }
-
- /** Schedules the record unless the invocation has already ended; the check and the hand-off are atomic. */
- boolean scheduleIfOpen(ExportScheduler scheduler, WorkflowInsightRecord record) {
- synchronized (this) {
- if (closed) {
- return false;
- }
- scheduler.schedule(record);
- return true;
- }
- }
-
- /** Marks the invocation ended and, when given a record, schedules it as the last one for this execution. */
- void closeAndSchedule(ExportScheduler scheduler, WorkflowInsightRecord finalRecord) {
- synchronized (this) {
- closed = true;
- if (finalRecord != null) {
- scheduler.schedule(finalRecord);
- }
- }
- }
+ /**
+ * Creates a Workflow Insight plugin factory from the given config. Mirrors the JS {@code workflowInsight(config)}.
+ *
+ *
The configuration is resolved once, here; the exporters and the scheduler that serializes exports across them
+ * are created once, here. The returned factory then builds one plugin instance per invocation, which is what lets
+ * that instance hold its execution's state in plain fields.
+ *
+ * @param config the plugin configuration
+ * @return a factory to hand to {@code DurableConfig.Builder.withPlugins}
+ */
+ public static DurableExecutionPluginFactory workflowInsight(WorkflowInsightConfig config) {
+ InsightSettings settings = new InsightSettings(config);
+ ExportScheduler scheduler = new ExportScheduler(
+ settings.exporters, WorkflowInsight::exportRecord, t -> logSafely("export scheduling failed", t));
+ return info -> new InsightPlugin(settings, scheduler, info);
}
- static final class InsightPlugin implements DurableExecutionPlugin {
- private final double samplingRate;
- private final WorkflowInsightConfig.EmitMode emitMode;
- private final boolean topLevelOnly;
- private final boolean includeErrors;
- private final ContentConfig content;
- private final Map overridesByName = new LinkedHashMap<>();
- private final List exporters;
- private final ExportScheduler scheduler;
-
- private final Map byArn = new ConcurrentHashMap<>();
-
- /** Test seam: number of live per-execution state entries retained across invocations. */
- int retainedStateCount() {
- return byArn.size();
- }
-
- /** Test seam: waits until every scheduled record has been handed to the exporters. */
- void drainExports() {
- scheduler.drain();
- }
-
- InsightPlugin(WorkflowInsightConfig config) {
- this.samplingRate = resolveSamplingRate(config.samplingRate());
- this.emitMode = config.emitMode() != null ? config.emitMode() : WorkflowInsightConfig.EmitMode.ON_COMPLETE;
- this.topLevelOnly = config.operationDetail() != WorkflowInsightConfig.OperationDetail.FULL_TREE;
- this.content = config.content();
- this.includeErrors = content == null || content.includeErrors();
- if (content != null) {
- for (OperationOverride o : content.overrides()) {
- overridesByName.put(o.operationName(), o);
- }
- }
- this.exporters =
- config.exporters().isEmpty() ? List.of(new LambdaLogExporter()) : List.copyOf(config.exporters());
- this.scheduler =
- new ExportScheduler(exporters, this::exportRecord, t -> logSafely("export scheduling failed", t));
- }
-
- private ExecutionState getState(String arn, Instant startTime) {
- return byArn.computeIfAbsent(
- arn, a -> new ExecutionState(startTime, ArnParser.parse(a), shouldSample(a, samplingRate)));
- }
-
- @Override
- public void onInvocationStart(InvocationInfo info) {
- try {
- ExecutionState state = getState(info.durableExecutionArn(), info.executionStartTime());
- if (!state.sampledIn) {
- return;
- }
- // Detach the execution input from the live handler value immediately, before the user handler or any
- // content transform can mutate it. This raw, detached snapshot is the single source of truth for input
- // on every emission (start / change / end); each build hands transforms a separate defensive copy so a
- // mutating transform cannot corrupt it. Guard the snapshot: a Throwable here (e.g. a payload whose
- // serialization overflows the stack) must omit the captured input, never fail the user handler.
- try {
- state.cachedInput = Json.deepCopyContent(info.executionInput());
- } catch (Throwable t) {
- logSafely("failed to snapshot execution input; omitting input", t);
- state.cachedInput = null;
- }
- if (emitMode == WorkflowInsightConfig.EmitMode.ON_CHANGE) {
- scheduler.schedule(buildRecord(
- state,
- info.durableExecutionArn(),
- "RUNNING",
- info.operations(),
- null,
- state.cachedInput,
- null,
- null));
- }
- } catch (Throwable t) {
- logSafely("onInvocationStart failed", t);
- }
- }
-
- @Override
- public void onOperationChange(OperationChangeInfo info) {
- try {
- if (emitMode != WorkflowInsightConfig.EmitMode.ON_CHANGE) {
- return;
- }
- ExecutionState state = byArn.get(info.durableExecutionArn());
- if (state == null || !state.sampledIn) {
- return;
- }
- state.scheduleIfOpen(
- scheduler,
- buildRecord(
- state,
- info.durableExecutionArn(),
- "RUNNING",
- info.operations(),
- null,
- state.cachedInput,
- null,
- null));
- } catch (Throwable t) {
- logSafely("onOperationChange failed", t);
- }
- }
-
- // onInvocationEnd is the hook the SDK awaits, so it is where the export queue is drained before the invocation
- // returns; this guarantees the final record (scheduled above the drain) is delivered. The drain and flush run
- // in finally so they also cover the paths where record construction fails.
- @Override
- public void onInvocationEnd(InvocationEndInfo info) {
- ExecutionState state = null;
- try {
- state = getState(info.durableExecutionArn(), info.executionStartTime());
- String status = mapStatus(info.invocationStatus());
- boolean isTerminal = "SUCCEEDED".equals(status) || "FAILED".equals(status);
- boolean isFailure = "FAILED".equals(status);
- boolean shouldEmit;
- switch (emitMode) {
- case ON_CHANGE:
- shouldEmit = true;
- break;
- case ON_FAILURE:
- shouldEmit = isFailure;
- break;
- case ON_COMPLETE:
- default:
- shouldEmit = isTerminal;
- break;
- }
-
- WorkflowInsightRecord finalRecord = null;
- if (state.sampledIn && shouldEmit) {
- finalRecord = buildRecord(
- state,
- info.durableExecutionArn(),
- status,
- info.operations(),
- Instant.now(),
- state.cachedInput,
- info.executionResult(),
- info.executionError());
- }
- // Close before the drain below: an operation-change hook arriving from a checkpoint that completes
- // during the drain is rejected, so no RUNNING snapshot can follow (or replace) the final record.
- state.closeAndSchedule(scheduler, finalRecord);
- } catch (Throwable t) {
- // A plugin failure at end-of-invocation (record construction, transforms, truncation, export/flush,
- // or optional exporter class linkage) must never disrupt durable execution.
- logSafely("onInvocationEnd failed", t);
- } finally {
- // If record construction failed above, the state is still open: close it so a late change hook cannot
- // schedule into the drain. Idempotent when already closed.
- if (state != null) {
- state.closeAndSchedule(scheduler, null);
- }
- // Sampled-out executions never schedule a record, so there is nothing to drain or flush. If the state
- // lookup itself failed, drain anyway: it is a no-op when idle and otherwise delivers what is pending.
- if (state == null || state.sampledIn) {
- drainAndFlush();
- }
- // Remove per-execution state on EVERY invocation end, including non-terminal PENDING/RETRYING suspends,
- // once any emission work above is done. Nothing durable is lost: the next invocation's onInvocation
- // start recreates the stable startTime from InvocationInfo.executionStartTime() (stable across
- // resumes),
- // the one-time sampling decision deterministically from the ARN, and the input snapshot from
- // InvocationInfo.executionInput(). Retaining state instead leaked one entry per suspended execution for
- // the lifetime of the warm container. This runs even if emission above threw, so a plugin failure can
- // never turn into a state leak.
- byArn.remove(info.durableExecutionArn());
- }
- }
-
- /** Waits for every scheduled record to reach the exporters, then flushes each exporter once, concurrently. */
- private void drainAndFlush() {
- try {
- scheduler.drain();
- } catch (Throwable t) {
- logSafely("failed to drain export scheduler", t);
- }
- try {
- scheduler.flushAll();
- } catch (Throwable t) {
- logSafely("exporter flush failed", t);
- }
- }
-
- /** Shapes and exports one record to one exporter; runs on a scheduler worker, never on an SDK hook thread. */
- private void exportRecord(WorkflowInsightRecord record, InsightExporter exporter) {
- try {
- // Give each exporter its own deep copy: truncation returns the original record when it already fits,
- // so without this a custom exporter that mutates operations or nested content would corrupt every
- // other exporter's view of the same record.
- WorkflowInsightRecord isolated = record.deepCopy();
- WorkflowInsightRecord shaped =
- Truncation.truncateRecord(isolated, exporter.maxRecordSizeBytes(), exporter::render);
- exporter.export(shaped);
- } catch (Throwable t) {
- // Catch Throwable, not just RuntimeException: deep copy, truncation, an exporter's render/export, or
- // the linkage of an optional exporter class (a NoClassDefFoundError when the S3 / CloudWatch SDK is
- // absent) can each fail with an Error. Isolating every Throwable here guarantees one failing exporter
- // cannot affect the others, nor disrupt the execution.
- logSafely("exporter failed", t);
- }
- }
-
- private WorkflowInsightRecord buildRecord(
- ExecutionState state,
- String arn,
- String status,
- Map operations,
- Instant endTime,
- Object input,
- Object output,
- Throwable error) {
- WorkflowInsightRecord record = new WorkflowInsightRecord();
- ArnParser a = state.arn;
- record.emittedAt = Instant.now().toString();
- record.executionArn = arn;
- record.executionName = emptyToNull(a.executionName());
- record.functionName = a.functionName();
- record.functionQualifier = a.qualifier();
- record.region = a.region();
- record.accountId = a.accountId();
- record.status = status;
- record.startTime = state.startTime != null ? state.startTime.toString() : null;
- if (endTime != null) {
- record.endTime = endTime.toString();
- if (state.startTime != null) {
- record.durationMs = endTime.toEpochMilli() - state.startTime.toEpochMilli();
- }
- }
- record.input = applyDataContent(
- "input",
- input,
- content == null || content.includeInput(),
- content == null ? null : content.inputTransform());
- record.output = applyDataContent(
- "output",
- output,
- content == null || content.includeOutput(),
- content == null ? null : content.outputTransform());
- // Honor ContentConfig.includeErrors for the execution-level error exactly as for operation-level errors
- // below: with includeErrors(false) no execution error is emitted, so a sensitive failure message never
- // reaches a record. Without this gate the execution error leaked even when errors were disabled.
- if (includeErrors && error != null) {
- record.error = toErrorInfo(error);
- }
- record.operations = buildOperationRecords(operations);
- return record;
- }
+ // --- helpers ---
- private List buildOperationRecords(Map operations) {
- List out = new ArrayList<>();
- if (operations == null) {
- return out;
- }
- // The hook contract supplies a map with no iteration-order guarantee (the core snapshot originates from a
- // concurrent map). Sort by startTimestamp ascending (null timestamps last), then by a stable operation id
- // tie-breaker, so the emitted operations array is deterministic and OperationsIndex's "latest occurrence"
- // scalar fields reflect true chronological order rather than arbitrary map iteration order.
- List items = new ArrayList<>(operations.values());
- items.sort(Comparator.comparing(
- OperationChangeItemInfo::startTimestamp, Comparator.nullsLast(Comparator.naturalOrder()))
- .thenComparing(OperationChangeItemInfo::id, Comparator.nullsLast(Comparator.naturalOrder())));
- for (OperationChangeItemInfo item : items) {
- // The SDK core tracks the invocation/execution itself as a pseudo-entry of type EXECUTION; it is not a
- // customer operation and the record already carries the execution status/timing at top level.
- if ("EXECUTION".equals(item.type())) {
- continue;
- }
- // Unnamed operations can't be targeted or keyed — excluded by default (matches JS `if (!op.name)`).
- if (item.name() == null) {
- continue;
- }
- // top-level detail drops anything nested under a context (parallel branches, map items, nested steps).
- if (topLevelOnly && item.parentId() != null) {
- continue;
- }
- OperationOverride override = overridesByName.get(item.name());
- if (override != null && override.isExclude()) {
- continue;
- }
- OperationRecord rec = new OperationRecord()
- .id(item.id())
- .name(item.name())
- .type(item.type())
- .subType(item.subType())
- .parentId(item.parentId())
- .status(item.status() != null ? item.status().toString() : "UNKNOWN")
- .startTime(
- item.startTimestamp() != null
- ? item.startTimestamp().toString()
- : null)
- .endTime(
- item.endTimestamp() != null
- ? item.endTimestamp().toString()
- : null)
- .attempt(item.attempt());
- if (item.startTimestamp() != null && item.endTimestamp() != null) {
- rec.durationMs(item.endTimestamp().toEpochMilli()
- - item.startTimestamp().toEpochMilli());
- }
- if (includeErrors && item.error() != null) {
- rec.error(toErrorInfo(item.error()));
- }
- // Results are omitted unless an override explicitly opts in via a transform (matches JS).
- if (override != null && override.result() != null) {
- rec.result(applyResultOverride(override.result(), item.result()));
- }
- out.add(rec);
- }
- return out;
+ /** Shapes and exports one record to one exporter; runs on a scheduler worker, never on an SDK hook thread. */
+ static void exportRecord(WorkflowInsightRecord record, InsightExporter exporter) {
+ try {
+ // Give each exporter its own deep copy: truncation returns the original record when it already fits,
+ // so without this a custom exporter that mutates operations or nested content would corrupt every
+ // other exporter's view of the same record.
+ WorkflowInsightRecord isolated = record.deepCopy();
+ WorkflowInsightRecord shaped =
+ Truncation.truncateRecord(isolated, exporter.maxRecordSizeBytes(), exporter::render);
+ exporter.export(shaped);
+ } catch (Throwable t) {
+ // Ordinary Errors (such as missing optional dependencies) are isolated too.
+ // logSafely rethrows VM/thread termination before logging or continuing.
+ logSafely("exporter failed", t);
}
}
- // --- helpers ---
-
/**
* Applies a user-supplied result transform to an operation's checkpointed (serialized JSON) result. Parses the JSON
* before handing it to the transform, so the transform always receives a detached, JSON-compatible value
* (a {@code Map} for a former POJO, a {@code List} for an array, or a scalar such as a {@code String} for a Java
* time value) — never the SDK's original Java object. The raw string is passed through only when the checkpointed
- * result is not valid JSON. User transforms are untrusted: a throwing transform (any {@link Throwable}) omits the
- * field rather than leaking the raw value or failing the execution, and the failure is logged for diagnosis.
- * Because the value is freshly parsed from the immutable checkpoint string on every build, a transform that mutates
- * its argument cannot corrupt any cached state or a later emission.
+ * result is not valid JSON. Non-fatal transform failures omit the field rather than leaking the raw value and are
+ * logged for diagnosis. VM/thread-termination errors propagate. Because the value is freshly parsed from the
+ * immutable checkpoint string on every build, a transform that mutates its argument cannot corrupt any cached state
+ * or a later emission.
*/
static Object applyResultOverride(Function transform, String rawResult) {
if (rawResult == null) {
@@ -429,7 +101,8 @@ static Object applyResultOverride(Function transform, String raw
Object parsed;
try {
parsed = Json.MAPPER.readValue(rawResult, Object.class);
- } catch (RuntimeException | com.fasterxml.jackson.core.JsonProcessingException e) {
+ } catch (RuntimeException | JsonProcessingException e) {
+ FatalErrors.rethrow(e);
parsed = rawResult;
}
try {
@@ -457,8 +130,8 @@ static Object applyDataContent(String label, Object value, boolean include, Func
// reused across multiple emissions (ON_CHANGE), so a transform that mutates its argument in place must
// not corrupt that snapshot or any later emission's view of it. deepCopyContent also normalizes POJOs
// to Maps and Java-time types to their JSON representation, so the transform operates on the same
- // JSON-compatible shape the record will emit. Omit and log on any Throwable so a failing redactor never
- // leaks the raw value and never disrupts the execution.
+ // JSON-compatible shape the record will emit. Ordinary failures omit the raw value;
+ // fatal VM/thread termination is rethrown by logSafely.
return transform.apply(Json.deepCopyContent(value));
} catch (Throwable e) {
logSafely(label + " transform failed; value omitted", e);
@@ -468,16 +141,18 @@ static Object applyDataContent(String label, Object value, boolean include, Func
return value;
}
- /** Logs a plugin failure without ever letting the logging itself disrupt durable execution. */
- private static void logSafely(String message, Throwable t) {
+ /** Rethrows fatal failures; contains and logs ordinary plugin/logging failures. */
+ static void logSafely(String message, Throwable t) {
+ FatalErrors.rethrow(t);
try {
logger.warn("[workflow-insight] {}", message, t);
} catch (Throwable ignored) {
- // Never allow a logging failure to propagate into the SDK control flow.
+ FatalErrors.rethrow(ignored);
+ // Ordinary logging failures stay isolated from SDK control flow.
}
}
- private static ErrorInfo toErrorInfo(Throwable t) {
+ static ErrorInfo toErrorInfo(Throwable t) {
// Operation and execution snapshot errors are exposed wrapped: operation failures as DurableOperationException
// and unrecoverable execution failures as UnrecoverableDurableExecutionException. The wrapper's own
// class/message would lose the original checkpointed failure identity. When the checkpointed ErrorObject is
@@ -509,11 +184,11 @@ private static ErrorObject extractErrorObject(Throwable t) {
return null;
}
- private static String emptyToNull(String s) {
+ static String emptyToNull(String s) {
return s == null || s.isEmpty() ? null : s;
}
- private static String mapStatus(InvocationStatus status) {
+ static String mapStatus(InvocationStatus status) {
if (status == InvocationStatus.SUCCEEDED) {
return "SUCCEEDED";
}
diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/exporters/LazyClient.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/exporters/LazyClient.java
index bfd427288..7f6ad7d0c 100644
--- a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/exporters/LazyClient.java
+++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/exporters/LazyClient.java
@@ -7,6 +7,7 @@
import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.utils.builder.SdkBuilder;
+import software.amazon.lambda.durable.insight.internal.FatalErrors;
/**
* Holds an exporter's transport client: an injected instance, or one created on first use from an optional AWS SDK
@@ -66,6 +67,7 @@ private static T buildSdkClient(String clientClassName, String region) {
} catch (ClassNotFoundException e) {
throw new MissingArtifactException(e);
} catch (ReflectiveOperationException e) {
+ FatalErrors.rethrow(e);
Throwable cause = e instanceof InvocationTargetException ? e.getCause() : e;
throw new IllegalStateException("Failed to create " + clientClassName, cause);
}
diff --git a/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/internal/FatalErrors.java b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/internal/FatalErrors.java
new file mode 100644
index 000000000..b9efab61f
--- /dev/null
+++ b/insight-plugin/src/main/java/software/amazon/lambda/durable/insight/internal/FatalErrors.java
@@ -0,0 +1,51 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.insight.internal;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.Set;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
+
+/** Internal fatal-failure policy for Insight hooks, serializers and transport adapters. */
+public final class FatalErrors {
+ private FatalErrors() {}
+
+ /** Finds only direct fatal errors and known transport wrappers, not arbitrary business causes. */
+ @SuppressWarnings("removal") // ThreadDeath remains supported on the SDK's Java 17 baseline.
+ public static Error find(Throwable failure) {
+ Set seen = null;
+ while (failure != null) {
+ if (failure instanceof VirtualMachineError fatal) return fatal;
+ if (failure instanceof ThreadDeath fatal) return fatal;
+ if (!isTransport(failure)) return null;
+ if (seen == null) seen = Collections.newSetFromMap(new IdentityHashMap<>());
+ if (!seen.add(failure)) return null;
+ try {
+ failure = failure.getCause();
+ } catch (VirtualMachineError | ThreadDeath fatal) {
+ return fatal;
+ } catch (Throwable ignored) {
+ return null; // A broken non-fatal cause accessor must not break fail-open handling.
+ }
+ }
+ return null;
+ }
+
+ private static boolean isTransport(Throwable failure) {
+ return failure instanceof CompletionException
+ || failure instanceof ExecutionException
+ || failure instanceof InvocationTargetException
+ || failure instanceof UndeclaredThrowableException
+ || failure instanceof JsonProcessingException;
+ }
+
+ public static void rethrow(Throwable failure) {
+ Error fatal = find(failure);
+ if (fatal != null) throw fatal;
+ }
+}
diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ConcurrentExecutionsExportTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ConcurrentExecutionsExportTest.java
new file mode 100644
index 000000000..538e39240
--- /dev/null
+++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ConcurrentExecutionsExportTest.java
@@ -0,0 +1,409 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.insight;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Instant;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.services.lambda.model.OperationStatus;
+import software.amazon.lambda.durable.plugin.InvocationEndInfo;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+import software.amazon.lambda.durable.plugin.InvocationStatus;
+import software.amazon.lambda.durable.plugin.OperationChangeInfo;
+import software.amazon.lambda.durable.plugin.OperationChangeItemInfo;
+
+/**
+ * One {@link ExportScheduler} — created once by {@code workflowInsight()} and shared by every plugin instance the
+ * factory makes — serves a whole execution environment, and an environment can host several durable executions at once
+ * (routine under Lambda Managed Instances). These tests pin the per-execution guarantees that concurrency demands: one
+ * execution's record never displaces another's, and each execution's drain returns only after its own record reached
+ * the exporters.
+ */
+class ConcurrentExecutionsExportTest {
+
+ private static final Instant START = Instant.parse("2026-08-05T00:00:00Z");
+
+ private static String arn(int index) {
+ return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1";
+ }
+
+ private static class CapturingExporter implements InsightExporter {
+ final List records = new CopyOnWriteArrayList<>();
+
+ @Override
+ public void export(WorkflowInsightRecord record) {
+ records.add(record);
+ }
+
+ /** Identity, not equality: these tests track the exact record instance an execution scheduled. */
+ boolean exported(WorkflowInsightRecord record) {
+ for (WorkflowInsightRecord seen : records) {
+ if (seen == record) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
+ private static WorkflowInsightRecord record(String executionArn, String status) {
+ var r = new WorkflowInsightRecord();
+ r.executionArn = executionArn;
+ r.status = status;
+ return r;
+ }
+
+ private static ExportScheduler scheduler(List failures, InsightExporter... exporters) {
+ return new ExportScheduler(List.of(exporters), (rec, exp) -> exp.export(rec), failures::add, workers());
+ }
+
+ private static Executor workers() {
+ return command -> new Thread(command, "test-export-worker").start();
+ }
+
+ /**
+ * Parks the pump task and then reports rejection, leaving the scheduler idle with the record still queued. Nothing
+ * about the scheduler is faked: the parked task is its own {@code () -> pump(handle)} lambda, run later verbatim.
+ */
+ private static final class ParkingExecutor implements Executor {
+ final Deque parked = new ArrayDeque<>();
+
+ @Override
+ public void execute(Runnable command) {
+ parked.add(command);
+ throw new RejectedExecutionException("test: parked, reported as rejected");
+ }
+ }
+
+ /** Terminal records, by execution ARN, in the order the exporter received them. */
+ private static List terminalArns(CapturingExporter exporter) {
+ List out = new ArrayList<>();
+ for (WorkflowInsightRecord r : exporter.records) {
+ if ("SUCCEEDED".equals(r.status())) {
+ out.add(r.executionArn());
+ }
+ }
+ return out;
+ }
+
+ @Test
+ void everyConcurrentExecutionDeliversItsTerminalRecordExactlyOnce() throws Exception {
+ int executions = 10;
+ int changesEach = 3;
+ var failures = new CopyOnWriteArrayList();
+ var exporter = new CapturingExporter();
+ var scheduler = scheduler(failures, exporter);
+
+ var barrier = new CyclicBarrier(executions);
+ // Executions whose drain returned before their own terminal record had reached the exporter. Each thread checks
+ // its own postcondition the instant its drain returns; inspecting the exporter only after joining every thread
+ // would also pass if a drain returned early and the export landed a moment later.
+ var returnedBeforeExport = Collections.synchronizedList(new ArrayList());
+ var threads = new ArrayList();
+ for (int i = 0; i < executions; i++) {
+ String executionArn = arn(i);
+ // One plugin instance per execution, as the SDK creates one per invocation; the thread below holds it
+ // exactly as an invocation's hooks do.
+ InsightPlugin execution = Executions.plugin(scheduler, executionArn);
+ var thread = new Thread(
+ () -> {
+ awaitBarrier(barrier);
+ for (int c = 0; c < changesEach; c++) {
+ scheduler.schedule(execution, record(executionArn, "RUNNING"));
+ }
+ WorkflowInsightRecord terminal = record(executionArn, "SUCCEEDED");
+ scheduler.schedule(execution, terminal);
+ scheduler.drain(execution);
+ // This thread is the only one scheduling for this ARN, so no later record can supersede the
+ // terminal one: once drain returns, it must already have reached the exporter.
+ if (!exporter.exported(terminal)) {
+ returnedBeforeExport.add(executionArn);
+ }
+ },
+ "execution-" + i);
+ threads.add(thread);
+ thread.start();
+ }
+ for (Thread thread : threads) {
+ thread.join(30_000);
+ assertFalse(thread.isAlive(), "every execution's drain returned");
+ }
+
+ assertEquals(
+ List.of(),
+ returnedBeforeExport,
+ "drain returned before this execution's own terminal record reached the exporter");
+ List delivered = terminalArns(exporter);
+ Set expected = new HashSet<>();
+ for (int i = 0; i < executions; i++) {
+ expected.add(arn(i));
+ }
+ assertEquals(expected, new HashSet<>(delivered), "no execution lost its terminal record");
+ assertEquals(executions, delivered.size(), "and none was exported twice");
+ assertTrue(failures.isEmpty(), "no scheduler failure was reported: " + failures);
+ }
+
+ /**
+ * Regression: a pump that is exiting must not complete the drain signal of an execution whose record it does not
+ * own. Such a record has already left the queue — it is inside the exporters — so "no record queued for this ARN"
+ * is not enough to call the signal orphaned. If it were, the exiting pump would release {@code drain(arn)} mid
+ * export and the invocation could return before its final record was delivered.
+ *
+ * The state is built through the {@link Executor} seam rather than by racing threads: the executor parks the
+ * pump task and reports rejection, which is the same shape the scheduler produces on its own in the window between
+ * the pump loop's return to idle and its {@code finally} — a live pump whose handle is no longer the installed one.
+ */
+ @Test
+ void anExitingPumpDoesNotReleaseADrainWhoseRecordIsStillInsideTheExporter() throws Exception {
+ var executor = new ParkingExecutor();
+ var exporting = new CountDownLatch(1);
+ var release = new CountDownLatch(1);
+ Set exported = ConcurrentHashMap.newKeySet();
+ var scheduler = new ExportScheduler(
+ List.of(record -> {}),
+ (rec, exp) -> {
+ exporting.countDown();
+ await(release, 10);
+ exported.add(rec);
+ },
+ new CopyOnWriteArrayList()::add,
+ executor);
+
+ String executionArn = arn(1);
+ InsightPlugin execution = Executions.plugin(scheduler, executionArn);
+ WorkflowInsightRecord terminal = record(executionArn, "SUCCEEDED");
+ scheduler.schedule(execution, terminal);
+ assertEquals(1, executor.parked.size(), "the pump task was parked, so the record is still queued");
+
+ // A drainer picks the record up on the inline path and is now inside the exporter.
+ var inlineDrained = new CountDownLatch(1);
+ var inlineDrainer = new Thread(
+ () -> {
+ scheduler.drain(execution);
+ inlineDrained.countDown();
+ },
+ "inline-drainer");
+ inlineDrainer.setDaemon(true);
+ inlineDrainer.start();
+ assertTrue(exporting.await(5, TimeUnit.SECONDS), "the terminal record is inside the exporter");
+
+ // Now let the parked pump run to completion. It finds nothing queued and exits; its cleanup must leave the
+ // record that is mid-export alone.
+ executor.parked.poll().run();
+
+ var secondDrained = new CountDownLatch(1);
+ var secondDrainer = new Thread(
+ () -> {
+ scheduler.drain(execution);
+ secondDrained.countDown();
+ },
+ "second-drainer");
+ secondDrainer.setDaemon(true);
+ secondDrainer.start();
+ assertFalse(
+ secondDrained.await(500, TimeUnit.MILLISECONDS),
+ "drain returned while the execution's record was still inside the exporter");
+ assertTrue(exported.isEmpty(), "the exporter has not finished with the record yet");
+
+ release.countDown();
+ assertTrue(secondDrained.await(5, TimeUnit.SECONDS), "the drain returns once the export completes");
+ assertTrue(inlineDrained.await(5, TimeUnit.SECONDS), "so does the drain that ran the export");
+ assertEquals(Set.of(terminal), exported, "the terminal record was exported exactly once");
+ }
+
+ @Test
+ void aRecordForAnotherExecutionNeverDisplacesAPendingTerminalRecord() throws Exception {
+ String slowExecution = arn(1);
+ String otherExecution = arn(2);
+ var exporting = new CountDownLatch(1);
+ var release = new CountDownLatch(1);
+ var exporter = new CapturingExporter() {
+ @Override
+ public void export(WorkflowInsightRecord record) {
+ super.export(record);
+ if (records.size() == 1) {
+ exporting.countDown();
+ await(release);
+ }
+ }
+ };
+ var scheduler = scheduler(new CopyOnWriteArrayList<>(), exporter);
+ InsightPlugin slow = Executions.plugin(scheduler, slowExecution);
+ InsightPlugin other = Executions.plugin(scheduler, otherExecution);
+
+ // One execution's export is in flight and blocked...
+ scheduler.schedule(slow, record(slowExecution, "RUNNING"));
+ assertTrue(exporting.await(5, TimeUnit.SECONDS), "the first export is in flight");
+ // ...while a second execution's terminal record is queued, followed by an update for the first execution.
+ // The first execution's own update must coalesce only with its own slot, never over the second execution's.
+ scheduler.schedule(other, record(otherExecution, "SUCCEEDED"));
+ scheduler.schedule(slow, record(slowExecution, "SUCCEEDED"));
+
+ var seenBySlowDrain = Collections.synchronizedList(new ArrayList());
+ var seenByOtherDrain = Collections.synchronizedList(new ArrayList());
+ var slowDrained = new CountDownLatch(1);
+ var otherDrained = new CountDownLatch(1);
+ var slowDrainer = new Thread(
+ () -> {
+ scheduler.drain(slow);
+ seenBySlowDrain.addAll(terminalArns(exporter));
+ slowDrained.countDown();
+ },
+ "slow-drainer");
+ var otherDrainer = new Thread(
+ () -> {
+ scheduler.drain(other);
+ seenByOtherDrain.addAll(terminalArns(exporter));
+ otherDrained.countDown();
+ },
+ "other-drainer");
+ slowDrainer.start();
+ otherDrainer.start();
+
+ assertFalse(slowDrained.await(200, TimeUnit.MILLISECONDS), "a drain cannot return while its record is pending");
+ assertFalse(otherDrained.await(50, TimeUnit.MILLISECONDS), "nor can the other execution's drain");
+
+ release.countDown();
+ assertTrue(slowDrained.await(5, TimeUnit.SECONDS), "the blocked execution's drain completes");
+ assertTrue(otherDrained.await(5, TimeUnit.SECONDS), "the other execution's drain completes");
+
+ assertTrue(
+ seenByOtherDrain.contains(otherExecution),
+ "drain returned only after this execution's own terminal record was exported");
+ assertTrue(
+ seenBySlowDrain.contains(slowExecution),
+ "drain returned only after this execution's own terminal record was exported");
+ List delivered = terminalArns(exporter);
+ assertEquals(
+ Set.of(slowExecution, otherExecution),
+ new HashSet<>(delivered),
+ "neither execution's terminal record was lost");
+ assertEquals(2, delivered.size(), "and neither was exported twice");
+ }
+
+ @Test
+ void concurrentExecutionsDrivenThroughThePluginHooksAllDeliverTheirTerminalRecord() throws Exception {
+ int executions = 5;
+ var exporter = new CapturingExporter();
+ // One factory — one environment, one scheduler, one set of exporters — and one plugin instance per invocation,
+ // which is how the SDK drives several concurrent executions through the same exporters.
+ var factory = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder()
+ .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE)
+ .addExporter(exporter)
+ .build());
+
+ var barrier = new CyclicBarrier(executions);
+ var plugins = new ArrayList();
+ var threads = new ArrayList();
+ for (int i = 0; i < executions; i++) {
+ String executionArn = arn(i);
+ InvocationInfo startInfo = start(executionArn);
+ var plugin = Executions.plugin(factory, startInfo);
+ plugins.add(plugin);
+ var thread = new Thread(
+ () -> {
+ awaitBarrier(barrier);
+ plugin.onInvocationStart(startInfo);
+ for (int c = 0; c < 3; c++) {
+ plugin.onOperationChange(new OperationChangeInfo(
+ "req",
+ executionArn,
+ ops(OperationStatus.SUCCEEDED),
+ ops(OperationStatus.SUCCEEDED)));
+ }
+ plugin.onInvocationEnd(end(executionArn, InvocationStatus.SUCCEEDED));
+ },
+ "execution-" + i);
+ threads.add(thread);
+ thread.start();
+ }
+ for (Thread thread : threads) {
+ thread.join(30_000);
+ assertFalse(thread.isAlive(), "every invocation-end hook returned");
+ }
+
+ List delivered = terminalArns(exporter);
+ Set expected = new HashSet<>();
+ for (int i = 0; i < executions; i++) {
+ expected.add(arn(i));
+ }
+ assertEquals(expected, new HashSet<>(delivered), "every execution's terminal record arrived");
+ assertEquals(executions, delivered.size(), "and none arrived twice");
+ for (InsightPlugin plugin : plugins) {
+ assertFalse(
+ Executions.outstanding(plugin),
+ "the scheduler still owes this execution work after its invocation end: " + plugin);
+ }
+ }
+
+ private static Map ops(OperationStatus status) {
+ Map operations = new LinkedHashMap<>();
+ operations.put(
+ "op-1",
+ new OperationChangeItemInfo(
+ "op-1",
+ "greet",
+ "STEP",
+ "Step",
+ null,
+ START,
+ START.plusMillis(5),
+ status,
+ 1,
+ false,
+ null,
+ null));
+ return operations;
+ }
+
+ private static InvocationInfo start(String executionArn) {
+ return new InvocationInfo("req", executionArn, true, START, "in", ops(OperationStatus.STARTED), Map.of());
+ }
+
+ private static InvocationEndInfo end(String executionArn, InvocationStatus status) {
+ return new InvocationEndInfo(
+ "req", executionArn, true, START, ops(OperationStatus.SUCCEEDED), status, null, "in", "out");
+ }
+
+ private static void awaitBarrier(CyclicBarrier barrier) {
+ try {
+ barrier.await(30, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ private static void await(CountDownLatch latch) {
+ await(latch, 5);
+ }
+
+ private static void await(CountDownLatch latch, long timeoutSeconds) {
+ try {
+ if (!latch.await(timeoutSeconds, TimeUnit.SECONDS)) {
+ throw new AssertionError("latch not released");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(e);
+ }
+ }
+}
diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ErrorPrivacyGateTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ErrorPrivacyGateTest.java
index 0cd5ede12..2e0a47af9 100644
--- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ErrorPrivacyGateTest.java
+++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ErrorPrivacyGateTest.java
@@ -14,7 +14,6 @@
import java.util.Map;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.services.lambda.model.OperationStatus;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
@@ -64,10 +63,15 @@ private Map failingOp() {
}
private WorkflowInsightRecord runFailedExecution(boolean includeErrors, CapturingExporter exporter) {
- DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder()
- .content(ContentConfig.builder().includeErrors(includeErrors).build())
- .addExporter(exporter)
- .build());
+ var plugin = Executions.plugin(
+ WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder()
+ .content(ContentConfig.builder()
+ .includeErrors(includeErrors)
+ .build())
+ .addExporter(exporter)
+ .build()),
+ ARN,
+ START);
plugin.onInvocationStart(new InvocationInfo("req", ARN, true, START, "in", failingOp(), Map.of()));
plugin.onInvocationEnd(new InvocationEndInfo(
"req",
diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/Executions.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/Executions.java
new file mode 100644
index 000000000..51d30e414
--- /dev/null
+++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/Executions.java
@@ -0,0 +1,77 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.insight;
+
+import java.time.Instant;
+import java.util.Map;
+import software.amazon.lambda.durable.plugin.DurableExecutionPluginFactory;
+import software.amazon.lambda.durable.plugin.InvocationInfo;
+
+/**
+ * Test helper: builds the per-invocation plugin instances the {@link ExportScheduler} schedules for.
+ *
+ * The scheduler no longer resolves an execution ARN to anything — an invocation's state is its plugin
+ * instance, created from the {@link InvocationInfo} the SDK is about to hand the first hook — so tests hold the
+ * instance exactly as the SDK does. Everything here goes through the same constructor and the same factory production
+ * uses; nothing is a test-only back door into the scheduler.
+ */
+final class Executions {
+
+ private static final Instant START = Instant.parse("2026-08-05T00:00:00Z");
+
+ private Executions() {}
+
+ /** The invocation description the SDK would build for one execution, with no payload or operation snapshot. */
+ static InvocationInfo info(String executionArn) {
+ return new InvocationInfo("req", executionArn, true, START, null, Map.of(), Map.of());
+ }
+
+ /**
+ * One invocation's plugin instance, bound to this scheduler and configured with the plugin's defaults. For tests
+ * that drive the scheduler directly and do not care how records are shaped.
+ */
+ static InsightPlugin plugin(ExportScheduler scheduler, String executionArn) {
+ return new InsightPlugin(
+ new InsightSettings(WorkflowInsightConfig.builder().build()), scheduler, info(executionArn));
+ }
+
+ /**
+ * One invocation's plugin instance from the factory, for the identity the SDK would have built it with. The
+ * invocation's own {@code InvocationInfo} still goes to {@code onInvocationStart}; this is the same pair of facts
+ * that info carries, which is all an instance's identity is.
+ */
+ static InsightPlugin plugin(DurableExecutionPluginFactory factory, String executionArn, Instant startTime) {
+ return plugin(factory, new InvocationInfo("req", executionArn, true, startTime, null, Map.of(), Map.of()));
+ }
+
+ /**
+ * One invocation's plugin instance, exactly as the SDK creates it: from the factory, with that invocation's info.
+ */
+ static InsightPlugin plugin(DurableExecutionPluginFactory factory, InvocationInfo info) {
+ return (InsightPlugin) factory.createPlugin(info);
+ }
+
+ /**
+ * Whether the scheduler still owes this invocation anything: a queued record, a record inside the exporters, an
+ * uncompleted drain signal, or a drain waiting on it. Read under the monitor those fields are guarded by.
+ *
+ *
This is the question the plugin's {@code retainedStateCount()} seam used to answer for a whole registry. There
+ * is no registry to count now — an invocation's state is its plugin instance, and the SDK drops it — so the
+ * property worth asserting is that nothing the environment outlives keeps hold of it.
+ */
+ static boolean outstanding(InsightPlugin plugin) {
+ synchronized (plugin.scheduler) {
+ return plugin.record != null || plugin.exporting || plugin.settled != null || plugin.drainWaiters > 0;
+ }
+ }
+
+ /**
+ * The instance plus its first hook, in the order the SDK dispatches them: the factory is called with the very
+ * {@link InvocationInfo} that {@code onInvocationStart} then receives.
+ */
+ static InsightPlugin started(DurableExecutionPluginFactory factory, InvocationInfo info) {
+ InsightPlugin plugin = plugin(factory, info);
+ plugin.onInvocationStart(info);
+ return plugin;
+ }
+}
diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFanOutReentryTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFanOutReentryTest.java
new file mode 100644
index 000000000..973fe97d3
--- /dev/null
+++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFanOutReentryTest.java
@@ -0,0 +1,199 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.insight;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.api.Test;
+
+/**
+ * A {@code flush()} or {@code drain()} issued from an exporter fan-out worker is refused and reported, not waited on.
+ *
+ *
With two or more exporters configured, the pump does not run the exporter callbacks itself: it submits one task
+ * per exporter and then waits for all of them. A callback therefore runs on a worker the pump is blocked on, and a wait
+ * for the pump issued from that worker is a wait-for cycle two threads wide — the worker parks on a future only the
+ * pump can complete, and the pump cannot resume its loop until that worker returns. The single-exporter case runs the
+ * callback on the pump thread itself and is covered by {@link ExportSchedulerReentrantFlushTest}; this covers the
+ * fan-out, which the pump-thread identity check alone does not recognize.
+ */
+class ExportSchedulerFanOutReentryTest {
+
+ /** Longest a call that must return promptly may take before the property under test is considered broken. */
+ private static final long DEADLINE_MILLIS = 5_000;
+
+ private static String arn(int index) {
+ return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1";
+ }
+
+ private static WorkflowInsightRecord record(String executionArn, String status) {
+ var r = new WorkflowInsightRecord();
+ r.executionArn = executionArn;
+ r.status = status;
+ return r;
+ }
+
+ /** Unbounded thread-per-task executor, like the cached pool the plugin injects in production. */
+ private static Executor sharedWorkers() {
+ return command -> {
+ var thread = new Thread(command, "fan-out-reentry-worker");
+ thread.setDaemon(true);
+ thread.start();
+ };
+ }
+
+ private static final class CountingExporter implements InsightExporter {
+ final AtomicInteger exports = new AtomicInteger();
+ final AtomicInteger flushes = new AtomicInteger();
+
+ @Override
+ public void export(WorkflowInsightRecord record) {
+ exports.incrementAndGet();
+ }
+
+ @Override
+ public void flush() {
+ flushes.incrementAndGet();
+ }
+ }
+
+ @Test
+ void flushReenteredFromAFanOutWorkerIsRefusedReportedAndLosesNoWork() throws Exception {
+ var first = new CountingExporter();
+ var second = new CountingExporter();
+ var failures = new CopyOnWriteArrayList();
+ var holder = new AtomicReference();
+ var reentrantFlushReturned = new CountDownLatch(1);
+ var flushesSeenByTheRefusedCall = new AtomicInteger(-1);
+ var reentered = new AtomicInteger();
+
+ var scheduler = new ExportScheduler(
+ List.of(first, second),
+ (rec, exp) -> {
+ exp.export(rec);
+ // Only the first exporter re-enters, and only once, so exactly one refusal is expected.
+ if (exp == first && "SUCCEEDED".equals(rec.status()) && reentered.getAndIncrement() == 0) {
+ holder.get().flush();
+ flushesSeenByTheRefusedCall.set(first.flushes.get() + second.flushes.get());
+ reentrantFlushReturned.countDown();
+ }
+ },
+ failures::add,
+ sharedWorkers());
+ holder.set(scheduler);
+
+ var drainReturned = new CountDownLatch(1);
+ var firstExecution = Executions.plugin(scheduler, arn(0));
+ var invocation = new Thread(
+ () -> {
+ scheduler.schedule(firstExecution, record(arn(0), "SUCCEEDED"));
+ scheduler.drain(firstExecution);
+ drainReturned.countDown();
+ },
+ "fan-out-reentry-invocation");
+ invocation.setDaemon(true);
+ invocation.start();
+
+ assertTrue(
+ reentrantFlushReturned.await(DEADLINE_MILLIS, MILLISECONDS),
+ "flush() re-entered from an exporter fan-out worker never returned: the pump is waiting for that"
+ + " worker, so nothing can serve the request it made");
+ assertTrue(
+ drainReturned.await(DEADLINE_MILLIS, MILLISECONDS),
+ "drain() never returned after the re-entrant flush");
+ invocation.join(DEADLINE_MILLIS);
+
+ assertEquals(1, failures.size(), "exactly one failure reported: " + failures);
+ assertTrue(
+ failures.get(0) instanceof IllegalStateException,
+ "the refusal is reported as an IllegalStateException: " + failures.get(0));
+ assertTrue(
+ failures.get(0).getMessage().contains("flush()"),
+ "the report names the refused call: " + failures.get(0).getMessage());
+ assertEquals(0, flushesSeenByTheRefusedCall.get(), "the refused request must not have reached an exporter");
+
+ // No work lost: the record that was in flight reached both exporters.
+ assertEquals(1, first.exports.get(), "the record reached the first exporter");
+ assertEquals(1, second.exports.get(), "the record reached the second exporter");
+
+ // Still usable from a thread that is not pump-dependent.
+ var secondExecution = Executions.plugin(scheduler, arn(1));
+ scheduler.schedule(secondExecution, record(arn(1), "SUCCEEDED"));
+ scheduler.drain(secondExecution);
+ scheduler.flush();
+
+ assertEquals(2, first.exports.get(), "both records reached the first exporter");
+ assertEquals(2, second.exports.get(), "both records reached the second exporter");
+ assertEquals(1, first.flushes.get(), "the later flush is served normally on the first exporter");
+ assertEquals(1, second.flushes.get(), "the later flush is served normally on the second exporter");
+ assertEquals(1, failures.size(), "no further failure after the refusal: " + failures);
+ }
+
+ @Test
+ void drainReenteredFromAFanOutWorkerIsRefusedReportedAndLosesNoWork() throws Exception {
+ var first = new CountingExporter();
+ var second = new CountingExporter();
+ var failures = new CopyOnWriteArrayList();
+ var holder = new AtomicReference();
+ var pluginHolder = new AtomicReference();
+ var reentrantDrainReturned = new CountDownLatch(1);
+ var reentered = new AtomicInteger();
+
+ var scheduler = new ExportScheduler(
+ List.of(first, second),
+ (rec, exp) -> {
+ exp.export(rec);
+ if (exp == first && "SUCCEEDED".equals(rec.status()) && reentered.getAndIncrement() == 0) {
+ holder.get().drain(pluginHolder.get());
+ reentrantDrainReturned.countDown();
+ }
+ },
+ failures::add,
+ sharedWorkers());
+ holder.set(scheduler);
+
+ var execution = Executions.plugin(scheduler, arn(0));
+ pluginHolder.set(execution);
+
+ var drainReturned = new CountDownLatch(1);
+ var invocation = new Thread(
+ () -> {
+ scheduler.schedule(execution, record(arn(0), "SUCCEEDED"));
+ scheduler.drain(execution);
+ drainReturned.countDown();
+ },
+ "fan-out-reentry-drain-invocation");
+ invocation.setDaemon(true);
+ invocation.start();
+
+ assertTrue(
+ reentrantDrainReturned.await(DEADLINE_MILLIS, MILLISECONDS),
+ "drain() re-entered from an exporter fan-out worker never returned: the pump is waiting for that"
+ + " worker, so nothing can settle the signal it waited for");
+ assertTrue(
+ drainReturned.await(DEADLINE_MILLIS, MILLISECONDS),
+ "the invocation's own drain() never returned after the re-entrant drain");
+ invocation.join(DEADLINE_MILLIS);
+
+ assertEquals(1, failures.size(), "exactly one failure reported: " + failures);
+ assertTrue(
+ failures.get(0) instanceof IllegalStateException,
+ "the refusal is reported as an IllegalStateException: " + failures.get(0));
+ assertTrue(
+ failures.get(0).getMessage().contains("drain"),
+ "the report names the refused call: " + failures.get(0).getMessage());
+
+ // No work lost by refusing the drain: the record still reached every exporter, and the invocation's own drain
+ // returned only once it had.
+ assertEquals(1, first.exports.get(), "the record reached the first exporter");
+ assertEquals(1, second.exports.get(), "the record reached the second exporter");
+ assertTrue(!Executions.outstanding(execution), "the scheduler owes the invocation nothing after its drain");
+ }
+}
diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushCoalescingTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushCoalescingTest.java
new file mode 100644
index 000000000..fc162981f
--- /dev/null
+++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushCoalescingTest.java
@@ -0,0 +1,311 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.insight;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Contract tests for the flush cadence: at most one flush per invocation end that asks for one, invocation ends that
+ * overlap may share a flush, and a request made while a flush is already running is never satisfied by that flush.
+ *
+ * Coalescing is sound because every requester drains its own record before asking, so a flush that starts
+ * after the request was made has that record in the buffer. It is what stops N ends that ask together from paying for N
+ * serialized flush fan-outs — a 60 ms exporter flush cost the slowest of 8 ends ~520 ms before this change.
+ */
+class ExportSchedulerFlushCoalescingTest {
+
+ /** A slow flush must not cost the caller more than a small multiple of the one flush it asked for. */
+ private static final int PROMPTNESS_FACTOR = 3;
+
+ private static String arn(int index) {
+ return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1";
+ }
+
+ private static WorkflowInsightRecord record(String executionArn, String status) {
+ var r = new WorkflowInsightRecord();
+ r.executionArn = executionArn;
+ r.status = status;
+ return r;
+ }
+
+ private static ExportScheduler scheduler(Executor executor, InsightExporter... exporters) {
+ return new ExportScheduler(List.of(exporters), (rec, exp) -> exp.export(rec), t -> {}, executor);
+ }
+
+ /** Unbounded thread-per-task executor, like the cached pool the plugin injects in production. */
+ private static Executor sharedWorkers() {
+ return command -> {
+ var thread = new Thread(command, "coalescing-worker");
+ thread.setDaemon(true);
+ thread.start();
+ };
+ }
+
+ private static final class SlowFlushExporter implements InsightExporter {
+ private final long flushMillis;
+ final AtomicInteger flushes = new AtomicInteger();
+
+ SlowFlushExporter(long flushMillis) {
+ this.flushMillis = flushMillis;
+ }
+
+ @Override
+ public void export(WorkflowInsightRecord record) {}
+
+ @Override
+ public void flush() {
+ flushes.incrementAndGet();
+ sleep(flushMillis);
+ }
+ }
+
+ /**
+ * N invocation ends whose records have already been delivered ask for their flush together: they share one flush,
+ * so the slowest pays a small multiple of one flush rather than N times one.
+ *
+ *
The records are drained before the requests are made on purpose: this is coalescing on its own, with every
+ * request already queued when the pump reaches its flush step. The harder case — ends that are still inside
+ * {@code drain()} when the first request is served, and so cannot have asked yet — is
+ * {@link #simultaneousDrainAndFlushEndsShareAFlushRatherThanOneEach()}.
+ */
+ @Test
+ void invocationEndsAskingForAFlushTogetherShareOneFlush() {
+ long flushMillis = 60;
+ int executions = 8;
+ var exporter = new SlowFlushExporter(flushMillis);
+ var scheduler = scheduler(sharedWorkers(), exporter);
+
+ var durations = Collections.synchronizedList(new java.util.ArrayList());
+ var recordsDelivered = new CyclicBarrier(executions);
+ var done = new CountDownLatch(executions);
+ for (int i = 0; i < executions; i++) {
+ String executionArn = arn(i);
+ InsightPlugin execution = Executions.plugin(scheduler, executionArn);
+ start("end-" + i, () -> {
+ scheduler.schedule(execution, record(executionArn, "SUCCEEDED"));
+ scheduler.drain(execution);
+ awaitBarrier(recordsDelivered);
+ long began = System.nanoTime();
+ scheduler.flush();
+ durations.add((System.nanoTime() - began) / 1_000_000L);
+ done.countDown();
+ });
+ }
+ assertTrue(await(done, 60_000), "an invocation end never returned");
+
+ long slowest = Collections.max(durations);
+ int flushes = exporter.flushes.get();
+ System.out.printf(
+ "COALESCING: %d invocation ends flushing together, %d ms exporter flush | slowest end returned after"
+ + " %d ms | flushes run: %d (one per end, %d, before coalescing)%n",
+ executions, flushMillis, slowest, flushes, executions);
+
+ assertTrue(flushes >= 1, "every invocation end must be covered by a flush");
+ assertTrue(flushes <= executions, "at most one flush per invocation end: " + flushes + " for " + executions);
+ assertTrue(
+ slowest <= flushMillis * PROMPTNESS_FACTOR,
+ "the slowest invocation end waited " + slowest + " ms for a " + flushMillis + " ms flush: ends that ask"
+ + " together must share a flush rather than serialize one fan-out each");
+ }
+
+ /**
+ * The realistic shape: schedule, drain, flush, all landing at once. An end cannot ask for its flush until its own
+ * record has been exported, so the pump exports the records a drain is waiting for before it spends a flush
+ * fan-out; without that the ends are staggered one record per flush and each pays for a flush of its own.
+ */
+ @Test
+ void simultaneousDrainAndFlushEndsShareAFlushRatherThanOneEach() {
+ long flushMillis = 40;
+ int executions = 8;
+ var exporter = new SlowFlushExporter(flushMillis);
+ var scheduler = scheduler(sharedWorkers(), exporter);
+
+ var durations = Collections.synchronizedList(new java.util.ArrayList());
+ var barrier = new CyclicBarrier(executions);
+ var done = new CountDownLatch(executions);
+ for (int i = 0; i < executions; i++) {
+ String executionArn = arn(i);
+ InsightPlugin execution = Executions.plugin(scheduler, executionArn);
+ start("drain-and-flush-" + i, () -> {
+ awaitBarrier(barrier);
+ long began = System.nanoTime();
+ scheduler.schedule(execution, record(executionArn, "SUCCEEDED"));
+ scheduler.drain(execution);
+ scheduler.flush();
+ durations.add((System.nanoTime() - began) / 1_000_000L);
+ done.countDown();
+ });
+ }
+ assertTrue(await(done, 60_000), "an invocation end never returned");
+
+ int flushes = exporter.flushes.get();
+ long slowest = Collections.max(durations);
+ System.out.printf(
+ "COALESCING (drain then flush): %d simultaneous ends, %d ms exporter flush | slowest end after %d ms |"
+ + " flushes run: %d (one per end, %d, before coalescing)%n",
+ executions, flushMillis, slowest, flushes, executions);
+ assertTrue(flushes >= 1, "every invocation end must be covered by a flush");
+ assertTrue(flushes <= executions, "at most one flush per invocation end: " + flushes + " for " + executions);
+ assertTrue(
+ slowest <= flushMillis * (PROMPTNESS_FACTOR + 1),
+ "the slowest of " + executions + " simultaneous ends waited " + slowest + " ms for a " + flushMillis
+ + " ms flush: an end must not pay for one flush per end");
+ }
+
+ /**
+ * The exact counts, deterministically: five requests made while a flush is running are not satisfied by it — that
+ * flush cannot have seen their records — and they share the single flush that follows it.
+ */
+ @Test
+ void requestsMadeWhileAFlushRunsShareTheNextFlushAndAreNeverSatisfiedByTheRunningOne() {
+ var insideFirstFlush = new CountDownLatch(1);
+ var releaseFirstFlush = new CountDownLatch(1);
+ var flushStarts = new AtomicInteger();
+ var flushCompletions = new AtomicInteger();
+ var exporter = new InsightExporter() {
+ @Override
+ public void export(WorkflowInsightRecord record) {}
+
+ @Override
+ public void flush() {
+ if (flushStarts.incrementAndGet() == 1) {
+ insideFirstFlush.countDown();
+ await(releaseFirstFlush, 30_000);
+ }
+ flushCompletions.incrementAndGet();
+ }
+ };
+ var scheduler = scheduler(sharedWorkers(), exporter);
+
+ start("first-flusher", scheduler::flush);
+ assertTrue(await(insideFirstFlush, 5_000), "the pump never entered the first flush");
+
+ int latecomers = 5;
+ var returned = new CountDownLatch(latecomers);
+ var startsSeenOnReturn = new CopyOnWriteArrayList();
+ for (int i = 0; i < latecomers; i++) {
+ start("latecomer-" + i, () -> {
+ scheduler.flush();
+ startsSeenOnReturn.add(flushStarts.get());
+ returned.countDown();
+ });
+ }
+ sleep(300); // every latecomer is queued while the first flush is still inside the exporter
+
+ assertFalse(
+ await(returned, 200),
+ "a request made while a flush was already running was satisfied by that flush, which cannot have seen"
+ + " the requester's record");
+ releaseFirstFlush.countDown();
+ assertTrue(await(returned, 10_000), "a queued request was never served");
+
+ assertEquals(
+ 2,
+ flushStarts.get(),
+ "the five latecomers must share exactly one flush, taken as a batch after the first one ended");
+ assertTrue(
+ startsSeenOnReturn.stream().allMatch(starts -> starts >= 2),
+ "each latecomer must be served by a flush that started after it was enqueued: " + startsSeenOnReturn);
+ assertEquals(2, flushCompletions.get(), "no flush ran twice for the same batch");
+ }
+
+ /**
+ * The same property under load, measured per request: when a request returns, a flush that started after the
+ * request was made must already have completed. Flushes are serialized by the pump, so counting completions is
+ * enough — a request satisfied by a flush that was already running would return with the completion count still at
+ * or below the value observed before it asked.
+ */
+ @Test
+ void everyRequestIsSatisfiedByAFlushThatStartedAfterItWasMade() {
+ var flushStarts = new AtomicInteger();
+ var flushCompletions = new AtomicInteger();
+ var exporter = new InsightExporter() {
+ @Override
+ public void export(WorkflowInsightRecord record) {}
+
+ @Override
+ public void flush() {
+ flushStarts.incrementAndGet();
+ sleep(1);
+ flushCompletions.incrementAndGet();
+ }
+ };
+ var scheduler = scheduler(sharedWorkers(), exporter);
+
+ int requests = 120;
+ var done = new CountDownLatch(requests);
+ var violations = new CopyOnWriteArrayList();
+ for (int i = 0; i < requests; i++) {
+ String executionArn = arn(i);
+ InsightPlugin execution = Executions.plugin(scheduler, executionArn);
+ start("load-flusher-" + i, () -> {
+ scheduler.schedule(execution, record(executionArn, "SUCCEEDED"));
+ scheduler.drain(execution);
+ int startsBefore = flushStarts.get();
+ scheduler.flush();
+ if (flushCompletions.get() <= startsBefore) {
+ violations.add("returned with completions=" + flushCompletions.get() + " after observing starts="
+ + startsBefore);
+ }
+ done.countDown();
+ });
+ }
+ assertTrue(await(done, 60_000), "a request was never served");
+ scheduler.drainAll();
+
+ System.out.printf("COALESCING under load: %d requests satisfied by %d flushes%n", requests, flushStarts.get());
+ assertEquals(List.of(), violations, "a request was credited to a flush that was already running");
+ assertTrue(flushStarts.get() >= 1);
+ assertTrue(
+ flushStarts.get() <= requests,
+ "at most one flush per request: " + flushStarts.get() + " for " + requests);
+ }
+
+ private static Thread start(String name, Runnable body) {
+ var thread = new Thread(body, name);
+ thread.setDaemon(true);
+ thread.start();
+ return thread;
+ }
+
+ private static boolean await(CountDownLatch latch, long timeoutMillis) {
+ try {
+ return latch.await(timeoutMillis, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return false;
+ }
+ }
+
+ private static void awaitBarrier(CyclicBarrier barrier) {
+ try {
+ barrier.await(60, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ private static void sleep(long millis) {
+ if (millis <= 0) {
+ return;
+ }
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(e);
+ }
+ }
+}
diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushSerializationTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushSerializationTest.java
new file mode 100644
index 000000000..ff6254539
--- /dev/null
+++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerFlushSerializationTest.java
@@ -0,0 +1,434 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.insight;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Contract tests for the exporter-facing flush guarantee: {@code flush()} is served by the export pump, so it is never
+ * called concurrently with {@code export()} on the same plugin instance, and cannot be starved by a queue that keeps
+ * receiving records.
+ *
+ * The cadence itself — at most one flush per request, requests that overlap sharing one flush — is covered by
+ * {@link ExportSchedulerFlushCoalescingTest}.
+ */
+class ExportSchedulerFlushSerializationTest {
+
+ /** Longest a flush may take to be served before the property under test is considered broken. */
+ private static final long FLUSH_DEADLINE_MILLIS = 2_000;
+
+ private static String arn(int index) {
+ return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1";
+ }
+
+ private static WorkflowInsightRecord record(String executionArn, String status) {
+ var r = new WorkflowInsightRecord();
+ r.executionArn = executionArn;
+ r.status = status;
+ return r;
+ }
+
+ private static ExportScheduler scheduler(
+ Executor executor, List failures, InsightExporter... exporters) {
+ return new ExportScheduler(List.of(exporters), (rec, exp) -> exp.export(rec), failures::add, executor);
+ }
+
+ /** Unbounded thread-per-task executor, like the cached pool the plugin injects in production. */
+ private static Executor sharedWorkers() {
+ return command -> {
+ var thread = new Thread(command, "test-export-worker");
+ thread.setDaemon(true);
+ thread.start();
+ };
+ }
+
+ /**
+ * Records, per exporter instance, whether an {@code export()} and a {@code flush()} were ever inside the exporter
+ * at the same time, and how many of each ran concurrently.
+ */
+ private static final class OverlapProbeExporter implements InsightExporter {
+ private final long exportMillis;
+ private final long flushMillis;
+ final AtomicInteger inExport = new AtomicInteger();
+ final AtomicInteger inFlush = new AtomicInteger();
+ final AtomicInteger maxConcurrentExports = new AtomicInteger();
+ final AtomicInteger maxConcurrentFlushes = new AtomicInteger();
+ final AtomicInteger flushes = new AtomicInteger();
+ final List exported = new CopyOnWriteArrayList<>();
+ final AtomicBoolean overlapped = new AtomicBoolean();
+
+ OverlapProbeExporter(long exportMillis, long flushMillis) {
+ this.exportMillis = exportMillis;
+ this.flushMillis = flushMillis;
+ }
+
+ @Override
+ public void export(WorkflowInsightRecord record) {
+ trackMax(maxConcurrentExports, inExport.incrementAndGet());
+ try {
+ checkOverlap();
+ sleep(exportMillis);
+ checkOverlap();
+ exported.add(record.status() + "@" + record.executionArn());
+ } finally {
+ inExport.decrementAndGet();
+ }
+ }
+
+ @Override
+ public void flush() {
+ trackMax(maxConcurrentFlushes, inFlush.incrementAndGet());
+ try {
+ checkOverlap();
+ sleep(flushMillis);
+ checkOverlap();
+ flushes.incrementAndGet();
+ } finally {
+ inFlush.decrementAndGet();
+ }
+ }
+
+ private void checkOverlap() {
+ if (inExport.get() > 0 && inFlush.get() > 0) {
+ overlapped.set(true);
+ }
+ }
+
+ private static void trackMax(AtomicInteger max, int observed) {
+ max.accumulateAndGet(observed, Math::max);
+ }
+ }
+
+ @Test
+ void aFlushNeverOverlapsAnExportEvenWithManyExecutionsEndingAtOnce() throws Exception {
+ var first = new OverlapProbeExporter(15, 5);
+ var second = new OverlapProbeExporter(15, 5);
+ var failures = new CopyOnWriteArrayList();
+ var scheduler = scheduler(sharedWorkers(), failures, first, second);
+
+ int executions = 6;
+ var barrier = new CyclicBarrier(executions);
+ var threads = new ArrayList();
+ for (int i = 0; i < executions; i++) {
+ String executionArn = arn(i);
+ InsightPlugin execution = Executions.plugin(scheduler, executionArn);
+ var thread = new Thread(
+ () -> {
+ awaitBarrier(barrier);
+ // What an invocation does: a few RUNNING snapshots, the terminal record, then drain + flush.
+ for (int change = 0; change < 3; change++) {
+ scheduler.schedule(execution, record(executionArn, "RUNNING"));
+ }
+ scheduler.schedule(execution, record(executionArn, "SUCCEEDED"));
+ scheduler.drain(execution);
+ scheduler.flush();
+ },
+ "invocation-" + i);
+ thread.setDaemon(true);
+ threads.add(thread);
+ thread.start();
+ }
+ for (Thread thread : threads) {
+ thread.join(30_000);
+ assertFalse(thread.isAlive(), thread.getName() + " never returned from drain/flush");
+ }
+
+ for (OverlapProbeExporter exporter : List.of(first, second)) {
+ assertFalse(exporter.overlapped.get(), "flush() ran while an export was in flight on the same exporter");
+ assertEquals(1, exporter.maxConcurrentExports.get(), "exports must stay serialized");
+ assertEquals(1, exporter.maxConcurrentFlushes.get(), "one flush at a time, one fan-out per batch");
+ // At most one flush per invocation end, and at least one: ends that land together share a flush, so the
+ // count is bounded by the number of ends rather than equal to it.
+ assertTrue(exporter.flushes.get() >= 1, "every invocation end must be covered by a flush");
+ assertTrue(
+ exporter.flushes.get() <= executions,
+ "at most one flush per invocation end: " + exporter.flushes.get() + " for " + executions);
+ for (int i = 0; i < executions; i++) {
+ assertTrue(
+ exporter.exported.contains("SUCCEEDED@" + arn(i)),
+ "terminal record of " + arn(i) + " never reached the exporter");
+ }
+ }
+ assertTrue(failures.isEmpty(), "no failure should be reported: " + failures);
+ }
+
+ @Test
+ void twoFlushRequestsQueuedTogetherShareOneFlush() throws Exception {
+ var exporting = new CountDownLatch(1);
+ var release = new CountDownLatch(1);
+ var flushes = new AtomicInteger();
+ var exporter = new InsightExporter() {
+ @Override
+ public void export(WorkflowInsightRecord record) {
+ exporting.countDown();
+ await(release);
+ }
+
+ @Override
+ public void flush() {
+ flushes.incrementAndGet();
+ }
+ };
+ var scheduler = scheduler(sharedWorkers(), new CopyOnWriteArrayList<>(), exporter);
+
+ scheduler.schedule(Executions.plugin(scheduler, arn(0)), record(arn(0), "SUCCEEDED"));
+ assertTrue(exporting.await(5, TimeUnit.SECONDS), "the pump is inside the exporter");
+
+ var flushed = new CountDownLatch(2);
+ for (int i = 0; i < 2; i++) {
+ var flusher = new Thread(
+ () -> {
+ scheduler.flush();
+ flushed.countDown();
+ },
+ "flusher-" + i);
+ flusher.setDaemon(true);
+ flusher.start();
+ }
+ Thread.sleep(200); // let both requests queue up behind the in-flight export
+
+ release.countDown();
+ assertTrue(flushed.await(5, TimeUnit.SECONDS), "both flush requests must be served");
+ assertEquals(
+ 1,
+ flushes.get(),
+ "two requests queued together are taken as one batch and share a single flush: both drained their own"
+ + " record before asking, so one flush covers both");
+ }
+
+ /** Distinct {@link Error} type so the test asserts on this exact failure rather than any Error. */
+ private static final class FlushError extends Error {
+ FlushError() {
+ super("flush blew up with an Error");
+ }
+ }
+
+ @Test
+ void aFlushThatThrowsStillReleasesTheInvocationAndLetsTheOtherExportersFlush() throws Exception {
+ var flushed = new AtomicInteger();
+ var throwsException = new InsightExporter() {
+ @Override
+ public void export(WorkflowInsightRecord record) {}
+
+ @Override
+ public void flush() {
+ throw new IllegalStateException("flush blew up");
+ }
+ };
+ var throwsError = new InsightExporter() {
+ @Override
+ public void export(WorkflowInsightRecord record) {}
+
+ @Override
+ public void flush() {
+ throw new FlushError();
+ }
+ };
+ var healthy = new InsightExporter() {
+ @Override
+ public void export(WorkflowInsightRecord record) {}
+
+ @Override
+ public void flush() {
+ flushed.incrementAndGet();
+ }
+ };
+ var failures = new CopyOnWriteArrayList();
+ var scheduler = scheduler(sharedWorkers(), failures, throwsException, throwsError, healthy);
+
+ assertTrue(returnsWithin(scheduler::flush, FLUSH_DEADLINE_MILLIS), "a throwing flush stranded the invocation");
+
+ assertEquals(1, flushed.get(), "the healthy exporter still flushed");
+ assertEquals(2, failures.size(), "both failures are reported, neither escapes: " + failures);
+ assertTrue(
+ failures.stream().anyMatch(t -> t instanceof IllegalStateException),
+ "the thrown exception is reported");
+ assertTrue(failures.stream().anyMatch(t -> t instanceof FlushError), "the thrown Error is reported");
+ }
+
+ @Test
+ void anErrorFromTheOnlyExportersFlushStillReleasesTheInvocation() throws Exception {
+ var failures = new CopyOnWriteArrayList();
+ var onlyExporter = new InsightExporter() {
+ @Override
+ public void export(WorkflowInsightRecord record) {}
+
+ @Override
+ public void flush() {
+ throw new FlushError();
+ }
+ };
+ var scheduler = scheduler(sharedWorkers(), failures, onlyExporter);
+
+ assertTrue(returnsWithin(scheduler::flush, FLUSH_DEADLINE_MILLIS), "an Error from flush() stranded the caller");
+ assertEquals(1, failures.size(), "the Error is reported, not propagated: " + failures);
+ assertTrue(failures.get(0) instanceof FlushError);
+ }
+
+ @Test
+ void aFlushIsServedWhileTheQueueKeepsReceivingRecords() throws Exception {
+ var exports = new AtomicInteger();
+ var flushes = new AtomicInteger();
+ var exporter = new InsightExporter() {
+ @Override
+ public void export(WorkflowInsightRecord record) {
+ sleep(2);
+ exports.incrementAndGet();
+ }
+
+ @Override
+ public void flush() {
+ flushes.incrementAndGet();
+ }
+ };
+ var scheduler = scheduler(sharedWorkers(), new CopyOnWriteArrayList<>(), exporter);
+
+ // A producer that never lets the queue run dry: it keeps re-scheduling a fixed, rotating set of executions, so
+ // `pending` stays non-empty (and bounded, since records coalesce per execution) for as long as it runs.
+ var rotation = new ArrayList();
+ for (int i = 0; i < 50; i++) {
+ rotation.add(Executions.plugin(scheduler, arn(i)));
+ }
+ var stop = new AtomicBoolean();
+ var scheduled = new AtomicInteger();
+ var producing = new CountDownLatch(1);
+ var producer = new Thread(
+ () -> {
+ int index = 0;
+ while (!stop.get()) {
+ InsightPlugin execution = rotation.get(index++ % rotation.size());
+ scheduler.schedule(execution, record(execution.executionArn, "RUNNING"));
+ scheduled.incrementAndGet();
+ producing.countDown();
+ }
+ },
+ "record-producer");
+ producer.setDaemon(true);
+ producer.start();
+ int exportsBefore;
+ int scheduledBefore;
+ boolean served;
+ var exportsWhenServed = new AtomicInteger();
+ var scheduledWhenServed = new AtomicInteger();
+ var flushReturned = new CountDownLatch(1);
+ try {
+ assertTrue(producing.await(5, TimeUnit.SECONDS), "the producer never started scheduling");
+ exportsBefore = exports.get();
+ scheduledBefore = scheduled.get();
+
+ // On its own thread with a deadline: a starved flush must fail this test, not hang it.
+ var flusher = new Thread(
+ () -> {
+ scheduler.flush();
+ exportsWhenServed.set(exports.get());
+ scheduledWhenServed.set(scheduled.get());
+ flushReturned.countDown();
+ },
+ "flusher");
+ flusher.setDaemon(true);
+ flusher.start();
+ served = flushReturned.await(FLUSH_DEADLINE_MILLIS, TimeUnit.MILLISECONDS);
+ } finally {
+ // Stop the producer before asserting, so a starved flush is released and its thread does not leak.
+ stop.set(true);
+ producer.join(10_000);
+ }
+
+ assertTrue(
+ served,
+ "the flush was not served within " + FLUSH_DEADLINE_MILLIS + " ms while the queue kept receiving"
+ + " records; it must be served between records rather than after the queue drains");
+ assertTrue(flushReturned.await(5, TimeUnit.SECONDS));
+ assertEquals(1, flushes.get(), "the flush was served exactly once");
+ assertTrue(
+ exportsWhenServed.get() > exportsBefore, "the pump kept exporting: the flush did not stall the queue");
+ assertTrue(
+ scheduledWhenServed.get() > scheduledBefore,
+ "the queue was still receiving records when the flush was served");
+ }
+
+ @Test
+ void theFlushHappensOnTheCallingThreadWhenNoWorkerCouldBeStarted() {
+ Executor rejecting = command -> {
+ throw new RejectedExecutionException("no worker");
+ };
+ var flushThreads = new CopyOnWriteArrayList();
+ var exporter = new InsightExporter() {
+ @Override
+ public void export(WorkflowInsightRecord record) {}
+
+ @Override
+ public void flush() {
+ flushThreads.add(Thread.currentThread());
+ }
+ };
+ var failures = new CopyOnWriteArrayList();
+ var scheduler = scheduler(rejecting, failures, exporter);
+
+ scheduler.flush();
+
+ assertEquals(1, flushThreads.size(), "the flush must still happen when no worker can be started");
+ assertSame(Thread.currentThread(), flushThreads.get(0), "the invocation boundary flushes inline");
+ assertFalse(failures.isEmpty(), "the rejected worker is reported");
+ }
+
+ /** Runs {@code action} on its own thread and reports whether it returned within the deadline. */
+ private static boolean returnsWithin(Runnable action, long timeoutMillis) throws InterruptedException {
+ var returned = new CountDownLatch(1);
+ var thread = new Thread(
+ () -> {
+ action.run();
+ returned.countDown();
+ },
+ "deadline-runner");
+ thread.setDaemon(true);
+ thread.start();
+ return returned.await(timeoutMillis, TimeUnit.MILLISECONDS);
+ }
+
+ private static void awaitBarrier(CyclicBarrier barrier) {
+ try {
+ barrier.await(10, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ private static void await(CountDownLatch latch) {
+ try {
+ if (!latch.await(5, TimeUnit.SECONDS)) {
+ throw new AssertionError("latch not released");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(e);
+ }
+ }
+
+ private static void sleep(long millis) {
+ if (millis <= 0) {
+ return;
+ }
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(e);
+ }
+ }
+}
diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerReentrantFlushTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerReentrantFlushTest.java
new file mode 100644
index 000000000..bb0577ca2
--- /dev/null
+++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerReentrantFlushTest.java
@@ -0,0 +1,131 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.insight;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.junit.jupiter.api.Test;
+
+/**
+ * A {@code flush()} issued from the thread currently serving the export pump is refused and reported, not waited on.
+ *
+ * That thread is the only one able to serve the request it would be making — flush requests are served by the pump,
+ * between records — so waiting for it is a wait-for cycle one thread wide, and the invocation never returns. With a
+ * single exporter the fan-out runs on the pump thread, so anything an exporter's {@code export()} does synchronously is
+ * enough to reach it.
+ */
+class ExportSchedulerReentrantFlushTest {
+
+ /** Longest a call that must return promptly may take before the property under test is considered broken. */
+ private static final long DEADLINE_MILLIS = 5_000;
+
+ private static String arn(int index) {
+ return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1";
+ }
+
+ private static WorkflowInsightRecord record(String executionArn, String status) {
+ var r = new WorkflowInsightRecord();
+ r.executionArn = executionArn;
+ r.status = status;
+ return r;
+ }
+
+ /** Unbounded thread-per-task executor, like the cached pool the plugin injects in production. */
+ private static Executor sharedWorkers() {
+ return command -> {
+ var thread = new Thread(command, "reentrant-flush-worker");
+ thread.setDaemon(true);
+ thread.start();
+ };
+ }
+
+ private static final class CountingExporter implements InsightExporter {
+ final AtomicInteger exports = new AtomicInteger();
+ final AtomicInteger flushes = new AtomicInteger();
+
+ @Override
+ public void export(WorkflowInsightRecord record) {
+ exports.incrementAndGet();
+ }
+
+ @Override
+ public void flush() {
+ flushes.incrementAndGet();
+ }
+ }
+
+ @Test
+ void flushReenteredFromThePumpThreadIsRefusedReportedAndLeavesTheSchedulerUsable() throws Exception {
+ var exporter = new CountingExporter();
+ var failures = new CopyOnWriteArrayList();
+ var holder = new AtomicReference();
+ var reentrantFlushReturned = new CountDownLatch(1);
+ var flushesSeenByTheRefusedCall = new AtomicInteger(-1);
+
+ var scheduler = new ExportScheduler(
+ List.of(exporter),
+ (rec, exp) -> {
+ exp.export(rec);
+ if (reentrantFlushReturned.getCount() > 0 && "SUCCEEDED".equals(rec.status())) {
+ // Re-entering the scheduler from inside the fan-out: this is the pump's own thread.
+ holder.get().flush();
+ flushesSeenByTheRefusedCall.set(exporter.flushes.get());
+ reentrantFlushReturned.countDown();
+ }
+ },
+ failures::add,
+ sharedWorkers());
+ holder.set(scheduler);
+
+ var drainReturned = new CountDownLatch(1);
+ var firstExecution = Executions.plugin(scheduler, arn(0));
+ var invocation = new Thread(
+ () -> {
+ scheduler.schedule(firstExecution, record(arn(0), "SUCCEEDED"));
+ scheduler.drain(firstExecution);
+ drainReturned.countDown();
+ },
+ "reentrant-flush-invocation");
+ invocation.setDaemon(true);
+ invocation.start();
+
+ assertTrue(
+ reentrantFlushReturned.await(DEADLINE_MILLIS, MILLISECONDS),
+ "flush() re-entered from the pump thread never returned: the only thread that can serve the request is"
+ + " the one waiting for it");
+ assertTrue(
+ drainReturned.await(DEADLINE_MILLIS, MILLISECONDS),
+ "drain() never returned after the re-entrant flush");
+ invocation.join(DEADLINE_MILLIS);
+
+ // Reported, not silently swallowed, and nothing thrown into the caller.
+ assertEquals(1, failures.size(), "exactly one failure reported: " + failures);
+ assertTrue(
+ failures.get(0) instanceof IllegalStateException,
+ "the refusal is reported as an IllegalStateException: " + failures.get(0));
+ assertTrue(
+ failures.get(0).getMessage().contains("flush()"),
+ "the report names the refused call: " + failures.get(0).getMessage());
+ assertEquals(0, flushesSeenByTheRefusedCall.get(), "the refused request must not have reached an exporter");
+
+ // Still usable: the next invocation's record is exported and its flush — from a thread that is not the pump —
+ // is
+ // served exactly as before.
+ var secondExecution = Executions.plugin(scheduler, arn(1));
+ scheduler.schedule(secondExecution, record(arn(1), "SUCCEEDED"));
+ scheduler.drain(secondExecution);
+ scheduler.flush();
+
+ assertEquals(2, exporter.exports.get(), "both records reached the exporter");
+ assertEquals(1, exporter.flushes.get(), "the next invocation's flush is served normally");
+ assertEquals(1, failures.size(), "no further failure after the refusal: " + failures);
+ }
+}
diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerTest.java
index 4f2d8af42..2c3df1a3e 100644
--- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerTest.java
+++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExportSchedulerTest.java
@@ -22,6 +22,13 @@
/** Contract tests for {@link ExportScheduler}: serial exports, latest-wins coalescing, drain, and exporter fan-out. */
class ExportSchedulerTest {
+ /** All single-execution cases below drive one invocation's plugin instance through the scheduler. */
+ private static final String ARN = arn(0);
+
+ private static String arn(int index) {
+ return "arn:aws:lambda:us-west-2:111122223333:function:f:$LATEST/durable-execution/exec-" + index + "/inv-1";
+ }
+
/** Runs submitted tasks only when the test asks, so pump timing is fully controlled. */
private static final class ManualExecutor implements Executor {
final Deque tasks = new ArrayDeque<>();
@@ -57,7 +64,7 @@ public void export(WorkflowInsightRecord record) {
}
private static WorkflowInsightRecord record(String status) {
- return record("arn:exec-a", status);
+ return record(ARN, status);
}
private static WorkflowInsightRecord record(String executionArn, String status) {
@@ -72,9 +79,11 @@ void recordsOfDifferentExecutionsNeverDisplaceEachOther() {
var executor = new ManualExecutor();
var exporter = new CapturingExporter();
var scheduler = scheduler(executor, new ArrayList<>(), exporter);
+ var executionA = Executions.plugin(scheduler, "arn:exec-a");
+ var executionB = Executions.plugin(scheduler, "arn:exec-b");
- scheduler.schedule(record("arn:exec-a", "a-final"));
- scheduler.schedule(record("arn:exec-b", "b-running"));
+ scheduler.schedule(executionA, record("arn:exec-a", "a-final"));
+ scheduler.schedule(executionB, record("arn:exec-b", "b-running"));
executor.runAll();
assertEquals(List.of("a-final", "b-running"), statuses(exporter));
@@ -85,12 +94,16 @@ void coalescingStaysWithinOneExecutionAndExecutionsAreServedInFirstPendingOrder(
var executor = new ManualExecutor();
var exporter = new CapturingExporter();
var scheduler = scheduler(executor, new ArrayList<>(), exporter);
+ var executionA = Executions.plugin(scheduler, "arn:exec-a");
+ var executionB = Executions.plugin(scheduler, "arn:exec-b");
+
+ var executionC = Executions.plugin(scheduler, "arn:exec-c");
- scheduler.schedule(record("arn:exec-a", "a1"));
- scheduler.schedule(record("arn:exec-b", "b1"));
- scheduler.schedule(record("arn:exec-a", "a2")); // supersedes a1 but keeps a's place ahead of b
- scheduler.schedule(record("arn:exec-b", "b2"));
- scheduler.schedule(record("arn:exec-c", "c1"));
+ scheduler.schedule(executionA, record("arn:exec-a", "a1"));
+ scheduler.schedule(executionB, record("arn:exec-b", "b1"));
+ scheduler.schedule(executionA, record("arn:exec-a", "a2")); // supersedes a1 but keeps a's place ahead of b
+ scheduler.schedule(executionB, record("arn:exec-b", "b2"));
+ scheduler.schedule(executionC, record("arn:exec-c", "c1"));
executor.runAll();
assertEquals(List.of("a2", "b2", "c1"), statuses(exporter));
@@ -111,14 +124,17 @@ public void export(WorkflowInsightRecord record) {
}
};
var scheduler = scheduler(sharedWorkers(), new ArrayList<>(), exporter);
+ var executionA = Executions.plugin(scheduler, "arn:exec-a");
+ var executionB = Executions.plugin(scheduler, "arn:exec-b");
- scheduler.schedule(record("arn:exec-a", "a-running"));
+ scheduler.schedule(executionA, record("arn:exec-a", "a-running"));
assertTrue(entered.await(5, TimeUnit.SECONDS), "a's first export is in flight");
- scheduler.schedule(record("arn:exec-a", "a-final"));
- scheduler.schedule(record("arn:exec-b", "b-running"));
- scheduler.schedule(record("arn:exec-b", "b-final"));
+ scheduler.schedule(executionA, record("arn:exec-a", "a-final"));
+ scheduler.schedule(executionB, record("arn:exec-b", "b-running"));
+ scheduler.schedule(executionB, record("arn:exec-b", "b-final"));
release.countDown();
- scheduler.drain();
+ scheduler.drain(executionA);
+ scheduler.drain(executionB);
assertEquals(List.of("a-running", "a-final", "b-final"), statuses(exporter));
}
@@ -133,8 +149,9 @@ void scheduleHandsTheRecordToAWorkerRatherThanExportingOnTheCallingThread() {
var executor = new ManualExecutor();
var exporter = new CapturingExporter();
var scheduler = scheduler(executor, new ArrayList<>(), exporter);
+ var execution = Executions.plugin(scheduler, ARN);
- scheduler.schedule(record("RUNNING"));
+ scheduler.schedule(execution, record("RUNNING"));
assertTrue(exporter.records.isEmpty(), "nothing exported until a worker runs");
executor.runAll();
@@ -147,10 +164,11 @@ void updatesScheduledBeforeTheWorkerRunsCollapseIntoTheLatestRecord() {
var executor = new ManualExecutor();
var exporter = new CapturingExporter();
var scheduler = scheduler(executor, new ArrayList<>(), exporter);
+ var execution = Executions.plugin(scheduler, ARN);
- scheduler.schedule(record("r1"));
- scheduler.schedule(record("r2"));
- scheduler.schedule(record("r3"));
+ scheduler.schedule(execution, record("r1"));
+ scheduler.schedule(execution, record("r2"));
+ scheduler.schedule(execution, record("r3"));
executor.runAll();
assertEquals(1, exporter.records.size(), "one pump, one latest record");
@@ -163,10 +181,11 @@ void recordScheduledAfterAPumpFinishesStartsANewPump() {
var executor = new ManualExecutor();
var exporter = new CapturingExporter();
var scheduler = scheduler(executor, new ArrayList<>(), exporter);
+ var execution = Executions.plugin(scheduler, ARN);
- scheduler.schedule(record("first"));
+ scheduler.schedule(execution, record("first"));
executor.runAll();
- scheduler.schedule(record("second"));
+ scheduler.schedule(execution, record("second"));
executor.runAll();
assertEquals(List.of("first", "second"), statuses(exporter));
@@ -187,14 +206,15 @@ public void export(WorkflowInsightRecord record) {
}
};
var scheduler = scheduler(sharedWorkers(), new ArrayList<>(), exporter);
+ var execution = Executions.plugin(scheduler, ARN);
- scheduler.schedule(record("first"));
+ scheduler.schedule(execution, record("first"));
assertTrue(entered.await(5, TimeUnit.SECONDS), "first export is in flight");
- scheduler.schedule(record("dropped-1"));
- scheduler.schedule(record("dropped-2"));
- scheduler.schedule(record("final"));
+ scheduler.schedule(execution, record("dropped-1"));
+ scheduler.schedule(execution, record("dropped-2"));
+ scheduler.schedule(execution, record("final"));
release.countDown();
- scheduler.drain();
+ scheduler.drain(execution);
assertEquals(List.of("first", "final"), statuses(exporter));
}
@@ -202,8 +222,9 @@ public void export(WorkflowInsightRecord record) {
@Test
void drainReturnsImmediatelyWhenIdle() {
var scheduler = scheduler(new ManualExecutor(), new ArrayList<>(), new CapturingExporter());
- scheduler.drain();
- scheduler.drain();
+ var execution = Executions.plugin(scheduler, ARN);
+ scheduler.drain(execution);
+ scheduler.drain(execution);
}
@Test
@@ -221,14 +242,15 @@ public void export(WorkflowInsightRecord record) {
}
};
var scheduler = scheduler(sharedWorkers(), new ArrayList<>(), exporter);
+ var execution = Executions.plugin(scheduler, ARN);
- scheduler.schedule(record("slow"));
+ scheduler.schedule(execution, record("slow"));
assertTrue(entered.await(5, TimeUnit.SECONDS));
- scheduler.schedule(record("final"));
+ scheduler.schedule(execution, record("final"));
var drained = new CountDownLatch(1);
var drainer = new Thread(() -> {
- scheduler.drain();
+ scheduler.drain(execution);
drained.countDown();
});
drainer.start();
@@ -243,9 +265,10 @@ public void export(WorkflowInsightRecord record) {
void exportersRunOffTheSchedulingThread() {
var exporter = new CapturingExporter();
var scheduler = scheduler(sharedWorkers(), new ArrayList<>(), exporter);
+ var execution = Executions.plugin(scheduler, ARN);
- scheduler.schedule(record("RUNNING"));
- scheduler.drain();
+ scheduler.schedule(execution, record("RUNNING"));
+ scheduler.drain(execution);
assertEquals(1, exporter.threads.size());
assertNotSame(Thread.currentThread(), exporter.threads.get(0));
@@ -259,9 +282,10 @@ void aFailingExporterNeverBlocksTheOthersForTheSameRecord() {
throw new AssertionError("exporter blew up");
};
var scheduler = scheduler(sharedWorkers(), failures, bad, good);
+ var execution = Executions.plugin(scheduler, ARN);
- scheduler.schedule(record("RUNNING"));
- scheduler.drain();
+ scheduler.schedule(execution, record("RUNNING"));
+ scheduler.drain(execution);
assertEquals(1, good.records.size());
assertEquals(1, failures.size());
@@ -274,8 +298,9 @@ void exportersForOneRecordRunConcurrentlySoASlowExporterDoesNotDelayTheOthers()
var fast = new CapturingExporter();
InsightExporter slow = record -> await(release);
var scheduler = scheduler(sharedWorkers(), new ArrayList<>(), slow, fast);
+ var execution = Executions.plugin(scheduler, ARN);
- scheduler.schedule(record("RUNNING"));
+ scheduler.schedule(execution, record("RUNNING"));
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
while (fast.records.isEmpty() && System.nanoTime() < deadline) {
Thread.sleep(5);
@@ -283,7 +308,7 @@ void exportersForOneRecordRunConcurrentlySoASlowExporterDoesNotDelayTheOthers()
assertEquals(1, fast.records.size(), "fast exporter received the record while the slow one is still blocked");
release.countDown();
- scheduler.drain();
+ scheduler.drain(execution);
}
@Test
@@ -293,12 +318,13 @@ void drainExportsThePendingRecordInlineWhenNoWorkerCouldBeStarted() {
var failures = new ArrayList();
var exporter = new CapturingExporter();
var scheduler = scheduler(executor, failures, exporter);
+ var execution = Executions.plugin(scheduler, ARN);
- scheduler.schedule(record("final"));
+ scheduler.schedule(execution, record("final"));
assertTrue(exporter.records.isEmpty(), "the hook thread does not export");
assertEquals(1, failures.size(), "the worker failure is reported");
- scheduler.drain();
+ scheduler.drain(execution);
assertEquals(List.of("final"), statuses(exporter));
assertSame(Thread.currentThread(), exporter.threads.get(0), "the invocation boundary delivers it");
@@ -310,14 +336,15 @@ void aLaterScheduleRetriesTheWorkerAfterARejection() {
executor.reject = true;
var exporter = new CapturingExporter();
var scheduler = scheduler(executor, new ArrayList<>(), exporter);
+ var execution = Executions.plugin(scheduler, ARN);
- scheduler.schedule(record("older"));
+ scheduler.schedule(execution, record("older"));
executor.reject = false;
- scheduler.schedule(record("newer"));
+ scheduler.schedule(execution, record("newer"));
executor.runAll();
assertEquals(List.of("newer"), statuses(exporter), "the retry exports the latest record");
- scheduler.drain();
+ scheduler.drain(execution);
assertEquals(1, exporter.records.size());
}
@@ -333,15 +360,16 @@ void aDrainThatObservedTheHandleBeforeTheWorkerWasRejectedStillCompletesInline()
var failures = new CopyOnWriteArrayList();
var exporter = new CapturingExporter();
var scheduler = scheduler(blockingRejector, failures, exporter);
+ var execution = Executions.plugin(scheduler, ARN);
- var scheduling = new Thread(() -> scheduler.schedule(record("final")), "scheduling");
+ var scheduling = new Thread(() -> scheduler.schedule(execution, record("final")), "scheduling");
scheduling.start();
assertTrue(submitted.await(5, TimeUnit.SECONDS), "the pump handle is published before execute rejects");
var drained = new CountDownLatch(1);
var drainer = new Thread(
() -> {
- scheduler.drain();
+ scheduler.drain(execution);
drained.countDown();
},
"drainer");
@@ -357,7 +385,7 @@ void aDrainThatObservedTheHandleBeforeTheWorkerWasRejectedStillCompletesInline()
}
@Test
- void flushAllRunsExporterFlushesConcurrentlySoASlowFlushDoesNotDelayTheOthers() throws Exception {
+ void flushRunsExporterFlushesConcurrentlySoASlowFlushDoesNotDelayTheOthers() throws Exception {
var release = new CountDownLatch(1);
var fastFlushed = new CountDownLatch(1);
var slow = new InsightExporter() {
@@ -382,19 +410,19 @@ public void flush() {
var flushed = new CountDownLatch(1);
new Thread(() -> {
- scheduler.flushAll();
+ scheduler.flush();
flushed.countDown();
})
.start();
assertTrue(fastFlushed.await(5, TimeUnit.SECONDS), "fast exporter flushed while the slow one is blocked");
- assertFalse(flushed.await(100, TimeUnit.MILLISECONDS), "flushAll waits for every exporter");
+ assertFalse(flushed.await(100, TimeUnit.MILLISECONDS), "flush waits for every exporter");
release.countDown();
assertTrue(flushed.await(5, TimeUnit.SECONDS));
}
@Test
- void flushAllIsolatesAFailingFlush() {
+ void flushIsolatesAFailingFlush() {
var failures = new CopyOnWriteArrayList();
var flushed = new CountDownLatch(1);
var bad = new InsightExporter() {
@@ -417,7 +445,7 @@ public void flush() {
};
var scheduler = scheduler(sharedWorkers(), failures, bad, good);
- scheduler.flushAll();
+ scheduler.flush();
assertEquals(0, flushed.getCount(), "the healthy exporter still flushed");
assertEquals(1, failures.size());
diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExporterIsolationTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExporterIsolationTest.java
index 61fe8ae68..4a297c100 100644
--- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExporterIsolationTest.java
+++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/ExporterIsolationTest.java
@@ -69,14 +69,17 @@ public void export(WorkflowInsightRecord record) {
void firstExporterMutationsDoNotLeakIntoLaterExporter() {
var mutating = new MutatingExporter();
var good = new CapturingExporter();
- var plugin = (WorkflowInsight.InsightPlugin) WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder()
- .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE)
- .content(ContentConfig.builder()
- .addOverride(OperationOverride.withResult("compute", r -> r))
- .build())
- .addExporter(mutating)
- .addExporter(good)
- .build());
+ var plugin = Executions.plugin(
+ WorkflowInsight.workflowInsight(WorkflowInsightConfig.builder()
+ .emitMode(WorkflowInsightConfig.EmitMode.ON_CHANGE)
+ .content(ContentConfig.builder()
+ .addOverride(OperationOverride.withResult("compute", r -> r))
+ .build())
+ .addExporter(mutating)
+ .addExporter(good)
+ .build()),
+ ARN,
+ START);
Map input = new LinkedHashMap<>();
input.put("k", "v");
diff --git a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InputSnapshotTest.java b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InputSnapshotTest.java
index abd5f89f7..4af7d55fb 100644
--- a/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InputSnapshotTest.java
+++ b/insight-plugin/src/test/java/software/amazon/lambda/durable/insight/InputSnapshotTest.java
@@ -13,7 +13,6 @@
import java.util.function.Function;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.services.lambda.model.OperationStatus;
-import software.amazon.lambda.durable.plugin.DurableExecutionPlugin;
import software.amazon.lambda.durable.plugin.InvocationEndInfo;
import software.amazon.lambda.durable.plugin.InvocationInfo;
import software.amazon.lambda.durable.plugin.InvocationStatus;
@@ -64,8 +63,11 @@ private Map ops() {
@Test
void handlerMutationAfterStartDoesNotCorruptCachedInputSnapshot() {
var exporter = new CapturingExporter();
- DurableExecutionPlugin plugin = WorkflowInsight.workflowInsight(
- WorkflowInsightConfig.builder().addExporter(exporter).build());
+ var plugin = Executions.plugin(
+ WorkflowInsight.workflowInsight(
+ WorkflowInsightConfig.builder().addExporter(exporter).build()),
+ ARN,
+ START);
// A mutable input whose nested list the handler mutates after the invocation has started.
List