From d060122e107b3d0bab2312d0bece86cae78a70ff Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Thu, 29 Oct 2026 16:46:47 -0700 Subject: [PATCH 01/13] feat: Park FDv2 synchronizers after unexpected failures --- .../server/DataSourceSynchronizerAdapter.java | 10 +- .../sdk/server/DataSystemComponents.java | 2 + .../sdk/server/FDv2DataSource.java | 196 +++++- .../sdk/server/FDv2DataSourceConditions.java | 33 +- .../launchdarkly/sdk/server/PollingBase.java | 8 +- .../sdk/server/PollingSynchronizerImpl.java | 16 + .../sdk/server/SourceManager.java | 42 +- .../sdk/server/StreamingSynchronizerImpl.java | 87 ++- .../server/SynchronizerFactoryWithState.java | 48 ++ .../server/datasources/FDv2SourceResult.java | 54 +- .../DataSourceSynchronizerAdapterTest.java | 37 + ...FDv2DataSourceConditionsAggregateTest.java | 7 +- .../FDv2DataSourceFallbackConditionTest.java | 37 +- .../FDv2DataSourceRecoveryConditionTest.java | 35 +- .../sdk/server/FDv2DataSourceTest.java | 639 ++++++++++++------ .../server/PollingSynchronizerImplTest.java | 108 +++ .../server/StreamingSynchronizerImplTest.java | 217 ++++++ 17 files changed, 1274 insertions(+), 302 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java index ab98fde5..81a35189 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java @@ -1,6 +1,8 @@ package com.launchdarkly.sdk.server; import com.launchdarkly.sdk.internal.collections.IterableAsyncQueue; +import com.launchdarkly.sdk.internal.http.FailureClass; +import com.launchdarkly.sdk.internal.http.HttpErrors; import com.launchdarkly.sdk.fdv2.Selector; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; import com.launchdarkly.sdk.server.datasources.Synchronizer; @@ -189,7 +191,13 @@ public void updateStatus(DataSourceStatusProvider.State newState, DataSourceStat // Convert state changes to FDv2SourceResult status events switch (newState) { case INTERRUPTED: - resultQueue.put(FDv2SourceResult.interrupted(newError, false)); + if (newError != null + && newError.getKind() == DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE + && HttpErrors.classifyHttpFailure(newError.getStatusCode()) == FailureClass.UNEXPECTED) { + resultQueue.put(FDv2SourceResult.terminalError(newError, false)); + } else { + resultQueue.put(FDv2SourceResult.interrupted(newError, false)); + } break; case OFF: if (newError != null) { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystemComponents.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystemComponents.java index 51526036..0875b6cf 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystemComponents.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystemComponents.java @@ -105,8 +105,10 @@ public Synchronizer build(DataSourceBuildInputs context) { StandardEndpoints.FDV2_STREAMING_REQUEST_PATH, context.getBaseLogger(), context.getSelectorSource(), + context.getSharedExecutor(), payloadFilter, initialReconnectDelay, + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, context.getThreadPriority(), context.getDiagnosticStore() ); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 360be672..8ba69533 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -11,6 +11,8 @@ import com.launchdarkly.sdk.server.subsystems.DataSource; import com.launchdarkly.sdk.server.subsystems.DataSourceUpdateSinkV2; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -30,17 +32,38 @@ class FDv2DataSource implements DataSource { /** * Default fallback timeout of 2 minutes. The timeout is only configurable for testing. */ - private static final int defaultFallbackTimeoutSeconds = 2 * 60; + private static final Duration DEFAULT_FALLBACK_TIMEOUT = Duration.ofMinutes(2); /** * Default recovery timeout of 5 minutes. The timeout is only configurable for testing. */ - private static final long defaultRecoveryTimeout = 5 * 60; + private static final Duration DEFAULT_RECOVERY_TIMEOUT = Duration.ofMinutes(5); + + /** + * Default base of the park delay that follows an unexpected failure. The delay is only configurable for testing. + */ + private static final Duration DEFAULT_PARK_INITIAL_DELAY = Duration.ofMinutes(5); + + /** + * Default maximum of the park delay. The delay is only configurable for testing. + */ + private static final Duration DEFAULT_PARK_MAX_DELAY = Duration.ofHours(1); + + /** + * Default base of the failover delay that follows a park. The delay is only configurable for testing. + */ + private static final Duration DEFAULT_FAILOVER_DELAY = Duration.ofSeconds(1); private final SourceManager sourceManager; private final List conditionFactories; + private final Duration parkInitialDelay; + + private final Duration parkMaxDelay; + + private final Duration failoverDelay; + private final DataSourceUpdateSinkV2 dataSourceUpdates; private final CompletableFuture startFuture = new CompletableFuture<>(); @@ -52,6 +75,11 @@ class FDv2DataSource implements DataSource { private volatile boolean closed = false; + /** + * Completes when the data source is closed. + */ + private final CompletableFuture closedSignal = new CompletableFuture<>(); + /** * Avoid duplicate orchestration logs for the same synchronizer and {@link SourceSignal}. */ @@ -79,8 +107,11 @@ public FDv2DataSource( threadPriority, logger, sharedExecutor, - defaultFallbackTimeoutSeconds, - defaultRecoveryTimeout + DEFAULT_FALLBACK_TIMEOUT, + DEFAULT_RECOVERY_TIMEOUT, + DEFAULT_PARK_INITIAL_DELAY, + DEFAULT_PARK_MAX_DELAY, + DEFAULT_FAILOVER_DELAY ); } @@ -92,8 +123,11 @@ public FDv2DataSource( int threadPriority, LDLogger logger, ScheduledExecutorService sharedExecutor, - long fallbackTimeout, - long recoveryTimeout + Duration fallbackTimeout, + Duration recoveryTimeout, + Duration parkInitialDelay, + Duration parkMaxDelay, + Duration failoverDelay ) { List synchronizerFactories = synchronizers .stream() @@ -120,6 +154,9 @@ public FDv2DataSource( this.conditionFactories = new ArrayList<>(); this.conditionFactories.add(new FallbackCondition.Factory(sharedExecutor, fallbackTimeout)); this.conditionFactories.add(new RecoveryCondition.Factory(sharedExecutor, recoveryTimeout)); + this.parkInitialDelay = parkInitialDelay; + this.parkMaxDelay = parkMaxDelay; + this.failoverDelay = failoverDelay; } private void run() { @@ -350,13 +387,14 @@ private List getConditions() { private boolean runSynchronizers() { // When runSynchronizers exists, no matter how it exits, the synchronizerStateManager will be closed. try { - Synchronizer synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); + Synchronizer synchronizer = buildNextAvailableSynchronizer(); // We want to continue running synchronizers for as long as any are available. while (synchronizer != null) { String synchronizerName = synchronizer.name(); logger.info("Synchronizer '{}' is starting.", synchronizerName); resetSynchronizerStatusDedupe(); + boolean parked = false; try { boolean running = true; @@ -437,11 +475,8 @@ private boolean runSynchronizers() { synchronizer.name(), status.getState() ); - sourceManager.blockCurrentSynchronizer(); - logger.warn( - "Synchronizer '{}' permanently failed and will not be used again until application restart.", - synchronizer.name() - ); + parkActiveSynchronizer(synchronizer.name(), status.getErrorInfo()); + parked = true; running = false; dataSourceUpdates.updateStatus( DataSourceStatusProvider.State.INTERRUPTED, @@ -486,25 +521,26 @@ private boolean runSynchronizers() { } } } - } catch (ExecutionException | InterruptedException | CancellationException e) { - dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, - new DataSourceStatusProvider.ErrorInfo( - DataSourceStatusProvider.ErrorKind.UNKNOWN, - 0, - e.toString(), - new Date().toInstant() - )); + } catch (ExecutionException | InterruptedException | RuntimeException e) { + DataSourceStatusProvider.ErrorInfo errorInfo = new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + e.toString(), + new Date().toInstant() + ); + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, errorInfo); Throwable root = e instanceof ExecutionException && e.getCause() != null ? e.getCause() : e; logger.error("Error running synchronizer '{}': {}", synchronizer.name(), root.getMessage() != null ? root.getMessage() : LogValues.exceptionSummary(root)); - // Move to the next synchronizer. + parkActiveSynchronizer(synchronizer.name(), errorInfo); + parked = true; + } + if (parked && !waitUnlessClosed(jittered(failoverDelay))) { + break; } // Get the next available synchronizer and set it active - synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); - } - if (!closed) { - logger.warn("No more synchronizers available."); + synchronizer = buildNextAvailableSynchronizer(); } } catch (Exception e) { // We are not expecting to encounter this situation, but if we do, then we should log it. @@ -525,6 +561,115 @@ private static String detailForError(DataSourceStatusProvider.ErrorInfo errorInf return errorInfo.toString(); } + /** + * Waits for the given duration, or until the data source is closed. + * + * @param duration the time to wait + * @return true if the full duration passed, or false if the data source was closed + */ + private boolean waitUnlessClosed(Duration duration) { + try { + closedSignal.get(Math.max(0, duration.toMillis()), TimeUnit.MILLISECONDS); + return false; + } catch (TimeoutException e) { + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (ExecutionException e) { + return false; + } + } + + /** + * Subtracts a random jitter of up to half of the given delay. + * + * @param delay the delay before jitter + * @return the delay after jitter + */ + private static Duration jittered(Duration delay) { + return delay.minusMillis(ThreadLocalRandom.current().nextLong(delay.toMillis() / 2 + 1)); + } + + /** + * Computes the park delay for a synchronizer. + * + * @param parkCount the park count of the synchronizer, including the current failure + * @return the park delay + */ + private Duration parkDelay(int parkCount) { + double delayMillis = Math.min( + parkInitialDelay.toMillis() * Math.pow(2, parkCount - 1), + parkMaxDelay.toMillis()); + return jittered(Duration.ofMillis((long) delayMillis)); + } + + /** + * Parks the active synchronizer after an unexpected failure. + * + * @param synchronizerName the name of the active synchronizer, or null if it failed to build + * @param errorInfo the error that caused the park + */ + private void parkActiveSynchronizer(String synchronizerName, DataSourceStatusProvider.ErrorInfo errorInfo) { + Duration parkDelay = parkDelay(sourceManager.incrementCurrentSynchronizerParkCount()); + sourceManager.parkCurrentSynchronizer(Instant.now().plus(parkDelay)); + logger.warn( + "{} is backing off for {}s after an unexpected error ({}).", + synchronizerName != null ? "Synchronizer '" + synchronizerName + "'" : "A synchronizer that failed to build", + parkDelay.getSeconds(), + detailForError(errorInfo) + ); + } + + /** + * Gets the next available synchronizer, builds it, and sets it active. While no synchronizer is available, this + * waits for the earliest unpark time. A synchronizer that fails to build is parked, and the next available + * synchronizer is tried after the failover delay. + * + * @return the active synchronizer, or null if the data source was closed + */ + private Synchronizer buildNextAvailableSynchronizer() { + while (!closed) { + Synchronizer synchronizer; + try { + synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); + } catch (RuntimeException e) { + DataSourceStatusProvider.ErrorInfo errorInfo = new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + e.toString(), + Instant.now() + ); + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, errorInfo); + parkActiveSynchronizer(null, errorInfo); + if (!waitUnlessClosed(jittered(failoverDelay))) { + return null; + } + continue; + } + if (synchronizer != null) { + return synchronizer; + } + + Instant earliestUnparkTime = sourceManager.getEarliestUnparkTime(); + Duration wait; + if (earliestUnparkTime == null) { + // This should never happen. + wait = parkInitialDelay; + logger.error("No synchronizer can be started; checking again in {}s.", wait.getSeconds()); + } else { + wait = Duration.between(Instant.now(), earliestUnparkTime); + logger.warn("All synchronizers are backing off for at least {}s.", Math.max(0, wait.getSeconds())); + } + if (!waitUnlessClosed(wait)) { + return null; + } + // Start the most-preferred synchronizer that is no longer parked. + sourceManager.resetSourceIndex(); + } + return null; + } + private void resetSynchronizerStatusDedupe() { lastLoggedSynchronizerDedupeName = null; lastLoggedSynchronizerDedupeStatus = null; @@ -564,6 +709,7 @@ public boolean isInitialized() { @Override public void close() { closed = true; + closedSignal.complete(null); // If there is an active source, we will shut it down, and that will result in the loop handling that source // exiting. // If we do not have an active source, then the loop will check isShutdown when attempting to set one. When diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java index 30f81b28..f7f078c8 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java @@ -6,6 +6,7 @@ import java.io.Closeable; import java.io.IOException; +import java.time.Duration; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -61,11 +62,11 @@ static abstract class TimedCondition implements Condition { /** * Timeout duration for the fallback operation. */ - protected final long timeoutSeconds; + protected final Duration timeout; - public TimedCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { + public TimedCondition(ScheduledExecutorService sharedExecutor, Duration timeout) { this.sharedExecutor = sharedExecutor; - this.timeoutSeconds = timeoutSeconds; + this.timeout = timeout; } @Override @@ -83,11 +84,11 @@ public void close() { static abstract class Factory implements ConditionFactory { protected final ScheduledExecutorService sharedExecutor; - protected final long timeoutSeconds; + protected final Duration timeout; - public Factory(ScheduledExecutorService sharedExecutor, long timeout) { + public Factory(ScheduledExecutorService sharedExecutor, Duration timeout) { this.sharedExecutor = sharedExecutor; - this.timeoutSeconds = timeout; + this.timeout = timeout; } } } @@ -101,13 +102,13 @@ public Factory(ScheduledExecutorService sharedExecutor, long timeout) { */ static class FallbackCondition extends TimedCondition { static class Factory extends TimedCondition.Factory { - public Factory(ScheduledExecutorService sharedExecutor, long timeout) { + public Factory(ScheduledExecutorService sharedExecutor, Duration timeout) { super(sharedExecutor, timeout); } @Override public Condition build() { - return new FallbackCondition(sharedExecutor, timeoutSeconds); + return new FallbackCondition(sharedExecutor, timeout); } @Override @@ -116,8 +117,8 @@ public ConditionType getType() { } } - public FallbackCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { - super(sharedExecutor, timeoutSeconds); + public FallbackCondition(ScheduledExecutorService sharedExecutor, Duration timeout) { + super(sharedExecutor, timeout); } @Override @@ -133,7 +134,7 @@ public void inform(FDv2SourceResult sourceResult) { timerFuture = sharedExecutor.schedule(() -> { resultFuture.complete(this); return null; - }, timeoutSeconds, TimeUnit.SECONDS); + }, timeout.toMillis(), TimeUnit.MILLISECONDS); } } } @@ -147,13 +148,13 @@ public ConditionType getType() { static class RecoveryCondition extends TimedCondition { static class Factory extends TimedCondition.Factory { - public Factory(ScheduledExecutorService sharedExecutor, long timeout) { + public Factory(ScheduledExecutorService sharedExecutor, Duration timeout) { super(sharedExecutor, timeout); } @Override public Condition build() { - return new RecoveryCondition(sharedExecutor, timeoutSeconds); + return new RecoveryCondition(sharedExecutor, timeout); } @Override @@ -162,12 +163,12 @@ public ConditionType getType() { } } - public RecoveryCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { - super(sharedExecutor, timeoutSeconds); + public RecoveryCondition(ScheduledExecutorService sharedExecutor, Duration timeout) { + super(sharedExecutor, timeout); timerFuture = sharedExecutor.schedule(() -> { resultFuture.complete(this); return null; - }, timeoutSeconds, TimeUnit.SECONDS); + }, timeout.toMillis(), TimeUnit.MILLISECONDS); } @Override diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java index c21bd5ba..cb429c60 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java @@ -1,6 +1,7 @@ package com.launchdarkly.sdk.server; import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.sdk.internal.http.FailureClass; import com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event; import com.launchdarkly.sdk.internal.fdv2.sources.FDv2ProtocolHandler; import com.launchdarkly.sdk.fdv2.Selector; @@ -65,7 +66,10 @@ protected CompletableFuture poll(Selector selector, boolean on e.toString(), new Date().toInstant() ); - return oneShot ? FDv2SourceResult.terminalError(info, fdv1Fallback) : FDv2SourceResult.interrupted(info, fdv1Fallback); + boolean recoverable = classifyTransportFailure(e) == FailureClass.NORMAL; + return oneShot || !recoverable + ? FDv2SourceResult.terminalError(info, fdv1Fallback) + : FDv2SourceResult.interrupted(info, fdv1Fallback); } else if (ex instanceof SerializationException) { SerializationException e = (SerializationException) ex; logger.error("Polling request received malformed data: {}", e.toString()); @@ -101,7 +105,7 @@ protected CompletableFuture poll(Selector selector, boolean on } if(!pollingResponse.isSuccess()) { int statusCode = pollingResponse.getStatusCode(); - boolean recoverable = statusCode <= 0 || isHttpErrorRecoverable(statusCode); + boolean recoverable = classifyHttpFailure(statusCode) == FailureClass.NORMAL; DataSourceStatusProvider.ErrorInfo errorInfo = DataSourceStatusProvider.ErrorInfo.fromHttpError(statusCode); logger.error("Polling request failed with HTTP error: {}", statusCode); // For a one-shot request all errors are terminal. diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java index 7d0f8292..5407a214 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java @@ -10,6 +10,11 @@ import java.util.concurrent.*; class PollingSynchronizerImpl extends PollingBase implements Synchronizer { + /** + * The number of consecutive successful polls that resets the retry state. + */ + private static final int RETRY_RESET_SUCCESSFUL_POLLS = 2; + private final CompletableFuture shutdownFuture = new CompletableFuture<>(); private final SelectorSource selectorSource; @@ -17,6 +22,12 @@ class PollingSynchronizerImpl extends PollingBase implements Synchronizer { private final IterableAsyncQueue resultQueue = new IterableAsyncQueue<>(); + /** + * Only {@link #doPoll()} reads or writes this. {@code scheduleAtFixedRate} never starts a run of + * {@code doPoll} before the previous run has finished. + */ + private int consecutiveSuccessfulPolls = 0; + public PollingSynchronizerImpl( FDv2Requestor requestor, LDLogger logger, @@ -47,10 +58,15 @@ private void doPoll() { boolean shouldShutdown = false; switch (res.getResultType()) { case CHANGE_SET: + consecutiveSuccessfulPolls++; + if (consecutiveSuccessfulPolls == RETRY_RESET_SUCCESSFUL_POLLS) { + res = res.withRetryReset(); + } break; case STATUS: switch (res.getStatus().getState()) { case INTERRUPTED: + consecutiveSuccessfulPolls = 0; break; case SHUTDOWN: // The base poller doesn't emit shutdown, we instead handle it at this level. diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java index dd093d9a..64ad3b63 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java @@ -4,6 +4,7 @@ import java.io.Closeable; import java.io.IOException; +import java.time.Instant; import java.util.List; /** @@ -88,6 +89,7 @@ public void fdv1Fallback() { */ private SynchronizerFactoryWithState getNextAvailableSynchronizer() { SynchronizerFactoryWithState factory = null; + Instant now = Instant.now(); int visited = 0; while(visited < synchronizers.size()) { @@ -101,7 +103,7 @@ private SynchronizerFactoryWithState getNextAvailableSynchronizer() { } SynchronizerFactoryWithState candidate = synchronizers.get(synchronizerIndex); - if (candidate.getState() == SynchronizerFactoryWithState.State.Available) { + if (candidate.isEligible(now)) { factory = candidate; break; } @@ -116,9 +118,6 @@ private SynchronizerFactoryWithState getNextAvailableSynchronizer() { *

* If shutdown has been initiated, returns null without building or setting a source. * Any previously active source will be closed before setting the new one. - *

- * The current synchronizer factory can be retrieved with {@link #blockCurrentSynchronizer()} - * or {@link #isCurrentSynchronizerFDv1Fallback()} to interact with it. * * @return the built synchronizer that is now active, or null if no more synchronizers are available or shutdown has been initiated */ @@ -176,11 +175,40 @@ private FDv2DataSource.DataSourceFactory getNextInitializer() { return initializers.get(initializerIndex); } - public void blockCurrentSynchronizer() { + /** + * Records an unexpected failure of the current synchronizer. + * + * @return the park count of the current synchronizer, including this failure + */ + public int incrementCurrentSynchronizerParkCount() { + synchronized (activeSourceLock) { + return currentSynchronizerFactory.incrementParkCount(); + } + } + + public void parkCurrentSynchronizer(Instant unparkTime) { synchronized (activeSourceLock) { - if (currentSynchronizerFactory != null) { - currentSynchronizerFactory.block(); + currentSynchronizerFactory.park(unparkTime); + } + } + + /** + * The earliest unpark time of the available synchronizers. + * + * @return the earliest unpark time, or null if no available synchronizer was ever parked + */ + public Instant getEarliestUnparkTime() { + synchronized (activeSourceLock) { + Instant earliest = null; + for (SynchronizerFactoryWithState factory : synchronizers) { + Instant unparkTime = factory.getUnparkTime(); + if (factory.getState() == SynchronizerFactoryWithState.State.Available + && unparkTime != null + && (earliest == null || unparkTime.isBefore(earliest))) { + earliest = unparkTime; + } } + return earliest; } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java index 5babb2d4..bc7fabd7 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java @@ -18,6 +18,8 @@ import com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event; import com.launchdarkly.sdk.internal.fdv2.sources.FDv2ProtocolHandler; import com.launchdarkly.sdk.fdv2.Selector; +import com.launchdarkly.sdk.internal.http.FailureClass; +import com.launchdarkly.sdk.internal.http.HttpErrors; import com.launchdarkly.sdk.internal.http.HttpHelpers; import com.launchdarkly.sdk.internal.http.HttpProperties; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; @@ -25,6 +27,7 @@ import com.launchdarkly.sdk.server.datasources.Synchronizer; import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; import com.launchdarkly.sdk.fdv2.ChangeSet; +import com.launchdarkly.sdk.fdv2.ChangeSetType; import com.launchdarkly.sdk.server.subsystems.DataStoreTypes; import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; @@ -36,17 +39,18 @@ import org.jetbrains.annotations.NotNull; import java.io.Reader; +import java.util.Collections; import java.util.Map; import java.net.URI; import java.time.Duration; import java.time.Instant; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import static com.launchdarkly.sdk.internal.http.HttpErrors.checkIfErrorIsRecoverableAndLog; - /** * Implementation of FDv2 streaming synchronizer. * Maintains a long-running streaming connection and queues results as they arrive. @@ -54,6 +58,11 @@ class StreamingSynchronizerImpl implements Synchronizer { private static final Duration DEAD_CONNECTION_INTERVAL = Duration.ofSeconds(300); + /** + * The time a stream must stay healthy, from its first valid result, before the retry state resets. + */ + static final Duration RETRY_RESET_THRESHOLD = Duration.ofSeconds(60); + private final HttpProperties httpProperties; private final SelectorSource selectorSource; final URI streamUri; @@ -71,16 +80,28 @@ class StreamingSynchronizerImpl implements Synchronizer { private final int threadPriority; private final DiagnosticStore diagnosticStore; + private final ScheduledExecutorService sharedExecutor; + private final Duration retryResetThreshold; private volatile long streamStarted = 0; + private final Object retryResetLock = new Object(); + + /** + * The timer that reports a retry reset for the current stream, or null if the current stream has + * not produced a valid result. Guarded by {@link #retryResetLock}. + */ + private ScheduledFuture retryResetFuture; + public StreamingSynchronizerImpl( HttpProperties httpProperties, URI baseUri, String requestPath, LDLogger logger, SelectorSource selectorSource, + ScheduledExecutorService sharedExecutor, String payloadFilter, Duration initialReconnectDelaySeconds, + Duration retryResetThreshold, int threadPriority, DiagnosticStore diagnosticStore ) { @@ -92,6 +113,8 @@ public StreamingSynchronizerImpl( this.initialReconnectDelay = initialReconnectDelaySeconds; this.threadPriority = threadPriority; this.diagnosticStore = diagnosticStore; + this.sharedExecutor = sharedExecutor; + this.retryResetThreshold = retryResetThreshold; // The stream will lazily start when `next` is called. } @@ -187,6 +210,7 @@ private Thread getRunThread() { logger.error("Stream thread ended with exception: {}", LogValues.exceptionSummary(e)); logger.debug(LogValues.exceptionTrace(e)); + cancelRetryResetTimer(); recordStreamInit(true); // Record failed init for unexpected thread exception DataSourceStatusProvider.ErrorInfo errorInfo = new DataSourceStatusProvider.ErrorInfo( @@ -238,6 +262,7 @@ public void close() { } } + cancelRetryResetTimer(); shutdownFuture.complete(FDv2SourceResult.shutdown()); } @@ -248,6 +273,50 @@ private void recordStreamInit(boolean failed) { } } + /** + * Starts the retry reset timer if the current stream does not have one yet. Call this for each + * valid result, so that the first valid result on a stream starts the timer. + */ + private void startRetryResetTimerIfNeeded() { + synchronized (retryResetLock) { + if (retryResetFuture != null) { + return; + } + retryResetFuture = sharedExecutor.schedule( + this::reportRetryReset, + retryResetThreshold.toMillis(), + TimeUnit.MILLISECONDS); + } + } + + /** + * Stops the retry reset timer of the current stream. Call this when the stream ends. + */ + private void cancelRetryResetTimer() { + synchronized (retryResetLock) { + if (retryResetFuture != null) { + retryResetFuture.cancel(false); + retryResetFuture = null; + } + } + } + + private void reportRetryReset() { + synchronized (retryResetLock) { + // The timer can start to run before a cancel. Do not report a reset for a stream that ended. + if (retryResetFuture == null) { + return; + } + ChangeSet>>> noChanges = new ChangeSet<>( + ChangeSetType.None, + Selector.EMPTY, + Collections.emptyList(), + /* environmentId= */ null, + /* shouldPersist= */ true); + resultQueue.put(FDv2SourceResult.changeSet(noChanges, /* fdv1Fallback= */ false).withRetryReset()); + } + } + private boolean handleEvent(StreamEvent event) { if (event instanceof MessageEvent) { handleMessage((MessageEvent) event); @@ -296,6 +365,7 @@ private void handleMessage(MessageEvent event) { // Reset to 0 after successful init to prevent duplicate recordings until next restart streamStarted = 0; result = FDv2SourceResult.changeSet(converted, getFallback(event)); + startRetryResetTimerIfNeeded(); } catch (Exception e) { logger.error("Failed to convert FDv2 changeset: {}", LogValues.exceptionSummary(e)); logger.debug(LogValues.exceptionTrace(e)); @@ -387,19 +457,19 @@ private boolean handleError(StreamException e) { return true; } + cancelRetryResetTimer(); recordStreamInit(true); if (e instanceof StreamHttpErrorException) { int status = ((StreamHttpErrorException) e).getCode(); DataSourceStatusProvider.ErrorInfo errorInfo = DataSourceStatusProvider.ErrorInfo.fromHttpError(status); - boolean recoverable = checkIfErrorIsRecoverableAndLog(logger, - "HTTP error " + status, - "in FDv2 streaming connection", + FailureClass failureClass = HttpErrors.classifyAndLogHttpFailure(logger, status, + "in FDv2 streaming connection", "will retry"); - if (!recoverable) { + if (failureClass == FailureClass.UNEXPECTED) { shutdownFuture.complete(FDv2SourceResult.terminalError(errorInfo, getFallback(e))); return false; } else { @@ -419,6 +489,10 @@ private boolean handleError(StreamException e) { e.toString(), Instant.now() ); + if (HttpErrors.classifyTransportFailure(e) == FailureClass.UNEXPECTED) { + shutdownFuture.complete(FDv2SourceResult.terminalError(errorInfo, getFallback(e))); + return false; + } resultQueue.put(FDv2SourceResult.interrupted(errorInfo, getFallback(e))); streamStarted = System.currentTimeMillis(); return true; // allow reconnect @@ -426,6 +500,7 @@ private boolean handleError(StreamException e) { private void restartStream(boolean failed) { Objects.requireNonNull(eventSource, "eventSource must not be null"); + cancelRetryResetTimer(); recordStreamInit(failed); streamStarted = System.currentTimeMillis(); eventSource.interrupt(); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java index 31f02cc1..923ad2ff 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java @@ -2,6 +2,8 @@ import com.launchdarkly.sdk.server.datasources.Synchronizer; +import java.time.Instant; + class SynchronizerFactoryWithState { public enum State { /** @@ -21,6 +23,16 @@ public enum State { private boolean isFDv1Fallback = false; + /** + * The number of unexpected failures since the synchronizer last reported a retry reset. + */ + private int parkCount = 0; + + /** + * The time at which the synchronizer stops being parked, or null if it is not parked. + */ + private Instant unparkTime = null; + public SynchronizerFactoryWithState(FDv2DataSource.DataSourceFactory factory) { this.factory = factory; @@ -51,4 +63,40 @@ public void unblock() { public boolean isFDv1Fallback() { return isFDv1Fallback; } + + /** + * Records an unexpected failure. + * + * @return the park count, including this failure + */ + public int incrementParkCount() { + return ++parkCount; + } + + public void resetParkCount() { + parkCount = 0; + } + + public void park(Instant unparkTime) { + this.unparkTime = unparkTime; + } + + /** + * The time at which the synchronizer stops being parked. + * + * @return the unpark time, or null if the synchronizer was never parked + */ + public Instant getUnparkTime() { + return unparkTime; + } + + /** + * True if the synchronizer is available and is not parked at the given time. + * + * @param now the current time + * @return true if the synchronizer can be started + */ + public boolean isEligible(Instant now) { + return state == State.Available && (unparkTime == null || !now.isBefore(unparkTime)); + } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java index 32b6c72b..a2b4ef99 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java @@ -72,18 +72,21 @@ public String getReason() { private final Status status; private final SourceResultType resultType; private final boolean fdv1Fallback; + private final boolean retryReset; private final Function completionCallback; private FDv2SourceResult( ChangeSet>>> changeSet, Status status, SourceResultType resultType, boolean fdv1Fallback, + boolean retryReset, Function completionCallback ) { this.changeSet = changeSet; this.status = status; this.resultType = resultType; this.fdv1Fallback = fdv1Fallback; + this.retryReset = retryReset; this.completionCallback = completionCallback; } @@ -93,10 +96,11 @@ public static FDv2SourceResult interrupted(DataSourceStatusProvider.ErrorInfo er public static FDv2SourceResult interrupted(DataSourceStatusProvider.ErrorInfo errorInfo, boolean fdv1Fallback, Function completionCallback) { return new FDv2SourceResult( - null, + /* changeSet= */ null, Status.interrupted(errorInfo), SourceResultType.STATUS, fdv1Fallback, + /* retryReset= */ false, completionCallback); } @@ -105,10 +109,12 @@ public static FDv2SourceResult shutdown() { } public static FDv2SourceResult shutdown(Function completionCallback) { - return new FDv2SourceResult(null, + return new FDv2SourceResult( + /* changeSet= */ null, Status.shutdown(), SourceResultType.STATUS, - false, + /* fdv1Fallback= */ false, + /* retryReset= */ false, completionCallback); } @@ -117,10 +123,12 @@ public static FDv2SourceResult terminalError(DataSourceStatusProvider.ErrorInfo } public static FDv2SourceResult terminalError(DataSourceStatusProvider.ErrorInfo errorInfo, boolean fdv1Fallback, Function completionCallback) { - return new FDv2SourceResult(null, + return new FDv2SourceResult( + /* changeSet= */ null, Status.terminalError(errorInfo), SourceResultType.STATUS, fdv1Fallback, + /* retryReset= */ false, completionCallback); } @@ -131,9 +139,10 @@ public static FDv2SourceResult changeSet(ChangeSet>>> changeSet, boolean fdv1Fallback, Function completionCallback) { return new FDv2SourceResult( changeSet, - null, + /* status= */ null, SourceResultType.CHANGE_SET, fdv1Fallback, + /* retryReset= */ false, completionCallback); } @@ -143,10 +152,11 @@ public static FDv2SourceResult goodbye(String reason, boolean fdv1Fallback) { public static FDv2SourceResult goodbye(String reason, boolean fdv1Fallback, Function completionCallback) { return new FDv2SourceResult( - null, + /* changeSet= */ null, Status.goodbye(reason), SourceResultType.STATUS, fdv1Fallback, + /* retryReset= */ false, completionCallback); } @@ -166,6 +176,30 @@ public boolean isFdv1Fallback() { return fdv1Fallback; } + /** + * True if the source met its retry reset threshold at the time of this result. + * + * @return true if the retry state was reset + */ + public boolean isRetryReset() { + return retryReset; + } + + /** + * Creates a copy of this result that reports a retry reset. + * + * @return a new FDv2SourceResult with {@link #isRetryReset()} set to true + */ + public FDv2SourceResult withRetryReset() { + return new FDv2SourceResult( + changeSet, + status, + resultType, + fdv1Fallback, + /* retryReset= */ true, + completionCallback); + } + /** * Creates a new result wrapping this one with an additional completion callback. *

@@ -183,7 +217,13 @@ public FDv2SourceResult withCompletion(Function newCallback) { } return null; }; - return new FDv2SourceResult(changeSet, status, resultType, fdv1Fallback, combinedCallback); + return new FDv2SourceResult( + changeSet, + status, + resultType, + fdv1Fallback, + retryReset, + combinedCallback); } @Override diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java index ce81c226..1eeb5053 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java @@ -4,6 +4,7 @@ import com.launchdarkly.sdk.fdv2.SourceResultType; import com.launchdarkly.sdk.fdv2.SourceSignal; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; +import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; import com.launchdarkly.sdk.server.subsystems.DataSource; import com.launchdarkly.sdk.server.DataStoreTestTypes.DataBuilder; import com.launchdarkly.sdk.server.subsystems.DataSourceUpdateSink; @@ -361,4 +362,40 @@ public void environmentIdIsRetainedForSubsequentChangeSets() throws Exception { adapter.close(); } + + /** + * Test that an interrupted status with an unexpected HTTP error is reported as terminal, and an + * interrupted status with a normal HTTP error is reported as interrupted. + */ + @Test + public void interruptedStatusIsTerminalOnlyForUnexpectedHttpErrors() throws Exception { + AtomicReference capturedSink = new AtomicReference<>(); + + DataSourceSynchronizerAdapter adapter = new DataSourceSynchronizerAdapter(sink -> { + capturedSink.set(sink); + return new MockDataSource(new CountDownLatch(1), null); + }); + resourcesToClose.add(adapter); + + // Report a 503 as interrupted, as the FDv1 data sources do. + CompletableFuture normalFuture = adapter.next(); + capturedSink.get().updateStatus(DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.ErrorInfo.fromHttpError(503)); + FDv2SourceResult normalResult = normalFuture.get(2, TimeUnit.SECONDS); + + // The 503 stays interrupted. + assertEquals(SourceSignal.INTERRUPTED, normalResult.getStatus().getState()); + + // Report a 401 as interrupted, as the FDv1 data sources do. + CompletableFuture unexpectedFuture = adapter.next(); + capturedSink.get().updateStatus(DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.ErrorInfo.fromHttpError(401)); + FDv2SourceResult unexpectedResult = unexpectedFuture.get(2, TimeUnit.SECONDS); + + // The 401 becomes terminal. + assertEquals(SourceSignal.TERMINAL_ERROR, unexpectedResult.getStatus().getState()); + assertEquals(401, unexpectedResult.getStatus().getErrorInfo().getStatusCode()); + + adapter.close(); + } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java index cae52f8b..e3e996a9 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java @@ -11,6 +11,7 @@ import org.junit.Before; import org.junit.Test; +import java.time.Duration; import java.time.Instant; import java.util.Collections; import java.util.concurrent.CompletableFuture; @@ -59,7 +60,7 @@ public void tearDown() { */ @Test public void getFutureReturnsDistinctInstancesPerCall() { - Condition fallback = new FallbackCondition(executor, 60); + Condition fallback = new FallbackCondition(executor, Duration.ofSeconds(60)); try (FDv2DataSource.Conditions conditions = new FDv2DataSource.Conditions(Collections.singletonList(fallback))) { CompletableFuture f1 = conditions.getFuture(); @@ -96,7 +97,7 @@ public void getFutureReturnsDistinctInstancesEvenWithNoConditions() { @Test public void allFreshFuturesCompleteWhenAggregateFires() throws Exception { // 0-second timeout -> fires on first INTERRUPTED inform. - Condition fallback = new FallbackCondition(executor, 0); + Condition fallback = new FallbackCondition(executor, Duration.ofSeconds(0)); try (FDv2DataSource.Conditions conditions = new FDv2DataSource.Conditions(Collections.singletonList(fallback))) { CompletableFuture f1 = conditions.getFuture(); @@ -162,7 +163,7 @@ public void getFutureFailsExceptionallyWhenAggregateFailsExceptionally() public void getFutureAfterAggregateFiresReturnsCompletedFuture() throws Exception { // RecoveryCondition arms its timer in the constructor and fires after // the configured timeout. With timeout=0 it fires near-immediately. - Condition recovery = new RecoveryCondition(executor, 0); + Condition recovery = new RecoveryCondition(executor, Duration.ofSeconds(0)); try (FDv2DataSource.Conditions conditions = new FDv2DataSource.Conditions(Collections.singletonList(recovery))) { // Drain a future to confirm the aggregate has fired. diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java index 94bcf3bf..a234069d 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java @@ -17,6 +17,7 @@ import org.junit.Test; +import java.time.Duration; import java.time.Instant; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; @@ -52,7 +53,7 @@ private ChangeSet>>> mak @Test public void executeReturnsCompletableFuture() { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 120); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(120)); CompletableFuture result = condition.execute(); @@ -62,7 +63,7 @@ public void executeReturnsCompletableFuture() { @Test public void getTypeReturnsFallback() { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 120); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(120)); assertEquals(Condition.ConditionType.FALLBACK, condition.getType()); } @@ -70,7 +71,7 @@ public void getTypeReturnsFallback() { @Test public void interruptedStatusStartsTimerThatCompletesResultFuture() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); assertFalse(resultFuture.isDone()); @@ -97,7 +98,7 @@ public void interruptedStatusStartsTimerThatCompletesResultFuture() throws Excep @Test public void changeSetCancelsActiveTimer() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -123,7 +124,7 @@ public void changeSetCancelsActiveTimer() throws Exception { @Test public void changeSetWithoutActiveTimerDoesNothing() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -141,7 +142,7 @@ public void changeSetWithoutActiveTimerDoesNothing() throws Exception { @Test public void multipleInterruptedStatusesDoNotStartMultipleTimers() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 2); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(2)); CompletableFuture resultFuture = condition.execute(); @@ -181,7 +182,7 @@ public void multipleInterruptedStatusesDoNotStartMultipleTimers() throws Excepti @Test public void terminalErrorStatusDoesNotStartTimer() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -203,7 +204,7 @@ public void terminalErrorStatusDoesNotStartTimer() throws Exception { @Test public void shutdownStatusDoesNotStartTimer() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -220,7 +221,7 @@ public void shutdownStatusDoesNotStartTimer() throws Exception { @Test public void goodbyeStatusDoesNotStartTimer() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -237,7 +238,7 @@ public void goodbyeStatusDoesNotStartTimer() throws Exception { @Test public void closeCancelsActiveTimer() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -262,7 +263,7 @@ public void closeCancelsActiveTimer() throws Exception { @Test public void closeWithoutActiveTimerDoesNotFail() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 120); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(120)); // Close without starting a timer condition.close(); @@ -273,7 +274,7 @@ public void closeWithoutActiveTimerDoesNotFail() throws Exception { @Test public void timerCanBeStartedAfterBeingCancelled() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -307,7 +308,7 @@ public void timerCanBeStartedAfterBeingCancelled() throws Exception { @Test public void changeSetAfterTimerFiresDoesNotAffectCompletedFuture() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -335,7 +336,7 @@ public void changeSetAfterTimerFiresDoesNotAffectCompletedFuture() throws Except @Test public void factoryCreatesFallbackCondition() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition.Factory factory = new FallbackCondition.Factory(executor, 1); + FallbackCondition.Factory factory = new FallbackCondition.Factory(executor, Duration.ofSeconds(1)); FallbackCondition condition = (FallbackCondition) factory.build(); @@ -358,7 +359,7 @@ public void factoryCreatesFallbackCondition() throws Exception { @Test public void executeReturnsTheSameFutureOnMultipleCalls() { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 120); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(120)); CompletableFuture first = condition.execute(); CompletableFuture second = condition.execute(); @@ -369,7 +370,7 @@ public void executeReturnsTheSameFutureOnMultipleCalls() { @Test public void changeSetDuringTimerExecutionCancelsTimer() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -398,7 +399,7 @@ public void changeSetDuringTimerExecutionCancelsTimer() throws Exception { @Test public void multipleChangeSetCallsWithActiveTimerAreHandled() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -426,7 +427,7 @@ public void multipleChangeSetCallsWithActiveTimerAreHandled() throws Exception { @Test public void closeCanBeCalledMultipleTimes() throws Exception { executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); + FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); // Start timer condition.inform( diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java index b3878bad..0d6765f3 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java @@ -15,6 +15,7 @@ import org.junit.After; import org.junit.Test; +import java.time.Duration; import java.time.Instant; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -52,7 +53,7 @@ private ChangeSet>>> mak @Test public void getTypeReturnsRecovery() { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 120); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(120)); assertEquals(Condition.ConditionType.RECOVERY, condition.getType()); } @@ -60,7 +61,7 @@ public void getTypeReturnsRecovery() { @Test public void timerStartsImmediatelyAndCompletesResultFuture() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -78,7 +79,7 @@ public void timerStartsImmediatelyAndCompletesResultFuture() throws Exception { @Test public void closeCancelsActiveTimer() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -95,7 +96,7 @@ public void closeCancelsActiveTimer() throws Exception { @Test public void closeAfterTimerFiresDoesNotCauseError() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -113,7 +114,7 @@ public void closeAfterTimerFiresDoesNotCauseError() throws Exception { @Test public void closeCanBeCalledMultipleTimes() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); // Close multiple times before timer fires condition.close(); @@ -126,7 +127,7 @@ public void closeCanBeCalledMultipleTimes() throws Exception { @Test public void informWithChangeSetDoesNothing() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -143,7 +144,7 @@ public void informWithChangeSetDoesNothing() throws Exception { @Test public void informWithInterruptedStatusDoesNothing() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -164,7 +165,7 @@ public void informWithInterruptedStatusDoesNothing() throws Exception { @Test public void informWithTerminalErrorStatusDoesNothing() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -185,7 +186,7 @@ public void informWithTerminalErrorStatusDoesNothing() throws Exception { @Test public void informWithShutdownStatusDoesNothing() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -201,7 +202,7 @@ public void informWithShutdownStatusDoesNothing() throws Exception { @Test public void informWithGoodbyeStatusDoesNothing() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -217,7 +218,7 @@ public void informWithGoodbyeStatusDoesNothing() throws Exception { @Test public void informWithNullDoesNothing() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -233,7 +234,7 @@ public void informWithNullDoesNothing() throws Exception { @Test public void multipleInformCallsDoNotAffectTimer() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); CompletableFuture resultFuture = condition.execute(); @@ -257,7 +258,7 @@ public void multipleInformCallsDoNotAffectTimer() throws Exception { @Test public void factoryCreatesRecoveryCondition() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition.Factory factory = new RecoveryCondition.Factory(executor, 1); + RecoveryCondition.Factory factory = new RecoveryCondition.Factory(executor, Duration.ofSeconds(1)); RecoveryCondition condition = (RecoveryCondition) factory.build(); @@ -273,7 +274,7 @@ public void factoryCreatesRecoveryCondition() throws Exception { @Test public void factoryGetTypeReturnsRecovery() { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition.Factory factory = new RecoveryCondition.Factory(executor, 1); + RecoveryCondition.Factory factory = new RecoveryCondition.Factory(executor, Duration.ofSeconds(1)); assertEquals(Condition.ConditionType.RECOVERY, factory.getType()); } @@ -281,7 +282,7 @@ public void factoryGetTypeReturnsRecovery() { @Test public void executeReturnsTheSameFutureOnMultipleCalls() { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 120); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(120)); CompletableFuture first = condition.execute(); CompletableFuture second = condition.execute(); @@ -294,7 +295,7 @@ public void timerStartsImmediatelyOnConstruction() throws Exception { executor = Executors.newScheduledThreadPool(1); // Create condition with very short timeout - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); // Get the future CompletableFuture resultFuture = condition.execute(); @@ -312,7 +313,7 @@ public void timerStartsImmediatelyOnConstruction() throws Exception { @Test public void closeBeforeExecuteDoesNotPreventFutureAccess() throws Exception { executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); + RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); // Close immediately condition.close(); diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index d2015819..39b55346 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -25,8 +25,10 @@ import org.junit.After; import org.junit.Test; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -115,7 +117,8 @@ public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -163,7 +166,8 @@ public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() thro } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -212,7 +216,8 @@ public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() th ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -257,7 +262,8 @@ public void allInitializersFailSwitchesToSynchronizers() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -299,7 +305,8 @@ public void allThreeInitializersFailWithNoSynchronizers() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -339,7 +346,8 @@ public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -374,7 +382,8 @@ public void noInitializersOneSynchronizerIsWellBehaved() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -408,7 +417,8 @@ public void oneInitializerOneSynchronizerIsWellBehaved() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -429,7 +439,8 @@ public void noInitializersAndNoSynchronizersIsWellBehaved() throws Exception { ImmutableList> initializers = ImmutableList.of(); ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -468,7 +479,8 @@ public void errorWithFDv1FallbackTriggersFallback() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -512,7 +524,8 @@ public void fallbackAndRecoveryTasksWellBehaved() throws Exception { ); // Use short timeouts for testing - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -565,7 +578,8 @@ public void canDisposeWhenSynchronizersFallingBack() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -580,11 +594,11 @@ public void canDisposeWhenSynchronizersFallingBack() throws Exception { } // ============================================================================ - // Source Blocking + // Source Parking // ============================================================================ @Test - public void terminalErrorBlocksSynchronizer() throws Exception { + public void terminalErrorFailsOverToNextSynchronizer() throws Exception { executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); @@ -612,7 +626,8 @@ public void terminalErrorBlocksSynchronizer() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -625,13 +640,13 @@ public void terminalErrorBlocksSynchronizer() throws Exception { Integer secondCall = synchronizerCallQueue.poll(2, TimeUnit.SECONDS); assertNotNull("First synchronizer should be called", firstCall); - assertNotNull("Second synchronizer should be called after first is blocked", secondCall); + assertNotNull("Second synchronizer should be called after first is parked", secondCall); assertEquals(Integer.valueOf(1), firstCall); assertEquals(Integer.valueOf(2), secondCall); // Expected status sequence: // 1. VALID when first synchronizer sends initial changeset - // 2. Terminal error from first synchronizer (blocks it) + // 2. Terminal error from first synchronizer (parks it) // 3. VALID when second synchronizer sends changeset List statuses = sink.awaitStatuses(3, 2, TimeUnit.SECONDS); assertEquals("Should receive 3 status updates", 3, statuses.size()); @@ -645,47 +660,189 @@ public void terminalErrorBlocksSynchronizer() throws Exception { } @Test - public void allThreeSynchronizersFailReportsExhaustion() throws Exception { + public void terminalErrorParksSynchronizerAndRestartsItAfterUnparkTime() throws Exception { executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); - // All synchronizers send terminal errors + // The first instance fails with a terminal error, and the second instance delivers data. + BlockingQueue firstInstanceResults = new LinkedBlockingQueue<>(); + firstInstanceResults.add(makeTerminalErrorResult()); + BlockingQueue secondInstanceResults = new LinkedBlockingQueue<>(); + secondInstanceResults.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + AtomicInteger buildCount = new AtomicInteger(); + + ImmutableList> synchronizers = ImmutableList.of( + () -> buildCount.incrementAndGet() == 1 + ? new MockQueuedSynchronizer(firstInstanceResults) + : new MockQueuedSynchronizer(secondInstanceResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMillis(100), Duration.ofSeconds(1), Duration.ofMillis(10)); + resourcesToClose.add(dataSource); + + // Start the data system with its only synchronizer. + dataSource.start().get(5, TimeUnit.SECONDS); + + // The parked synchronizer is built again after its unpark time, and its data initializes the data system. + assertEquals(2, buildCount.get()); + assertTrue(dataSource.isInitialized()); + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); + } + + @Test + public void failedNextParksSynchronizer() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + + // The first instance fails from next(), and the second instance delivers data. + ImmutableList> synchronizers = ImmutableList.of( + () -> buildCount.incrementAndGet() == 1 + ? new MockSynchronizer(() -> { + throw new RuntimeException("next-boom"); + }, "throwing-sync") + : new MockSynchronizer( + CompletableFuture.completedFuture(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + "throwing-sync") + ); + + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, testLogger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMillis(100), Duration.ofSeconds(1), Duration.ofMillis(10)); + resourcesToClose.add(dataSource); + + // Start the data system with its only synchronizer. + dataSource.start().get(5, TimeUnit.SECONDS); + + // The failing synchronizer is parked, and is built again after its unpark time. + assertEquals(2, buildCount.get()); + assertTrue(logTextContains(logCapture, LDLogLevel.WARN, "Synchronizer 'throwing-sync' is backing off for ")); + } + + @Test + public void exceptionThrownFromNextParksSynchronizer() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + if (buildCount.incrementAndGet() == 1) { + // The first instance throws from next() instead of returning a future. + return new Synchronizer() { + @Override + public CompletableFuture next() { + throw new RuntimeException("next-boom"); + } + + @Override + public void close() { + } + }; + } + // The second instance delivers data. + return new MockSynchronizer( + CompletableFuture.completedFuture(FDv2SourceResult.changeSet(makeChangeSet(true), false))); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMillis(100), Duration.ofSeconds(1), Duration.ofMillis(10)); + resourcesToClose.add(dataSource); + + // Start the data system with its only synchronizer. + dataSource.start().get(5, TimeUnit.SECONDS); + + // The throwing synchronizer is parked, and is built again after its unpark time. + assertEquals(2, buildCount.get()); + assertTrue(dataSource.isInitialized()); + } + + @Test + public void constructionFailureParksSynchronizer() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + ImmutableList> synchronizers = ImmutableList.of( () -> { + if (buildCount.incrementAndGet() == 1) { + // The first build fails. + throw new RuntimeException("build-boom"); + } + // The second build delivers data. + return new MockSynchronizer( + CompletableFuture.completedFuture(FDv2SourceResult.changeSet(makeChangeSet(true), false))); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMillis(100), Duration.ofSeconds(1), Duration.ofMillis(10)); + resourcesToClose.add(dataSource); + + // Start the data system with its only synchronizer. + dataSource.start().get(5, TimeUnit.SECONDS); + + // The synchronizer that failed to build is parked, and is built again after its unpark time. + assertEquals(2, buildCount.get()); + assertTrue(dataSource.isInitialized()); + } + + @Test + public void allSynchronizersParkedRestartsAfterEarliestUnparkTime() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // Every instance of every synchronizer fails with a terminal error. + BlockingQueue buildOrder = new LinkedBlockingQueue<>(); + ImmutableList> synchronizers = ImmutableList.of( + () -> { + buildOrder.offer(1); BlockingQueue results = new LinkedBlockingQueue<>(); results.add(makeTerminalErrorResult()); return new MockQueuedSynchronizer(results); }, () -> { + buildOrder.offer(2); BlockingQueue results = new LinkedBlockingQueue<>(); results.add(makeTerminalErrorResult()); return new MockQueuedSynchronizer(results); }, () -> { + buildOrder.offer(3); BlockingQueue results = new LinkedBlockingQueue<>(); results.add(makeTerminalErrorResult()); return new MockQueuedSynchronizer(results); } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMillis(100), Duration.ofSeconds(1), Duration.ofMillis(10)); resourcesToClose.add(dataSource); - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Expected status sequence: 3 terminal errors (one per synchronizer) → OFF when all exhausted - // Terminal errors might show as INTERRUPTED status - List statuses = sink.awaitStatuses(4, 2, TimeUnit.SECONDS); - assertEquals("Should receive 4 status updates", 4, statuses.size()); - // Last status should be OFF - assertEquals(DataSourceStatusProvider.State.OFF, statuses.get(3)); - - assertFalse(dataSource.isInitialized()); - assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); - assertNotNull(sink.getLastError()); + // Start the data system, and wait for each synchronizer to be built and then for one more build. + dataSource.start(); + Integer firstBuild = buildOrder.poll(2, TimeUnit.SECONDS); + Integer secondBuild = buildOrder.poll(2, TimeUnit.SECONDS); + Integer thirdBuild = buildOrder.poll(2, TimeUnit.SECONDS); + Integer buildAfterAllParked = buildOrder.poll(2, TimeUnit.SECONDS); + + // The synchronizers fail over in order, and a parked synchronizer starts again after all of them are parked. + assertEquals(Integer.valueOf(1), firstBuild); + assertEquals(Integer.valueOf(2), secondBuild); + assertEquals(Integer.valueOf(3), thirdBuild); + assertNotNull(buildAfterAllParked); + + // Each park reports INTERRUPTED. + List statuses = sink.awaitStatuses(3, 2, TimeUnit.SECONDS); + assertEquals(Arrays.asList( + DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.State.INTERRUPTED), statuses); } // ============================================================================ @@ -718,7 +875,8 @@ public void disabledDataSourceCannotTriggerActions() throws Exception { () -> new MockQueuedSynchronizer(secondSyncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -764,7 +922,8 @@ public void disposeCompletesStartFuture() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -791,7 +950,8 @@ public void noSourcesProvidedCompletesImmediately() throws Exception{ ImmutableList> initializers = ImmutableList.of(); ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -831,7 +991,8 @@ public void startFutureCompletesExactlyOnce() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -854,7 +1015,8 @@ public void concurrentCloseAndStartHandledSafely() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); Future startFuture = dataSource.start(); @@ -878,7 +1040,8 @@ public void multipleStartCallsEventuallyComplete() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture1 = dataSource.start(); @@ -906,7 +1069,8 @@ public void isInitializedThreadSafe() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); dataSource.start(); @@ -941,7 +1105,8 @@ public void dataSourceUpdatesApplyThreadSafe() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -977,7 +1142,8 @@ public void initializerThrowsExecutionException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1009,7 +1175,8 @@ public void initializerThrowsInterruptedException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1039,7 +1206,8 @@ public void initializerThrowsCancellationException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1068,7 +1236,8 @@ public void synchronizerNextThrowsExecutionException() throws Exception { () -> new MockSynchronizer(goodFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1100,7 +1269,8 @@ public void synchronizerNextThrowsInterruptedException() throws Exception { () -> new MockSynchronizer(goodFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1130,7 +1300,8 @@ public void synchronizerNextThrowsCancellationException() throws Exception { () -> new MockSynchronizer(goodFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1152,7 +1323,8 @@ public void closeWithoutStartDoesNotHang() { ImmutableList> initializers = ImmutableList.of(); ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); dataSource.close(); @@ -1174,7 +1346,8 @@ public void closeAfterInitializersCompletesImmediately() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1205,7 +1378,8 @@ public void close() { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1230,7 +1404,8 @@ public void multipleCloseCallsAreIdempotent() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1256,7 +1431,8 @@ public void closingDataSourceDuringInitializationReportsOffWithoutErrors() throw ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1296,7 +1472,8 @@ public void dataSourceClosedDuringSynchronizationReportsOffWithoutError() throws } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1333,7 +1510,8 @@ public void closeInterruptsConditionWaiting() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1371,7 +1549,8 @@ public CompletableFuture run() { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); dataSource.close(); Future startFuture = dataSource.start(); @@ -1408,7 +1587,8 @@ public void close() { () -> new MockQueuedSynchronizer(secondSyncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1442,7 +1622,8 @@ public void close() { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1474,7 +1655,8 @@ public void setActiveSourceOnInitializerChecksShutdown() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); Future startFuture = dataSource.start(); @@ -1498,13 +1680,13 @@ public void setActiveSourceOnInitializerChecksShutdown() throws Exception { // ============================================================================ @Test - public void blockedSynchronizerSkippedInRotation() throws Exception { + public void parkedSynchronizerSkippedInRotation() throws Exception { executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); - // First: terminal error (blocked) + // First: terminal error (parked) BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); firstSyncResults.add(makeTerminalErrorResult()); @@ -1535,45 +1717,17 @@ public void blockedSynchronizerSkippedInRotation() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); - assertEquals(1, firstCallCount.get()); // Called once, then blocked + assertEquals(1, firstCallCount.get()); // Called once, then parked assertTrue(secondCallCount.get() >= 1); // Called } - @Test - public void allSynchronizersBlockedReturnsNullAndExits() throws Exception { - executor = Executors.newScheduledThreadPool(2); - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - ImmutableList> synchronizers = ImmutableList.of( - () -> { - BlockingQueue results = new LinkedBlockingQueue<>(); - results.add(makeTerminalErrorResult()); - return new MockQueuedSynchronizer(results); - }, - () -> { - BlockingQueue results = new LinkedBlockingQueue<>(); - results.add(makeTerminalErrorResult()); - return new MockQueuedSynchronizer(results); - } - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - assertFalse(dataSource.isInitialized()); - } - @Test public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { executor = Executors.newScheduledThreadPool(2); @@ -1605,7 +1759,8 @@ public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { ); // Short recovery timeout - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1645,7 +1800,8 @@ public void fallbackMovesToNextSynchronizer() throws Exception { ); // Short fallback timeout - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(1), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1675,7 +1831,8 @@ public void conditionsClosedAfterSynchronizerLoop() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1702,7 +1859,8 @@ public void conditionsInformedOfAllResults() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 10, 20); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(10), Duration.ofSeconds(20), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1730,7 +1888,8 @@ public void conditionsClosedOnException() throws Exception { )) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1755,7 +1914,8 @@ public void primeSynchronizerHasNoRecoveryCondition() throws Exception { () -> new MockQueuedSynchronizer(new LinkedBlockingQueue<>()) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1786,7 +1946,8 @@ public void nonPrimeSynchronizerHasBothConditions() throws Exception { () -> new MockQueuedSynchronizer(secondSyncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1810,7 +1971,8 @@ public void singleSynchronizerHasNoConditions() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1835,7 +1997,8 @@ public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1865,7 +2028,8 @@ public void changeSetAppliedToDataSourceUpdates() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1891,7 +2055,8 @@ public void multipleChangeSetsAppliedInOrder() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1924,7 +2089,8 @@ public void selectorNonEmptyCompletesInitialization() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, ImmutableList.of(), null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, ImmutableList.of(), null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1953,7 +2119,8 @@ public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() thro ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1986,7 +2153,8 @@ public void synchronizerChangeSetAlwaysCompletesStartFuture() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2015,7 +2183,8 @@ public void goodbyeStatusHandledGracefully() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2049,7 +2218,8 @@ public void shutdownStatusExitsImmediately() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2076,7 +2246,8 @@ public void fdv1FallbackFlagHonored() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2109,7 +2280,8 @@ public void emptyInitializerListSkipsToSynchronizers() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2137,7 +2309,8 @@ public void startedFlagPreventsMultipleRuns() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture1 = dataSource.start(); @@ -2167,7 +2340,8 @@ public void startBeforeRunCompletesAllComplete() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); resourcesToClose.add(dataSource); // Call start multiple times before completion @@ -2206,8 +2380,11 @@ public void statusTransitionsToValidAfterInitialization() throws Exception { Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2252,27 +2429,23 @@ public void statusIncludesErrorInfoOnFailure() throws Exception { Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Should receive INTERRUPTED first (from terminal error), then OFF (from exhausted synchronizers) - DataSourceStatusProvider.State firstStatus = sink.awaitStatus(2, TimeUnit.SECONDS); - assertNotNull("Should receive first status update", firstStatus); - assertEquals(DataSourceStatusProvider.State.INTERRUPTED, firstStatus); - - DataSourceStatusProvider.State secondStatus = sink.awaitStatus(2, TimeUnit.SECONDS); - assertNotNull("Should receive second status update", secondStatus); - assertEquals(DataSourceStatusProvider.State.OFF, secondStatus); + // Start the data system with a synchronizer that fails with a 401. + dataSource.start(); + DataSourceStatusProvider.State status = sink.awaitStatus(2, TimeUnit.SECONDS); - // Final state should be OFF with error info from the last status update - assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); + // The data system reports INTERRUPTED with the error from the synchronizer. + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, status); assertNotNull("Should have error info", sink.getLastError()); - assertEquals(DataSourceStatusProvider.ErrorKind.UNKNOWN, sink.getLastError().getKind()); + assertEquals(DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, sink.getLastError().getKind()); + assertEquals(401, sink.getLastError().getStatusCode()); } @Test @@ -2300,8 +2473,11 @@ public void statusRemainsValidDuringSynchronizerOperation() throws Exception { Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2317,16 +2493,16 @@ public void statusRemainsValidDuringSynchronizerOperation() throws Exception { } @Test - public void statusTransitionsFromValidToOffWhenAllSynchronizersFail() throws Exception { + public void statusStaysInterruptedWhileOnlySynchronizerIsParked() throws Exception { executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); - // First synchronizer sends a changeset then terminal error - BlockingQueue syncResults = new LinkedBlockingQueue<>(); - syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - syncResults.add(FDv2SourceResult.terminalError( + // The first instance sends a changeset and then a terminal error. Later instances send nothing. + BlockingQueue firstInstanceResults = new LinkedBlockingQueue<>(); + firstInstanceResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + firstInstanceResults.add(FDv2SourceResult.terminalError( new DataSourceStatusProvider.ErrorInfo( DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, @@ -2335,9 +2511,17 @@ public void statusTransitionsFromValidToOffWhenAllSynchronizersFail() throws Exc ), false )); + AtomicInteger buildCount = new AtomicInteger(); + BlockingQueue builds = new LinkedBlockingQueue<>(); ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(syncResults) + () -> { + int build = buildCount.incrementAndGet(); + builds.offer(build); + return build == 1 + ? new MockQueuedSynchronizer(firstInstanceResults) + : new MockQueuedSynchronizer(new LinkedBlockingQueue<>()); + } ); FDv2DataSource dataSource = new FDv2DataSource( @@ -2348,26 +2532,27 @@ public void statusTransitionsFromValidToOffWhenAllSynchronizersFail() throws Exc Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMillis(100), + Duration.ofSeconds(1), + Duration.ofMillis(10) ); resourcesToClose.add(dataSource); - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Should transition: VALID (from changeset) → INTERRUPTED (from terminal error) → OFF (from exhausted sources) - DataSourceStatusProvider.State firstStatus = sink.awaitStatus(2, TimeUnit.SECONDS); - assertEquals(DataSourceStatusProvider.State.VALID, firstStatus); - - DataSourceStatusProvider.State secondStatus = sink.awaitStatus(2, TimeUnit.SECONDS); - assertEquals(DataSourceStatusProvider.State.INTERRUPTED, secondStatus); - - DataSourceStatusProvider.State thirdStatus = sink.awaitStatus(2, TimeUnit.SECONDS); - assertEquals(DataSourceStatusProvider.State.OFF, thirdStatus); + // Start the data system, and wait until the parked synchronizer is built again. + dataSource.start(); + Integer firstBuild = builds.poll(2, TimeUnit.SECONDS); + Integer rebuild = builds.poll(2, TimeUnit.SECONDS); - assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); - assertNotNull("Should have error info when OFF", sink.getLastError()); + // The status goes from VALID to INTERRUPTED, and stays INTERRUPTED through the park. + assertEquals(Integer.valueOf(1), firstBuild); + assertEquals(Integer.valueOf(2), rebuild); + List statuses = sink.awaitStatuses(2, 2, TimeUnit.SECONDS); + assertEquals(Arrays.asList( + DataSourceStatusProvider.State.VALID, + DataSourceStatusProvider.State.INTERRUPTED), statuses); + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, sink.getLastState()); } // ============================================================================ @@ -2408,8 +2593,11 @@ public void fdv1FallbackActivatesWhenFlagSet() throws Exception { Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2456,8 +2644,11 @@ public void fdv1FallbackNotCalledWithoutFlag() throws Exception { Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2514,8 +2705,11 @@ public void fdv1FallbackWorksAfterInterruption() throws Exception { Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2571,8 +2765,11 @@ public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredHaltsDataSystem() thr Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2647,8 +2844,11 @@ public void fdv1FallbackBlocksOtherSynchronizers() throws Exception { Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2699,8 +2899,11 @@ public void fdv1FallbackOnlyCalledOncePerDataSource() throws Exception { Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2761,8 +2964,11 @@ public void fdv1FallbackOnInitializerSuccessAppliesPayloadAndSwitches() throws E Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2828,8 +3034,11 @@ public void fdv1FallbackOnInitializerErrorSwitchesToFDv1() throws Exception { Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2880,8 +3089,11 @@ public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredTransitionsToOff() thr Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2935,8 +3147,11 @@ public void fdv1FallbackOnInitializerSuccessNoSelectorAppliesPayloadAndSwitches( Thread.NORM_PRIORITY, logger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2967,8 +3182,11 @@ public void orchestrationLogging_warnsWhenNoInitializersOrSynchronizersConfigure Thread.NORM_PRIORITY, testLogger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -2998,8 +3216,11 @@ public void orchestrationLogging_logsInitializerStartSuccessAndErrorOnException( Thread.NORM_PRIORITY, testLogger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3035,8 +3256,11 @@ public void orchestrationLogging_logsInitializerFailedOnStatusResult() throws Ex Thread.NORM_PRIORITY, testLogger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3071,8 +3295,11 @@ public void orchestrationLogging_logsSynchronizerStartingStatusInterruptedAndShu Thread.NORM_PRIORITY, testLogger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3082,13 +3309,18 @@ public void orchestrationLogging_logsSynchronizerStartingStatusInterruptedAndShu } @Test - public void orchestrationLogging_logsSynchronizerPermanentFailureAndNoMoreSynchronizers() throws Exception { + public void orchestrationLogging_logsSynchronizerParkedAndAllParked() throws Exception { executor = Executors.newScheduledThreadPool(1); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - IterableAsyncQueue q = new IterableAsyncQueue<>(); - q.put(makeTerminalErrorResult()); + BlockingQueue builds = new LinkedBlockingQueue<>(); + // Every instance of the only synchronizer fails with a terminal error. Each build is recorded. ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(q, "failing-sync") + () -> { + builds.offer(1); + IterableAsyncQueue q = new IterableAsyncQueue<>(); + q.put(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(q, "failing-sync"); + } ); FDv2DataSource dataSource = new FDv2DataSource( ImmutableList.of(), @@ -3098,32 +3330,36 @@ public void orchestrationLogging_logsSynchronizerPermanentFailureAndNoMoreSynchr Thread.NORM_PRIORITY, testLogger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMillis(100), + Duration.ofSeconds(1), + Duration.ofMillis(10) ); resourcesToClose.add(dataSource); - dataSource.start().get(2, TimeUnit.SECONDS); + + // Start the data system, and wait until the parked synchronizer is built again. + dataSource.start(); + builds.poll(2, TimeUnit.SECONDS); + Integer rebuild = builds.poll(2, TimeUnit.SECONDS); + + // The data system logs the park, and logs that every synchronizer is parked. + assertNotNull(rebuild); assertTrue(logTextContains(logCapture, LDLogLevel.INFO, "Synchronizer 'failing-sync' is starting.")); assertTrue(logTextContains(logCapture, LDLogLevel.INFO, "Synchronizer 'failing-sync' reported status: TERMINAL_ERROR.")); - assertTrue( - logTextContains( - logCapture, - LDLogLevel.WARN, - "Synchronizer 'failing-sync' permanently failed and will not be used again until application restart." - ) - ); - assertTrue(logTextContains(logCapture, LDLogLevel.WARN, "No more synchronizers available.")); + assertTrue(logTextContains(logCapture, LDLogLevel.WARN, "Synchronizer 'failing-sync' is backing off for ")); + assertTrue(logTextContains(logCapture, LDLogLevel.WARN, "All synchronizers are backing off for at least ")); } @Test public void orchestrationLogging_logsErrorWhenSynchronizerNextThrows() throws Exception { executor = Executors.newScheduledThreadPool(1); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - // Two tiers: the first throws on next() once; the second shuts down immediately so the outer loop - // does not spin forever re-building the same throwing factory (single-tier would never block). + // The second synchronizer shuts down at once. That ends the run, so start() completes. IterableAsyncQueue followUpQueue = new IterableAsyncQueue<>(); followUpQueue.put(FDv2SourceResult.shutdown()); ImmutableList> synchronizers = ImmutableList.of( + // The first synchronizer fails from next(). () -> new MockSynchronizer(() -> { throw new RuntimeException("next-boom"); }, "throwing-sync"), @@ -3137,8 +3373,11 @@ public void orchestrationLogging_logsErrorWhenSynchronizerNextThrows() throws Ex Thread.NORM_PRIORITY, testLogger, executor, - 120, - 300 + Duration.ofSeconds(120), + Duration.ofSeconds(300), + Duration.ofMinutes(5), + Duration.ofHours(1), + Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingSynchronizerImplTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingSynchronizerImplTest.java index 5a8b5e79..1526be54 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingSynchronizerImplTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingSynchronizerImplTest.java @@ -10,6 +10,7 @@ import org.junit.Test; import java.io.IOException; +import java.security.cert.CertificateException; import java.time.Duration; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; @@ -18,6 +19,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -1073,4 +1075,110 @@ public void bothFdv1FallbackAndEnvironmentIdExtractedFromHeaders() throws Except executor.shutdown(); } } + + @Test + public void retryResetIsReportedOnSecondConsecutiveSuccessfulPoll() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(makeSuccessResponse())); + + try { + PollingSynchronizerImpl synchronizer = new PollingSynchronizerImpl( + requestor, + testLogger, + selectorSource, + executor, + Duration.ofMillis(10) + ); + + // Read the results of three consecutive successful polls. + FDv2SourceResult first = synchronizer.next().get(1, TimeUnit.SECONDS); + FDv2SourceResult second = synchronizer.next().get(1, TimeUnit.SECONDS); + FDv2SourceResult third = synchronizer.next().get(1, TimeUnit.SECONDS); + + // Only the second successful poll reports a retry reset. + assertFalse(first.isRetryReset()); + assertTrue(second.isRetryReset()); + assertFalse(third.isRetryReset()); + + synchronizer.close(); + } finally { + executor.shutdown(); + } + } + + @Test + public void failedPollRestartsRetryResetCount() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(makeSuccessResponse())) + .thenReturn(failedFuture(new IOException("network error"))) + .thenReturn(CompletableFuture.completedFuture(makeSuccessResponse())); + + try { + PollingSynchronizerImpl synchronizer = new PollingSynchronizerImpl( + requestor, + testLogger, + selectorSource, + executor, + Duration.ofMillis(10) + ); + + // Read the results of a successful poll, a failed poll, and two more successful polls. + FDv2SourceResult successBeforeFailure = synchronizer.next().get(1, TimeUnit.SECONDS); + FDv2SourceResult failure = synchronizer.next().get(1, TimeUnit.SECONDS); + FDv2SourceResult firstSuccessAfterFailure = synchronizer.next().get(1, TimeUnit.SECONDS); + FDv2SourceResult secondSuccessAfterFailure = synchronizer.next().get(1, TimeUnit.SECONDS); + + // The failure restarts the count, so the reset comes on the second success after it. + assertFalse(successBeforeFailure.isRetryReset()); + assertEquals(SourceSignal.INTERRUPTED, failure.getStatus().getState()); + assertFalse(failure.isRetryReset()); + assertFalse(firstSuccessAfterFailure.isRetryReset()); + assertTrue(secondSuccessAfterFailure.isRetryReset()); + + synchronizer.close(); + } finally { + executor.shutdown(); + } + } + + @Test + public void certificateFailureIsTerminal() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + + IOException certificateFailure = new IOException( + "TLS handshake failed", + new CertificateException("untrusted certificate")); + when(requestor.Poll(any(Selector.class))).thenReturn(failedFuture(certificateFailure)); + + try { + PollingSynchronizerImpl synchronizer = new PollingSynchronizerImpl( + requestor, + testLogger, + selectorSource, + executor, + Duration.ofMillis(10) + ); + + // Read the result of a poll that fails certificate validation. + FDv2SourceResult result = synchronizer.next().get(1, TimeUnit.SECONDS); + + // A certificate failure is terminal for this synchronizer. + assertEquals(SourceResultType.STATUS, result.getResultType()); + assertEquals(SourceSignal.TERMINAL_ERROR, result.getStatus().getState()); + + synchronizer.close(); + } finally { + executor.shutdown(); + } + } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java index 2ebd7a71..74bd7793 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java @@ -2,6 +2,7 @@ import com.launchdarkly.sdk.LDValue; import com.launchdarkly.sdk.internal.events.DiagnosticStore; +import com.launchdarkly.sdk.fdv2.ChangeSetType; import com.launchdarkly.sdk.fdv2.Selector; import com.launchdarkly.sdk.internal.http.HttpProperties; import com.launchdarkly.sdk.fdv2.SourceResultType; @@ -12,14 +13,18 @@ import com.launchdarkly.testhelpers.httptest.Handlers; import com.launchdarkly.testhelpers.httptest.HttpServer; import com.launchdarkly.testhelpers.httptest.RequestInfo; +import com.launchdarkly.testhelpers.httptest.ServerTLSConfiguration; import com.launchdarkly.testhelpers.tcptest.TcpHandler; import com.launchdarkly.testhelpers.tcptest.TcpHandlers; import com.launchdarkly.testhelpers.tcptest.TcpServer; +import org.junit.After; import org.junit.Test; import java.net.URI; import java.time.Duration; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -45,6 +50,13 @@ public class StreamingSynchronizerImplTest extends BaseTest { + private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + + @After + public void shutDownExecutor() { + executor.shutdownNow(); + } + private SelectorSource mockSelectorSource() { SelectorSource source = mock(SelectorSource.class); when(source.getSelector()).thenReturn(Selector.EMPTY); @@ -79,8 +91,10 @@ public void receivesMultipleChangesets() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -117,8 +131,10 @@ public void httpNonRecoverableError() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -147,8 +163,10 @@ public void httpRecoverableError() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -178,8 +196,10 @@ public void networkError() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -213,8 +233,10 @@ public void invalidEventData() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -246,8 +268,10 @@ public void shutdownBeforeEventReceived() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -288,8 +312,10 @@ public void shutdownAfterEventReceived() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -332,8 +358,10 @@ public void goodbyeEventInResponse() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -394,8 +422,10 @@ public void reconnectOpensNewConnection() throws Exception { "/stream", testLogger, mockSelectorSource(), + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -435,8 +465,10 @@ public void heartbeatEvent() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -475,8 +507,10 @@ public void selectorWithVersionAndState() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -531,8 +565,10 @@ public void selectorRefetchedOnReconnection() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -593,8 +629,10 @@ public void errorEventFromServer() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -633,8 +671,10 @@ public void selectorWithVersionOnly() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -675,8 +715,10 @@ public void selectorWithEmptyState() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -710,8 +752,10 @@ public void closeCalledMultipleTimes() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -748,8 +792,10 @@ public void invalidEventStructureCausesInterrupt() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -785,8 +831,10 @@ public void payloadFilterIsAddedToRequest() throws Exception { "/stream", testLogger, selectorSource, + executor, "myFilter", Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -828,8 +876,10 @@ public void payloadFilterWithSelectorBothAddedToRequest() throws Exception { "/stream", testLogger, selectorSource, + executor, "testFilter", Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -869,8 +919,10 @@ public void emptyPayloadFilterNotAddedToRequest() throws Exception { "/stream", testLogger, selectorSource, + executor, "", Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -910,8 +962,10 @@ public void nullPayloadFilterNotAddedToRequest() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -952,8 +1006,10 @@ public void fdv1FallbackFlagSetToTrueInSuccessResponse() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -989,8 +1045,10 @@ public void fdv1FallbackFlagSetToFalseWhenHeaderNotPresent() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1021,8 +1079,10 @@ public void fdv1FallbackFlagSetToTrueInErrorResponse() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1060,8 +1120,10 @@ public void environmentIdExtractedFromHeaders() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1100,8 +1162,10 @@ public void bothFdv1FallbackAndEnvironmentIdExtractedFromHeaders() throws Except "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1139,8 +1203,10 @@ public void serializationExceptionWithoutFallbackHeader() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1179,8 +1245,10 @@ public void serializationExceptionPreservesFallbackHeader() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1221,8 +1289,10 @@ public void streamInitDiagnosticRecordedOnSuccessfulChangeset() throws Exception "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1273,8 +1343,10 @@ public void streamInitDiagnosticRecordedOnErrorDuringInit() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1341,8 +1413,10 @@ public void streamRestartNotRecordedAsFailed() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1394,8 +1468,10 @@ public void nullDiagnosticStoreDoesNotCauseError() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null // null diagnosticStore ); @@ -1451,8 +1527,10 @@ public void multipleRestartsRecordMultipleDiagnostics() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1524,8 +1602,10 @@ public void streamRestartAfterInvalidDataRecordsMultipleDiagnostics() throws Exc "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1579,8 +1659,10 @@ public void multipleErrorsRecordMultipleDiagnostics() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1657,8 +1739,10 @@ public void errorAfterSuccessfulChangesetRecordsNewDiagnostic() throws Exception "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1733,8 +1817,10 @@ public void goodbyeAfterEarlierDisconnectRestartsStream() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1808,8 +1894,10 @@ public void invalidDataAfterEarlierDisconnectRestartsStream() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1881,8 +1969,10 @@ public void closeDuringRestartAfterEarlierDisconnectDoesNotReconnect() throws Ex "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(200), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1939,8 +2029,10 @@ public void closeAfterEarlierDisconnectDoesNotReconnect() throws Exception { "/stream", testLogger, selectorSource, + executor, null, Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1960,4 +2052,129 @@ public void closeAfterEarlierDisconnectDoesNotReconnect() throws Exception { assertEquals(2, server.getRecorder().count()); } } + + @Test + public void retryResetIsReportedAfterStreamStaysHealthy() throws Exception { + String serverIntent = makeEvent("server-intent", "{\"payloads\":[{\"id\":\"payload-1\",\"target\":100,\"intentCode\":\"xfer-full\",\"reason\":\"payload-missing\"}]}"); + String payloadTransferred = makeEvent("payload-transferred", "{\"state\":\"(p:payload-1:100)\",\"version\":100}"); + + try (HttpServer server = HttpServer.start(Handlers.all( + Handlers.SSE.start(), + Handlers.SSE.event(serverIntent), + Handlers.SSE.event(payloadTransferred), + Handlers.SSE.leaveOpen()))) { + + HttpProperties httpProperties = toHttpProperties(clientContext("sdk-key", baseConfig().build()).getHttp()); + + StreamingSynchronizerImpl synchronizer = new StreamingSynchronizerImpl( + httpProperties, + server.getUri(), + "/stream", + testLogger, + mockSelectorSource(), + executor, + null, + Duration.ofMillis(100), + Duration.ofMillis(100), + Thread.NORM_PRIORITY, + null + ); + + // Read the first valid result, then the result that follows while the stream stays open. + FDv2SourceResult changeSet = synchronizer.next().get(5, TimeUnit.SECONDS); + FDv2SourceResult reset = synchronizer.next().get(5, TimeUnit.SECONDS); + + // After the stream stays healthy for the threshold, it reports a retry reset with no changes. + assertFalse(changeSet.isRetryReset()); + assertEquals(SourceResultType.CHANGE_SET, reset.getResultType()); + assertEquals(ChangeSetType.None, reset.getChangeSet().getType()); + assertTrue(reset.isRetryReset()); + + synchronizer.close(); + } + } + + @Test + public void streamFailureCancelsPendingRetryReset() throws Exception { + String serverIntent = makeEvent("server-intent", "{\"payloads\":[{\"id\":\"payload-1\",\"target\":100,\"intentCode\":\"xfer-full\",\"reason\":\"payload-missing\"}]}"); + String payloadTransferred = makeEvent("payload-transferred", "{\"state\":\"(p:payload-1:100)\",\"version\":100}"); + + try (HttpServer server = HttpServer.start(Handlers.sequential( + Handlers.all( + Handlers.SSE.start(), + Handlers.SSE.event(serverIntent), + Handlers.SSE.event(payloadTransferred)), + Handlers.all( + Handlers.SSE.start(), + Handlers.SSE.event(serverIntent), + Handlers.SSE.event(payloadTransferred), + Handlers.SSE.leaveOpen())))) { + + HttpProperties httpProperties = toHttpProperties(clientContext("sdk-key", baseConfig().build()).getHttp()); + + // The reconnect delay is longer than the reset threshold, so a reset from the first stream + // would arrive before the second stream connects. + StreamingSynchronizerImpl synchronizer = new StreamingSynchronizerImpl( + httpProperties, + server.getUri(), + "/stream", + testLogger, + mockSelectorSource(), + executor, + null, + Duration.ofSeconds(1), + Duration.ofMillis(200), + Thread.NORM_PRIORITY, + null + ); + + // Read the results of a stream that ends after one change set, then of a second stream that stays open. + FDv2SourceResult firstStreamChangeSet = synchronizer.next().get(5, TimeUnit.SECONDS); + FDv2SourceResult failure = synchronizer.next().get(5, TimeUnit.SECONDS); + FDv2SourceResult secondStreamChangeSet = synchronizer.next().get(5, TimeUnit.SECONDS); + FDv2SourceResult reset = synchronizer.next().get(5, TimeUnit.SECONDS); + + // The failure cancels the first stream's reset, so the only reset follows the second stream's change set. + assertEquals(SourceResultType.CHANGE_SET, firstStreamChangeSet.getResultType()); + assertFalse(firstStreamChangeSet.isRetryReset()); + assertEquals(SourceSignal.INTERRUPTED, failure.getStatus().getState()); + assertEquals(SourceResultType.CHANGE_SET, secondStreamChangeSet.getResultType()); + assertFalse(secondStreamChangeSet.isRetryReset()); + assertTrue(reset.isRetryReset()); + + synchronizer.close(); + } + } + + @Test + public void certificateFailureIsTerminal() throws Exception { + ServerTLSConfiguration untrustedCertificate = ServerTLSConfiguration.makeSelfSignedCertificate(); + + try (HttpServer server = HttpServer.startSecure(untrustedCertificate, Handlers.SSE.start())) { + HttpProperties httpProperties = toHttpProperties(clientContext("sdk-key", baseConfig().build()).getHttp()); + + StreamingSynchronizerImpl synchronizer = new StreamingSynchronizerImpl( + httpProperties, + server.getUri(), + "/stream", + testLogger, + mockSelectorSource(), + executor, + null, + Duration.ofMillis(100), + StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + Thread.NORM_PRIORITY, + null + ); + + // Connect to a server whose certificate the client does not trust. + FDv2SourceResult result = synchronizer.next().get(5, TimeUnit.SECONDS); + + // A certificate failure is terminal for this synchronizer. + assertEquals(SourceResultType.STATUS, result.getResultType()); + assertEquals(SourceSignal.TERMINAL_ERROR, result.getStatus().getState()); + + synchronizer.close(); + } + } } From 54810b2175c3ea53813b537fef8a10ee0453b096 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Tue, 3 Nov 2026 09:49:45 -0800 Subject: [PATCH 02/13] refactor: Rename retryReset to resetThresholdMet --- .../sdk/server/DataSystemComponents.java | 2 +- .../sdk/server/PollingSynchronizerImpl.java | 8 +- .../sdk/server/StreamingSynchronizerImpl.java | 64 +++++------ .../server/SynchronizerFactoryWithState.java | 2 +- .../server/datasources/FDv2SourceResult.java | 34 +++--- .../server/PollingSynchronizerImplTest.java | 22 ++-- .../server/StreamingSynchronizerImplTest.java | 104 +++++++++--------- 7 files changed, 118 insertions(+), 118 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystemComponents.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystemComponents.java index 0875b6cf..0b770ecc 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystemComponents.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystemComponents.java @@ -108,7 +108,7 @@ public Synchronizer build(DataSourceBuildInputs context) { context.getSharedExecutor(), payloadFilter, initialReconnectDelay, - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, context.getThreadPriority(), context.getDiagnosticStore() ); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java index 5407a214..bf7535dd 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java @@ -11,9 +11,9 @@ class PollingSynchronizerImpl extends PollingBase implements Synchronizer { /** - * The number of consecutive successful polls that resets the retry state. + * The number of consecutive successful polls that meets the reset threshold. */ - private static final int RETRY_RESET_SUCCESSFUL_POLLS = 2; + private static final int RESET_THRESHOLD_SUCCESSFUL_POLLS = 2; private final CompletableFuture shutdownFuture = new CompletableFuture<>(); private final SelectorSource selectorSource; @@ -59,8 +59,8 @@ private void doPoll() { switch (res.getResultType()) { case CHANGE_SET: consecutiveSuccessfulPolls++; - if (consecutiveSuccessfulPolls == RETRY_RESET_SUCCESSFUL_POLLS) { - res = res.withRetryReset(); + if (consecutiveSuccessfulPolls == RESET_THRESHOLD_SUCCESSFUL_POLLS) { + res = res.withResetThresholdMet(); } break; case STATUS: diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java index bc7fabd7..62613929 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java @@ -59,9 +59,9 @@ class StreamingSynchronizerImpl implements Synchronizer { private static final Duration DEAD_CONNECTION_INTERVAL = Duration.ofSeconds(300); /** - * The time a stream must stay healthy, from its first valid result, before the retry state resets. + * The time a stream must stay healthy, from its first valid result, to meet the reset threshold. */ - static final Duration RETRY_RESET_THRESHOLD = Duration.ofSeconds(60); + static final Duration RESET_THRESHOLD = Duration.ofSeconds(60); private final HttpProperties httpProperties; private final SelectorSource selectorSource; @@ -81,16 +81,16 @@ class StreamingSynchronizerImpl implements Synchronizer { private final int threadPriority; private final DiagnosticStore diagnosticStore; private final ScheduledExecutorService sharedExecutor; - private final Duration retryResetThreshold; + private final Duration resetThreshold; private volatile long streamStarted = 0; - private final Object retryResetLock = new Object(); + private final Object resetThresholdLock = new Object(); /** - * The timer that reports a retry reset for the current stream, or null if the current stream has - * not produced a valid result. Guarded by {@link #retryResetLock}. + * The timer that reports that the current stream met its reset threshold, or null if the current stream has + * not produced a valid result. Guarded by {@link #resetThresholdLock}. */ - private ScheduledFuture retryResetFuture; + private ScheduledFuture resetThresholdFuture; public StreamingSynchronizerImpl( HttpProperties httpProperties, @@ -101,7 +101,7 @@ public StreamingSynchronizerImpl( ScheduledExecutorService sharedExecutor, String payloadFilter, Duration initialReconnectDelaySeconds, - Duration retryResetThreshold, + Duration resetThreshold, int threadPriority, DiagnosticStore diagnosticStore ) { @@ -114,7 +114,7 @@ public StreamingSynchronizerImpl( this.threadPriority = threadPriority; this.diagnosticStore = diagnosticStore; this.sharedExecutor = sharedExecutor; - this.retryResetThreshold = retryResetThreshold; + this.resetThreshold = resetThreshold; // The stream will lazily start when `next` is called. } @@ -210,7 +210,7 @@ private Thread getRunThread() { logger.error("Stream thread ended with exception: {}", LogValues.exceptionSummary(e)); logger.debug(LogValues.exceptionTrace(e)); - cancelRetryResetTimer(); + cancelResetThresholdTimer(); recordStreamInit(true); // Record failed init for unexpected thread exception DataSourceStatusProvider.ErrorInfo errorInfo = new DataSourceStatusProvider.ErrorInfo( @@ -262,7 +262,7 @@ public void close() { } } - cancelRetryResetTimer(); + cancelResetThresholdTimer(); shutdownFuture.complete(FDv2SourceResult.shutdown()); } @@ -274,37 +274,37 @@ private void recordStreamInit(boolean failed) { } /** - * Starts the retry reset timer if the current stream does not have one yet. Call this for each + * Starts the reset threshold timer if the current stream does not have one yet. Call this for each * valid result, so that the first valid result on a stream starts the timer. */ - private void startRetryResetTimerIfNeeded() { - synchronized (retryResetLock) { - if (retryResetFuture != null) { + private void startResetThresholdTimerIfNeeded() { + synchronized (resetThresholdLock) { + if (resetThresholdFuture != null) { return; } - retryResetFuture = sharedExecutor.schedule( - this::reportRetryReset, - retryResetThreshold.toMillis(), + resetThresholdFuture = sharedExecutor.schedule( + this::reportResetThresholdMet, + resetThreshold.toMillis(), TimeUnit.MILLISECONDS); } } /** - * Stops the retry reset timer of the current stream. Call this when the stream ends. + * Stops the reset threshold timer of the current stream. Call this when the stream ends. */ - private void cancelRetryResetTimer() { - synchronized (retryResetLock) { - if (retryResetFuture != null) { - retryResetFuture.cancel(false); - retryResetFuture = null; + private void cancelResetThresholdTimer() { + synchronized (resetThresholdLock) { + if (resetThresholdFuture != null) { + resetThresholdFuture.cancel(false); + resetThresholdFuture = null; } } } - private void reportRetryReset() { - synchronized (retryResetLock) { - // The timer can start to run before a cancel. Do not report a reset for a stream that ended. - if (retryResetFuture == null) { + private void reportResetThresholdMet() { + synchronized (resetThresholdLock) { + // The timer can start to run before a cancel. Do not report anything for a stream that ended. + if (resetThresholdFuture == null) { return; } ChangeSet>>> noChanges = new ChangeSet<>( @@ -313,7 +313,7 @@ private void reportRetryReset() { Collections.emptyList(), /* environmentId= */ null, /* shouldPersist= */ true); - resultQueue.put(FDv2SourceResult.changeSet(noChanges, /* fdv1Fallback= */ false).withRetryReset()); + resultQueue.put(FDv2SourceResult.changeSet(noChanges, /* fdv1Fallback= */ false).withResetThresholdMet()); } } @@ -365,7 +365,7 @@ private void handleMessage(MessageEvent event) { // Reset to 0 after successful init to prevent duplicate recordings until next restart streamStarted = 0; result = FDv2SourceResult.changeSet(converted, getFallback(event)); - startRetryResetTimerIfNeeded(); + startResetThresholdTimerIfNeeded(); } catch (Exception e) { logger.error("Failed to convert FDv2 changeset: {}", LogValues.exceptionSummary(e)); logger.debug(LogValues.exceptionTrace(e)); @@ -457,7 +457,7 @@ private boolean handleError(StreamException e) { return true; } - cancelRetryResetTimer(); + cancelResetThresholdTimer(); recordStreamInit(true); if (e instanceof StreamHttpErrorException) { @@ -500,7 +500,7 @@ private boolean handleError(StreamException e) { private void restartStream(boolean failed) { Objects.requireNonNull(eventSource, "eventSource must not be null"); - cancelRetryResetTimer(); + cancelResetThresholdTimer(); recordStreamInit(failed); streamStarted = System.currentTimeMillis(); eventSource.interrupt(); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java index 923ad2ff..d8a95f96 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java @@ -24,7 +24,7 @@ public enum State { private boolean isFDv1Fallback = false; /** - * The number of unexpected failures since the synchronizer last reported a retry reset. + * The number of unexpected failures since the synchronizer last reported that it met its reset threshold. */ private int parkCount = 0; diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java index a2b4ef99..d25d77c1 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java @@ -72,21 +72,21 @@ public String getReason() { private final Status status; private final SourceResultType resultType; private final boolean fdv1Fallback; - private final boolean retryReset; + private final boolean resetThresholdMet; private final Function completionCallback; private FDv2SourceResult( ChangeSet>>> changeSet, Status status, SourceResultType resultType, boolean fdv1Fallback, - boolean retryReset, + boolean resetThresholdMet, Function completionCallback ) { this.changeSet = changeSet; this.status = status; this.resultType = resultType; this.fdv1Fallback = fdv1Fallback; - this.retryReset = retryReset; + this.resetThresholdMet = resetThresholdMet; this.completionCallback = completionCallback; } @@ -100,7 +100,7 @@ public static FDv2SourceResult interrupted(DataSourceStatusProvider.ErrorInfo er Status.interrupted(errorInfo), SourceResultType.STATUS, fdv1Fallback, - /* retryReset= */ false, + /* resetThresholdMet= */ false, completionCallback); } @@ -114,7 +114,7 @@ public static FDv2SourceResult shutdown(Function completionCallback) Status.shutdown(), SourceResultType.STATUS, /* fdv1Fallback= */ false, - /* retryReset= */ false, + /* resetThresholdMet= */ false, completionCallback); } @@ -128,7 +128,7 @@ public static FDv2SourceResult terminalError(DataSourceStatusProvider.ErrorInfo Status.terminalError(errorInfo), SourceResultType.STATUS, fdv1Fallback, - /* retryReset= */ false, + /* resetThresholdMet= */ false, completionCallback); } @@ -142,7 +142,7 @@ public static FDv2SourceResult changeSet(ChangeSet newCallback) { status, resultType, fdv1Fallback, - retryReset, + resetThresholdMet, combinedCallback); } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingSynchronizerImplTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingSynchronizerImplTest.java index 1526be54..797f4ebb 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingSynchronizerImplTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingSynchronizerImplTest.java @@ -1077,7 +1077,7 @@ public void bothFdv1FallbackAndEnvironmentIdExtractedFromHeaders() throws Except } @Test - public void retryResetIsReportedOnSecondConsecutiveSuccessfulPoll() throws Exception { + public void resetThresholdMetIsReportedOnSecondConsecutiveSuccessfulPoll() throws Exception { FDv2Requestor requestor = mockRequestor(); SelectorSource selectorSource = mockSelectorSource(); ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); @@ -1099,10 +1099,10 @@ public void retryResetIsReportedOnSecondConsecutiveSuccessfulPoll() throws Excep FDv2SourceResult second = synchronizer.next().get(1, TimeUnit.SECONDS); FDv2SourceResult third = synchronizer.next().get(1, TimeUnit.SECONDS); - // Only the second successful poll reports a retry reset. - assertFalse(first.isRetryReset()); - assertTrue(second.isRetryReset()); - assertFalse(third.isRetryReset()); + // Only the second successful poll reports that the reset threshold was met. + assertFalse(first.isResetThresholdMet()); + assertTrue(second.isResetThresholdMet()); + assertFalse(third.isResetThresholdMet()); synchronizer.close(); } finally { @@ -1111,7 +1111,7 @@ public void retryResetIsReportedOnSecondConsecutiveSuccessfulPoll() throws Excep } @Test - public void failedPollRestartsRetryResetCount() throws Exception { + public void failedPollRestartsResetThresholdCount() throws Exception { FDv2Requestor requestor = mockRequestor(); SelectorSource selectorSource = mockSelectorSource(); ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); @@ -1136,12 +1136,12 @@ public void failedPollRestartsRetryResetCount() throws Exception { FDv2SourceResult firstSuccessAfterFailure = synchronizer.next().get(1, TimeUnit.SECONDS); FDv2SourceResult secondSuccessAfterFailure = synchronizer.next().get(1, TimeUnit.SECONDS); - // The failure restarts the count, so the reset comes on the second success after it. - assertFalse(successBeforeFailure.isRetryReset()); + // The failure restarts the count, so the threshold is met on the second success after it. + assertFalse(successBeforeFailure.isResetThresholdMet()); assertEquals(SourceSignal.INTERRUPTED, failure.getStatus().getState()); - assertFalse(failure.isRetryReset()); - assertFalse(firstSuccessAfterFailure.isRetryReset()); - assertTrue(secondSuccessAfterFailure.isRetryReset()); + assertFalse(failure.isResetThresholdMet()); + assertFalse(firstSuccessAfterFailure.isResetThresholdMet()); + assertTrue(secondSuccessAfterFailure.isResetThresholdMet()); synchronizer.close(); } finally { diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java index 74bd7793..8e32d5c0 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java @@ -94,7 +94,7 @@ public void receivesMultipleChangesets() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -134,7 +134,7 @@ public void httpNonRecoverableError() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -166,7 +166,7 @@ public void httpRecoverableError() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -199,7 +199,7 @@ public void networkError() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -236,7 +236,7 @@ public void invalidEventData() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -271,7 +271,7 @@ public void shutdownBeforeEventReceived() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -315,7 +315,7 @@ public void shutdownAfterEventReceived() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -361,7 +361,7 @@ public void goodbyeEventInResponse() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -425,7 +425,7 @@ public void reconnectOpensNewConnection() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -468,7 +468,7 @@ public void heartbeatEvent() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -510,7 +510,7 @@ public void selectorWithVersionAndState() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -568,7 +568,7 @@ public void selectorRefetchedOnReconnection() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -632,7 +632,7 @@ public void errorEventFromServer() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -674,7 +674,7 @@ public void selectorWithVersionOnly() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -718,7 +718,7 @@ public void selectorWithEmptyState() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -755,7 +755,7 @@ public void closeCalledMultipleTimes() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -795,7 +795,7 @@ public void invalidEventStructureCausesInterrupt() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -834,7 +834,7 @@ public void payloadFilterIsAddedToRequest() throws Exception { executor, "myFilter", Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -879,7 +879,7 @@ public void payloadFilterWithSelectorBothAddedToRequest() throws Exception { executor, "testFilter", Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -922,7 +922,7 @@ public void emptyPayloadFilterNotAddedToRequest() throws Exception { executor, "", Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -965,7 +965,7 @@ public void nullPayloadFilterNotAddedToRequest() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1009,7 +1009,7 @@ public void fdv1FallbackFlagSetToTrueInSuccessResponse() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1048,7 +1048,7 @@ public void fdv1FallbackFlagSetToFalseWhenHeaderNotPresent() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1082,7 +1082,7 @@ public void fdv1FallbackFlagSetToTrueInErrorResponse() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1123,7 +1123,7 @@ public void environmentIdExtractedFromHeaders() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1165,7 +1165,7 @@ public void bothFdv1FallbackAndEnvironmentIdExtractedFromHeaders() throws Except executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1206,7 +1206,7 @@ public void serializationExceptionWithoutFallbackHeader() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1248,7 +1248,7 @@ public void serializationExceptionPreservesFallbackHeader() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -1292,7 +1292,7 @@ public void streamInitDiagnosticRecordedOnSuccessfulChangeset() throws Exception executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1346,7 +1346,7 @@ public void streamInitDiagnosticRecordedOnErrorDuringInit() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1416,7 +1416,7 @@ public void streamRestartNotRecordedAsFailed() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1471,7 +1471,7 @@ public void nullDiagnosticStoreDoesNotCauseError() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null // null diagnosticStore ); @@ -1530,7 +1530,7 @@ public void multipleRestartsRecordMultipleDiagnostics() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1605,7 +1605,7 @@ public void streamRestartAfterInvalidDataRecordsMultipleDiagnostics() throws Exc executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1662,7 +1662,7 @@ public void multipleErrorsRecordMultipleDiagnostics() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1742,7 +1742,7 @@ public void errorAfterSuccessfulChangesetRecordsNewDiagnostic() throws Exception executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1820,7 +1820,7 @@ public void goodbyeAfterEarlierDisconnectRestartsStream() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1897,7 +1897,7 @@ public void invalidDataAfterEarlierDisconnectRestartsStream() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, diagnosticStore ); @@ -1972,7 +1972,7 @@ public void closeDuringRestartAfterEarlierDisconnectDoesNotReconnect() throws Ex executor, null, Duration.ofMillis(200), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -2032,7 +2032,7 @@ public void closeAfterEarlierDisconnectDoesNotReconnect() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); @@ -2054,7 +2054,7 @@ public void closeAfterEarlierDisconnectDoesNotReconnect() throws Exception { } @Test - public void retryResetIsReportedAfterStreamStaysHealthy() throws Exception { + public void resetThresholdMetIsReportedAfterStreamStaysHealthy() throws Exception { String serverIntent = makeEvent("server-intent", "{\"payloads\":[{\"id\":\"payload-1\",\"target\":100,\"intentCode\":\"xfer-full\",\"reason\":\"payload-missing\"}]}"); String payloadTransferred = makeEvent("payload-transferred", "{\"state\":\"(p:payload-1:100)\",\"version\":100}"); @@ -2084,18 +2084,18 @@ public void retryResetIsReportedAfterStreamStaysHealthy() throws Exception { FDv2SourceResult changeSet = synchronizer.next().get(5, TimeUnit.SECONDS); FDv2SourceResult reset = synchronizer.next().get(5, TimeUnit.SECONDS); - // After the stream stays healthy for the threshold, it reports a retry reset with no changes. - assertFalse(changeSet.isRetryReset()); + // After the stream stays healthy for the threshold, it reports that the reset threshold was met, with no changes. + assertFalse(changeSet.isResetThresholdMet()); assertEquals(SourceResultType.CHANGE_SET, reset.getResultType()); assertEquals(ChangeSetType.None, reset.getChangeSet().getType()); - assertTrue(reset.isRetryReset()); + assertTrue(reset.isResetThresholdMet()); synchronizer.close(); } } @Test - public void streamFailureCancelsPendingRetryReset() throws Exception { + public void streamFailureCancelsPendingResetThresholdReport() throws Exception { String serverIntent = makeEvent("server-intent", "{\"payloads\":[{\"id\":\"payload-1\",\"target\":100,\"intentCode\":\"xfer-full\",\"reason\":\"payload-missing\"}]}"); String payloadTransferred = makeEvent("payload-transferred", "{\"state\":\"(p:payload-1:100)\",\"version\":100}"); @@ -2122,8 +2122,8 @@ public void streamFailureCancelsPendingRetryReset() throws Exception { mockSelectorSource(), executor, null, - Duration.ofSeconds(1), - Duration.ofMillis(200), + Duration.ofSeconds(2), + Duration.ofMillis(500), Thread.NORM_PRIORITY, null ); @@ -2134,13 +2134,13 @@ public void streamFailureCancelsPendingRetryReset() throws Exception { FDv2SourceResult secondStreamChangeSet = synchronizer.next().get(5, TimeUnit.SECONDS); FDv2SourceResult reset = synchronizer.next().get(5, TimeUnit.SECONDS); - // The failure cancels the first stream's reset, so the only reset follows the second stream's change set. + // The failure cancels the first stream's timer, so the only report follows the second stream's change set. assertEquals(SourceResultType.CHANGE_SET, firstStreamChangeSet.getResultType()); - assertFalse(firstStreamChangeSet.isRetryReset()); + assertFalse(firstStreamChangeSet.isResetThresholdMet()); assertEquals(SourceSignal.INTERRUPTED, failure.getStatus().getState()); assertEquals(SourceResultType.CHANGE_SET, secondStreamChangeSet.getResultType()); - assertFalse(secondStreamChangeSet.isRetryReset()); - assertTrue(reset.isRetryReset()); + assertFalse(secondStreamChangeSet.isResetThresholdMet()); + assertTrue(reset.isResetThresholdMet()); synchronizer.close(); } @@ -2162,7 +2162,7 @@ public void certificateFailureIsTerminal() throws Exception { executor, null, Duration.ofMillis(100), - StreamingSynchronizerImpl.RETRY_RESET_THRESHOLD, + StreamingSynchronizerImpl.RESET_THRESHOLD, Thread.NORM_PRIORITY, null ); From a602098e88e5a356a28c1be11f37d458e9ae621f Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Wed, 4 Nov 2026 13:40:31 -0800 Subject: [PATCH 03/13] feat: Reset park counts and add unpark conditions --- .../sdk/server/FDv2DataSource.java | 186 +++++++++++++----- .../sdk/server/FDv2DataSourceConditions.java | 58 ++++++ .../sdk/server/SourceManager.java | 24 ++- .../server/SynchronizerFactoryWithState.java | 18 +- ...FDv2DataSourceConditionsAggregateTest.java | 119 +++++++++-- .../sdk/server/FDv2DataSourceTest.java | 38 ++++ 6 files changed, 373 insertions(+), 70 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 8ba69533..c0357aec 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -16,7 +16,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; +import java.util.EnumMap; +import java.util.EnumSet; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.WeakHashMap; import java.util.concurrent.*; @@ -27,6 +31,7 @@ import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.ConditionFactory; import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.FallbackCondition; import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.RecoveryCondition; +import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.UnparkCondition; class FDv2DataSource implements DataSource { /** @@ -154,6 +159,11 @@ public FDv2DataSource( this.conditionFactories = new ArrayList<>(); this.conditionFactories.add(new FallbackCondition.Factory(sharedExecutor, fallbackTimeout)); this.conditionFactories.add(new RecoveryCondition.Factory(sharedExecutor, recoveryTimeout)); + // getEarliestUnparkTime synchronizes on the lock of the source manager, so it is safe to call from the thread + // that builds the conditions. + // TODO: Decide whether to keep this guarantee, or to require that conditions are built only on the data + // source thread. + this.conditionFactories.add(new UnparkCondition.Factory(sharedExecutor, sourceManager::getEarliestUnparkTime)); this.parkInitialDelay = parkInitialDelay; this.parkMaxDelay = parkMaxDelay; this.failoverDelay = failoverDelay; @@ -368,10 +378,11 @@ private List getConditions() { // If there isn't a synchronizer to recover to, then don't add and recovery conditions. return conditionFactories.stream() .filter((ConditionFactory factory) -> factory.getType() != Condition.ConditionType.RECOVERY) - .map(ConditionFactory::build).collect(Collectors.toList()); + .map(ConditionFactory::build).filter(Objects::nonNull).collect(Collectors.toList()); } // The synchronizer can both fall back and recover. - return conditionFactories.stream().map(ConditionFactory::build).collect(Collectors.toList()); + return conditionFactories.stream() + .map(ConditionFactory::build).filter(Objects::nonNull).collect(Collectors.toList()); } /** @@ -439,6 +450,10 @@ private boolean runSynchronizers() { try (FDv2SourceResult result = (FDv2SourceResult) res) { conditions.inform(result); + if (result.isResetThresholdMet()) { + sourceManager.resetCurrentSynchronizerParkCount(); + } + switch (result.getResultType()) { case CHANGE_SET: // A data update breaks the "in a row" streak for status deduplication. @@ -767,84 +782,157 @@ private void maybeReportUnexpectedExhaustion(String message) { * exercise the API surface and assert per-call distinctness. */ static class Conditions implements AutoCloseable { - private final List conditions; - private final CompletableFuture aggregate; private final Object lock = new Object(); + /** + * The conditions, with at most one of each type. Accessed only under {@code lock}. + */ + private final Map conditions = + new EnumMap<>(Condition.ConditionType.class); + + /** + * The types of the conditions that the aggregate has completed with. Accessed only under {@code lock}. + */ + private final Set reported = EnumSet.noneOf(Condition.ConditionType.class); + + /** + * A future over the futures of all the conditions. It completes with the first condition that is met. + * Accessed only under {@code lock}. + */ + private CompletableFuture aggregate; + /** * Tracks futures previously returned by {@link #getFuture()} that have * not yet been completed. Held weakly via {@link WeakHashMap} so that * fresh futures abandoned by the caller (the typical end-of-iteration * case) become GC-collectable. Set to {@code null} once the aggregate - * has fired and the entries have been drained. Mutated only under + * has fired and the entries have been drained. Accessed only under * {@code lock}. */ - private Set> pending = - Collections.newSetFromMap(new WeakHashMap<>()); + private Set> pending; public Conditions(List conditions) { - this.conditions = conditions; - this.aggregate = conditions.isEmpty() - ? new CompletableFuture<>() // Never completes if no conditions - : CompletableFuture.anyOf( - conditions.stream().map(Condition::execute).toArray(CompletableFuture[]::new)); - - // Single permanent listener. This is the only Completion node ever - // attached to aggregate.stack while the aggregate is still pending - // -- subsequent pre-completion getFuture() calls do not touch the - // aggregate at all. - this.aggregate.whenComplete((result, throwable) -> { - List> snapshot; - synchronized (lock) { - if (pending == null) { - return; - } - // Copy under the lock: the ArrayList holds strong - // references so entries that survived GC to this point - // stay alive until we complete them below. - snapshot = new ArrayList<>(pending); - pending = null; + for (Condition condition : conditions) { + this.conditions.put(condition.getType(), condition); + } + arm(); + } + + /** + * Builds the aggregate future over the conditions. + */ + private void arm() { + synchronized (lock) { + CompletableFuture[] unreported = conditions.values().stream() + .filter(c -> !reported.contains(c.getType())) + .map(Condition::execute) + .toArray(CompletableFuture[]::new); + CompletableFuture newAggregate = unreported.length == 0 + ? new CompletableFuture<>() // Never completes if no conditions are unreported + : CompletableFuture.anyOf(unreported); + aggregate = newAggregate; + // Start a new set only if the previous aggregate drained it. Otherwise, the futures in it are still + // waiting, and the new aggregate completes them. + if (pending == null) { + pending = Collections.newSetFromMap(new WeakHashMap<>()); } - for (CompletableFuture cf : snapshot) { - if (throwable != null) { - cf.completeExceptionally(throwable); - } else { - cf.complete(result); + + // Single permanent listener. This is the only Completion node ever + // attached to aggregate.stack while the aggregate is still pending + // -- subsequent pre-completion getFuture() calls do not touch the + // aggregate at all. + newAggregate.whenComplete((result, throwable) -> { + List> snapshot; + synchronized (lock) { + // A newer aggregate replaced this one, and it completes the pending futures instead. + if (aggregate != newAggregate || pending == null) { + return; + } + if (throwable == null) { + reported.add(((Condition) result).getType()); + } + // Copy under the lock: the ArrayList holds strong + // references so entries that survived GC to this point + // stay alive until we complete them below. + snapshot = new ArrayList<>(pending); + pending = null; } - } - }); + for (CompletableFuture cf : snapshot) { + if (throwable != null) { + cf.completeExceptionally(throwable); + } else { + cf.complete(result); + } + } + }); + } } /** - * Returns a future that will complete when the underlying aggregate - * condition fires. Pre-completion, this is a fresh future per call; - * post-completion, this is the aggregate itself (already done). + * Returns a future that completes with a met condition that this method has not reported before. */ public CompletableFuture getFuture() { - if (aggregate.isDone()) { - return aggregate; - } - - CompletableFuture fresh = new CompletableFuture<>(); synchronized (lock) { if (pending == null) { - // Raced with aggregate completion between isDone() and - // the lock acquisition; aggregate is now done. - return aggregate; + // The aggregate has fired. + if (aggregate.isCompletedExceptionally()) { + return aggregate; + } + // Wait for the conditions that have not been reported. + arm(); + if (pending == null) { + // Another condition was already met, so the new aggregate has fired too. + return aggregate; + } } + CompletableFuture fresh = new CompletableFuture<>(); pending.add(fresh); + return fresh; + } + } + + /** + * True if the condition of the given type has been reported as met. + * + * @param type the type of condition + * @return true if the condition is met + */ + public boolean isMet(Condition.ConditionType type) { + synchronized (lock) { + return reported.contains(type); + } + } + + /** + * Closes the condition of the given type, if there is one, and puts the replacement in its place. + * + * @param type the type of condition + * @param replacement the new condition, or null to leave no condition of that type + */ + public void replace(Condition.ConditionType type, Condition replacement) { + synchronized (lock) { + Condition old = conditions.remove(type); + if (old != null) { + old.close(); + } + reported.remove(type); + if (replacement != null) { + conditions.put(type, replacement); + } + arm(); } - return fresh; } public void inform(FDv2SourceResult result) { - conditions.forEach(c -> c.inform(result)); + synchronized (lock) { + conditions.values().forEach(c -> c.inform(result)); + } } @Override public void close() { - conditions.forEach(Condition::close); synchronized (lock) { + conditions.values().forEach(Condition::close); if (pending != null) { pending.clear(); } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java index f7f078c8..464138aa 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java @@ -7,10 +7,12 @@ import java.io.Closeable; import java.io.IOException; import java.time.Duration; +import java.time.Instant; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; /** * Container class for FDv2 data source conditions and related types. @@ -32,6 +34,7 @@ interface Condition extends Closeable { enum ConditionType { FALLBACK, RECOVERY, + UNPARK, } CompletableFuture execute(); @@ -44,6 +47,11 @@ enum ConditionType { } interface ConditionFactory { + /** + * Builds the condition. + * + * @return the condition, or null if the condition does not apply + */ Condition build(); Condition.ConditionType getType(); @@ -181,4 +189,54 @@ public ConditionType getType() { return ConditionType.RECOVERY; } } + + /** + * This condition is met when its delay has passed. + */ + static class UnparkCondition extends TimedCondition { + /** + * Builds a condition that is met at the earliest unpark time, or no condition if no synchronizer is parked. + */ + static class Factory implements ConditionFactory { + private final ScheduledExecutorService sharedExecutor; + private final Supplier earliestUnparkTime; + + public Factory(ScheduledExecutorService sharedExecutor, Supplier earliestUnparkTime) { + this.sharedExecutor = sharedExecutor; + this.earliestUnparkTime = earliestUnparkTime; + } + + @Override + public Condition build() { + Instant unparkTime = earliestUnparkTime.get(); + if (unparkTime == null) { + return null; + } + return new UnparkCondition(sharedExecutor, Duration.between(Instant.now(), unparkTime)); + } + + @Override + public ConditionType getType() { + return ConditionType.UNPARK; + } + } + + public UnparkCondition(ScheduledExecutorService sharedExecutor, Duration delay) { + super(sharedExecutor, delay); + timerFuture = sharedExecutor.schedule(() -> { + resultFuture.complete(this); + return null; + }, delay.toMillis(), TimeUnit.MILLISECONDS); + } + + @Override + public void inform(FDv2SourceResult sourceResult) { + // Time-based. + } + + @Override + public ConditionType getType() { + return ConditionType.UNPARK; + } + } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java index 64ad3b63..8d0ee72b 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java @@ -74,6 +74,18 @@ public void fdv1Fallback() { } } + /** + * Stops parking every synchronizer whose unpark time has arrived. + *

+ * Note: This is an internal method that must be called while holding activeSourceLock. + */ + private void unparkExpiredSynchronizers() { + Instant now = Instant.now(); + for (SynchronizerFactoryWithState factory : synchronizers) { + factory.unparkIfExpired(now); + } + } + /** * Get the next synchronizer to use. This operates based on tracking the index of the currently active synchronizer, * which will loop through all available synchronizers handling interruptions. Then a non-prime synchronizer recovers @@ -89,7 +101,7 @@ public void fdv1Fallback() { */ private SynchronizerFactoryWithState getNextAvailableSynchronizer() { SynchronizerFactoryWithState factory = null; - Instant now = Instant.now(); + unparkExpiredSynchronizers(); int visited = 0; while(visited < synchronizers.size()) { @@ -103,7 +115,7 @@ private SynchronizerFactoryWithState getNextAvailableSynchronizer() { } SynchronizerFactoryWithState candidate = synchronizers.get(synchronizerIndex); - if (candidate.isEligible(now)) { + if (candidate.isEligible()) { factory = candidate; break; } @@ -186,6 +198,12 @@ public int incrementCurrentSynchronizerParkCount() { } } + public void resetCurrentSynchronizerParkCount() { + synchronized (activeSourceLock) { + currentSynchronizerFactory.resetParkCount(); + } + } + public void parkCurrentSynchronizer(Instant unparkTime) { synchronized (activeSourceLock) { currentSynchronizerFactory.park(unparkTime); @@ -195,7 +213,7 @@ public void parkCurrentSynchronizer(Instant unparkTime) { /** * The earliest unpark time of the available synchronizers. * - * @return the earliest unpark time, or null if no available synchronizer was ever parked + * @return the earliest unpark time, or null if no available synchronizer is parked */ public Instant getEarliestUnparkTime() { synchronized (activeSourceLock) { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java index d8a95f96..6ee885bd 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java @@ -84,19 +84,29 @@ public void park(Instant unparkTime) { /** * The time at which the synchronizer stops being parked. * - * @return the unpark time, or null if the synchronizer was never parked + * @return the unpark time, or null if the synchronizer is not parked */ public Instant getUnparkTime() { return unparkTime; } /** - * True if the synchronizer is available and is not parked at the given time. + * Stops parking the synchronizer if its unpark time has arrived. * * @param now the current time + */ + public void unparkIfExpired(Instant now) { + if (unparkTime != null && !now.isBefore(unparkTime)) { + unparkTime = null; + } + } + + /** + * True if the synchronizer is available and is not parked. + * * @return true if the synchronizer can be started */ - public boolean isEligible(Instant now) { - return state == State.Available && (unparkTime == null || !now.isBefore(unparkTime)); + public boolean isEligible() { + return state == State.Available && unparkTime == null; } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java index e3e996a9..7bacf864 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java @@ -13,6 +13,7 @@ import java.time.Duration; import java.time.Instant; +import java.util.Arrays; import java.util.Collections; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -23,7 +24,9 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; /** @@ -136,7 +139,7 @@ public void allFreshFuturesCompleteWhenAggregateFires() throws Exception { @Test public void getFutureFailsExceptionallyWhenAggregateFailsExceptionally() throws Exception { - ManualCondition manualCondition = new ManualCondition(); + ManualCondition manualCondition = new ManualCondition(ConditionType.FALLBACK); try (FDv2DataSource.Conditions conditions = new FDv2DataSource.Conditions(Collections.singletonList(manualCondition))) { // Future requested BEFORE the exceptional completion. @@ -156,22 +159,103 @@ public void getFutureFailsExceptionallyWhenAggregateFailsExceptionally() } /** - * getFuture() called after the aggregate has already fired returns an - * already-completed future synchronously (the fast path). + * After getFuture() reports a met condition, the next getFuture() waits for another condition. */ @Test - public void getFutureAfterAggregateFiresReturnsCompletedFuture() throws Exception { - // RecoveryCondition arms its timer in the constructor and fires after - // the configured timeout. With timeout=0 it fires near-immediately. - Condition recovery = new RecoveryCondition(executor, Duration.ofSeconds(0)); + public void getFutureAfterConditionIsReportedWaitsForAnotherCondition() throws Exception { + ManualCondition fallback = new ManualCondition(ConditionType.FALLBACK); + ManualCondition recovery = new ManualCondition(ConditionType.RECOVERY); try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.singletonList(recovery))) { - // Drain a future to confirm the aggregate has fired. - conditions.getFuture().get(2, TimeUnit.SECONDS); + new FDv2DataSource.Conditions(Arrays.asList(fallback, recovery))) { + // Meet the fallback condition. + CompletableFuture first = conditions.getFuture(); + fallback.future.complete(fallback); - CompletableFuture postFire = conditions.getFuture(); - assertTrue("post-fire getFuture() should be already complete", postFire.isDone()); - assertNotNull(postFire.get(0, TimeUnit.SECONDS)); + // The first future reports the fallback condition, and the next future waits. + assertSame(fallback, first.get(2, TimeUnit.SECONDS)); + CompletableFuture second = conditions.getFuture(); + assertFalse(second.isDone()); + + // Meet the recovery condition. + recovery.future.complete(recovery); + + // The second future reports the recovery condition. + assertSame(recovery, second.get(2, TimeUnit.SECONDS)); + } + } + + /** + * isMet() is true only for a condition that the aggregate has reported. + */ + @Test + public void isMetIsTrueOnlyForAReportedCondition() throws Exception { + ManualCondition fallback = new ManualCondition(ConditionType.FALLBACK); + try (FDv2DataSource.Conditions conditions = + new FDv2DataSource.Conditions(Collections.singletonList(fallback))) { + // Before the condition is met, it is not reported. + assertFalse(conditions.isMet(ConditionType.FALLBACK)); + + // Meet the condition, and wait for the aggregate to report it. + CompletableFuture future = conditions.getFuture(); + fallback.future.complete(fallback); + future.get(2, TimeUnit.SECONDS); + + // The condition is reported, and a type with no condition is not. + assertTrue(conditions.isMet(ConditionType.FALLBACK)); + assertFalse(conditions.isMet(ConditionType.RECOVERY)); + } + } + + /** + * replace() closes the old condition, and the next future waits for the replacement. + */ + @Test + public void replaceWaitsForTheReplacementCondition() throws Exception { + ManualCondition oldFallback = new ManualCondition(ConditionType.FALLBACK); + ManualCondition newFallback = new ManualCondition(ConditionType.FALLBACK); + try (FDv2DataSource.Conditions conditions = + new FDv2DataSource.Conditions(Collections.singletonList(oldFallback))) { + // Meet the old condition, and wait for the aggregate to report it. + CompletableFuture first = conditions.getFuture(); + oldFallback.future.complete(oldFallback); + first.get(2, TimeUnit.SECONDS); + + // Replace the old condition. + conditions.replace(ConditionType.FALLBACK, newFallback); + + // The old condition is closed, the type is no longer met, and the next future waits. + assertTrue(oldFallback.closed); + assertFalse(conditions.isMet(ConditionType.FALLBACK)); + CompletableFuture second = conditions.getFuture(); + assertFalse(second.isDone()); + + // Meet the replacement. + newFallback.future.complete(newFallback); + + // The next future reports the replacement. + assertSame(newFallback, second.get(2, TimeUnit.SECONDS)); + } + } + + /** + * replace() with null closes the old condition and leaves no condition of that type. + */ + @Test + public void replaceWithNullLeavesNoCondition() throws Exception { + ManualCondition fallback = new ManualCondition(ConditionType.FALLBACK); + try (FDv2DataSource.Conditions conditions = + new FDv2DataSource.Conditions(Collections.singletonList(fallback))) { + // Meet the condition, and wait for the aggregate to report it. + CompletableFuture first = conditions.getFuture(); + fallback.future.complete(fallback); + first.get(2, TimeUnit.SECONDS); + + // Replace the condition with nothing. + conditions.replace(ConditionType.FALLBACK, null); + + // The condition is closed, and the next future waits. + assertTrue(fallback.closed); + assertFalse(conditions.getFuture().isDone()); } } @@ -215,6 +299,12 @@ private static void assertThrowsExecutionExceptionWithCause( private static final class ManualCondition implements Condition { final CompletableFuture future = new CompletableFuture<>(); + private final ConditionType type; + volatile boolean closed = false; + + ManualCondition(ConditionType type) { + this.type = type; + } @Override public CompletableFuture execute() { @@ -229,11 +319,12 @@ public void inform(FDv2SourceResult sourceResult) { @Override public void close() { // No timer to cancel. + closed = true; } @Override public ConditionType getType() { - return ConditionType.FALLBACK; + return type; } } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index 39b55346..4a1b7a71 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -790,6 +790,44 @@ public void constructionFailureParksSynchronizer() throws Exception { assertTrue(dataSource.isInitialized()); } + @Test + public void resetThresholdMetReturnsParkDelayToBase() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + BlockingQueue buildTimes = new LinkedBlockingQueue<>(); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + buildTimes.offer(System.nanoTime()); + BlockingQueue results = new LinkedBlockingQueue<>(); + if (buildCount.incrementAndGet() == 3) { + // The third instance reports that it met its reset threshold before it fails. + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false).withResetThresholdMet()); + } + // Every instance fails with a terminal error. + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMillis(100), Duration.ofSeconds(10), Duration.ofMillis(10)); + resourcesToClose.add(dataSource); + + // Start the data system, and record when the synchronizer is built for the third and fourth times. + dataSource.start(); + buildTimes.poll(5, TimeUnit.SECONDS); + buildTimes.poll(5, TimeUnit.SECONDS); + Long thirdBuild = buildTimes.poll(5, TimeUnit.SECONDS); + Long fourthBuild = buildTimes.poll(5, TimeUnit.SECONDS); + + // After the reset, the park is at most the base delay. Without the reset, it would be at least twice the base. + assertNotNull(fourthBuild); + long parkMillis = TimeUnit.NANOSECONDS.toMillis(fourthBuild - thirdBuild); + assertTrue("park lasted " + parkMillis + "ms", parkMillis < 200); + } + @Test public void allSynchronizersParkedRestartsAfterEarliestUnparkTime() throws Exception { executor = Executors.newScheduledThreadPool(2); From 27589fa3dce880c681d2a049b4247d7027c85a87 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Wed, 4 Nov 2026 21:35:53 -0800 Subject: [PATCH 04/13] feat: Keep the current synchronizer when no other synchronizer is available --- .../sdk/server/FDv2DataSource.java | 130 ++- .../sdk/server/FDv2DataSourceConditions.java | 27 +- .../sdk/server/SourceManager.java | 47 +- ...FDv2DataSourceConditionsAggregateTest.java | 3 +- .../FDv2DataSourceFallbackConditionTest.java | 128 +-- .../sdk/server/FDv2DataSourceTest.java | 850 ++++++++++++++---- 6 files changed, 853 insertions(+), 332 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index c0357aec..56ea7615 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -61,7 +61,8 @@ class FDv2DataSource implements DataSource { private final SourceManager sourceManager; - private final List conditionFactories; + private final Map conditionFactories = + new EnumMap<>(Condition.ConditionType.class); private final Duration parkInitialDelay; @@ -156,14 +157,12 @@ public FDv2DataSource( this.dataSourceUpdates = dataSourceUpdates; this.threadPriority = threadPriority; this.logger = logger; - this.conditionFactories = new ArrayList<>(); - this.conditionFactories.add(new FallbackCondition.Factory(sharedExecutor, fallbackTimeout)); - this.conditionFactories.add(new RecoveryCondition.Factory(sharedExecutor, recoveryTimeout)); - // getEarliestUnparkTime synchronizes on the lock of the source manager, so it is safe to call from the thread - // that builds the conditions. - // TODO: Decide whether to keep this guarantee, or to require that conditions are built only on the data - // source thread. - this.conditionFactories.add(new UnparkCondition.Factory(sharedExecutor, sourceManager::getEarliestUnparkTime)); + this.conditionFactories.put( + Condition.ConditionType.FALLBACK, new FallbackCondition.Factory(sharedExecutor, fallbackTimeout)); + this.conditionFactories.put( + Condition.ConditionType.RECOVERY, new RecoveryCondition.Factory(sharedExecutor, recoveryTimeout)); + this.conditionFactories.put( + Condition.ConditionType.UNPARK, new UnparkCondition.Factory(sharedExecutor, sourceManager::getEarliestUnparkTime)); this.parkInitialDelay = parkInitialDelay; this.parkMaxDelay = parkMaxDelay; this.failoverDelay = failoverDelay; @@ -376,15 +375,26 @@ private List getConditions() { } if (isPrimeSynchronizer) { // If there isn't a synchronizer to recover to, then don't add and recovery conditions. - return conditionFactories.stream() + return conditionFactories.values().stream() .filter((ConditionFactory factory) -> factory.getType() != Condition.ConditionType.RECOVERY) .map(ConditionFactory::build).filter(Objects::nonNull).collect(Collectors.toList()); } // The synchronizer can both fall back and recover. - return conditionFactories.stream() + return conditionFactories.values().stream() .map(ConditionFactory::build).filter(Objects::nonNull).collect(Collectors.toList()); } + /** + * Builds the condition of the given type. + * + * @param type the type of condition + * @return the condition, or null if it does not apply + */ + private Condition buildCondition(Condition.ConditionType type) { + ConditionFactory factory = conditionFactories.get(type); + return factory == null ? null : factory.build(); + } + /** * Runs the configured synchronizers, falling back / recovering as conditions allow, * until the list is exhausted, the data source is closed, or a server-directed FDv1 @@ -410,45 +420,77 @@ private boolean runSynchronizers() { boolean running = true; try (Conditions conditions = new Conditions(getConditions())) { + // A condition doesn't necessarily stop the current synchronizer, so keep track of its pending result. + CompletableFuture nextResultFuture = null; while (running) { - CompletableFuture nextResultFuture = synchronizer.next(); + if (nextResultFuture == null) { + nextResultFuture = synchronizer.next(); + } - // The conditionsFuture will complete if any condition is met. Meeting any condition means we will - // switch to a different synchronizer. + // The conditions future completes when a condition is met. Object res = CompletableFuture.anyOf(conditions.getFuture(), nextResultFuture).get(); if (res instanceof Condition) { - Condition c = (Condition) res; - switch (c.getType()) { - case FALLBACK: - logger.info( - "Fallback condition met, falling back from synchronizer '{}'.", - synchronizer.name() - ); - break; - case RECOVERY: - // For recovery, we will start at the first available synchronizer. - // So we reset the source index, and finding the source will start at the beginning. - sourceManager.resetSourceIndex(); - logger.info( - "Recovery condition met, moving from synchronizer '{}' to primary synchronizer.", - synchronizer.name() - ); - break; + Condition.ConditionType metType = ((Condition) res).getType(); + // Clear expired unpark times before checking which synchronizers are available. + sourceManager.unparkExpiredSynchronizers(); + if (metType == Condition.ConditionType.UNPARK) { + // Replace the unpark condition with one for the next unpark. + conditions.replace( + Condition.ConditionType.UNPARK, + buildCondition(Condition.ConditionType.UNPARK)); } - // A running synchronizer will only have fallback and recovery conditions that it can act on. - // So, if there are no synchronizers to recover to or fallback to, then we will not have - // those conditions. - break; + + // An unpark does not switch synchronizers by itself, but other conditions might already + // be met. Check all of them before deciding whether to switch. Recovery takes precedence. + if (conditions.isMet(Condition.ConditionType.RECOVERY) + && sourceManager.hasMorePreferredAvailableSynchronizer()) { + // For recovery, we will start at the first available synchronizer. + // So we reset the source index, and finding the source will start at the beginning. + sourceManager.resetSourceIndex(); + logger.info( + "Recovery condition met, moving from synchronizer '{}' to primary synchronizer.", + synchronizer.name() + ); + break; + } + if (conditions.isMet(Condition.ConditionType.FALLBACK) + && sourceManager.hasOtherAvailableSynchronizer()) { + logger.info( + "Fallback condition met, falling back from synchronizer '{}'.", + synchronizer.name() + ); + break; + } + + // Keep running this synchronizer. + if (metType == Condition.ConditionType.FALLBACK) { + logger.info( + "Fallback condition met, but no other synchronizer is available; continuing with synchronizer '{}'.", + synchronizer.name() + ); + } else if (metType == Condition.ConditionType.RECOVERY) { + logger.info( + "Recovery condition met, but no more-preferred synchronizer is available; continuing with synchronizer '{}'.", + synchronizer.name() + ); + } + continue; } + // The pending result has arrived. + nextResultFuture = null; + if (!(res instanceof FDv2SourceResult)) { logger.error("Unexpected result type from synchronizer: {}", res.getClass().getName()); continue; } try (FDv2SourceResult result = (FDv2SourceResult) res) { - conditions.inform(result); + Set typesToReplace = conditions.inform(result); + for (Condition.ConditionType type : typesToReplace) { + conditions.replace(type, buildCondition(type)); + } if (result.isResetThresholdMet()) { sourceManager.resetCurrentSynchronizerParkCount(); @@ -923,9 +965,21 @@ public void replace(Condition.ConditionType type, Condition replacement) { } } - public void inform(FDv2SourceResult result) { + /** + * Informs the conditions of a result from the synchronizer. + * + * @param result the result + * @return the types of the conditions that should be replaced with new ones + */ + public Set inform(FDv2SourceResult result) { synchronized (lock) { - conditions.values().forEach(c -> c.inform(result)); + Set typesToReplace = EnumSet.noneOf(Condition.ConditionType.class); + for (Condition condition : conditions.values()) { + if (condition.inform(result)) { + typesToReplace.add(condition.getType()); + } + } + return typesToReplace; } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java index 464138aa..f042678a 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java @@ -39,7 +39,13 @@ enum ConditionType { CompletableFuture execute(); - void inform(FDv2SourceResult sourceResult); + /** + * Informs the condition of a result from the synchronizer. + * + * @param sourceResult the result + * @return true if the condition should be replaced with a new one + */ + boolean inform(FDv2SourceResult sourceResult); void close(); @@ -130,12 +136,10 @@ public FallbackCondition(ScheduledExecutorService sharedExecutor, Duration timeo } @Override - public void inform(FDv2SourceResult sourceResult) { + public boolean inform(FDv2SourceResult sourceResult) { if (sourceResult.getResultType() == SourceResultType.CHANGE_SET) { - if (timerFuture != null) { - timerFuture.cancel(false); - timerFuture = null; - } + // The synchronizer is healthy. If the timer has started, the condition must start over. + return timerFuture != null; } if (sourceResult.getResultType() == SourceResultType.STATUS && sourceResult.getStatus().getState() == SourceSignal.INTERRUPTED) { if (timerFuture == null) { @@ -145,6 +149,7 @@ public void inform(FDv2SourceResult sourceResult) { }, timeout.toMillis(), TimeUnit.MILLISECONDS); } } + return false; } @Override @@ -180,8 +185,9 @@ public RecoveryCondition(ScheduledExecutorService sharedExecutor, Duration timeo } @Override - public void inform(FDv2SourceResult sourceResult) { - // Time-based recovery. + public boolean inform(FDv2SourceResult sourceResult) { + // Recovery is time-based, so a result never requires a new condition. + return false; } @Override @@ -230,8 +236,9 @@ public UnparkCondition(ScheduledExecutorService sharedExecutor, Duration delay) } @Override - public void inform(FDv2SourceResult sourceResult) { - // Time-based. + public boolean inform(FDv2SourceResult sourceResult) { + // An unpark is time-based, so a result never requires a new condition. + return false; } @Override diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java index 8d0ee72b..859056d9 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java @@ -76,13 +76,13 @@ public void fdv1Fallback() { /** * Stops parking every synchronizer whose unpark time has arrived. - *

- * Note: This is an internal method that must be called while holding activeSourceLock. */ - private void unparkExpiredSynchronizers() { - Instant now = Instant.now(); - for (SynchronizerFactoryWithState factory : synchronizers) { - factory.unparkIfExpired(now); + public void unparkExpiredSynchronizers() { + synchronized (activeSourceLock) { + Instant now = Instant.now(); + for (SynchronizerFactoryWithState factory : synchronizers) { + factory.unparkIfExpired(now); + } } } @@ -269,6 +269,41 @@ public Initializer getNextInitializerAndSetActive() { } } + /** + * True if a synchronizer other than the current one is available. + * + * @return true if another synchronizer is available + */ + public boolean hasOtherAvailableSynchronizer() { + synchronized (activeSourceLock) { + for (SynchronizerFactoryWithState factory : synchronizers) { + if (factory != currentSynchronizerFactory && factory.isEligible()) { + return true; + } + } + return false; + } + } + + /** + * True if a synchronizer earlier in the list than the current one is available. + * + * @return true if a more-preferred synchronizer is available + */ + public boolean hasMorePreferredAvailableSynchronizer() { + synchronized (activeSourceLock) { + for (SynchronizerFactoryWithState factory : synchronizers) { + if (factory == currentSynchronizerFactory) { + return false; + } + if (factory.isEligible()) { + return true; + } + } + return false; + } + } + /** * Determine if the currently active synchronizer is the prime (first available) synchronizer. * @return true if the current synchronizer is the prime synchronizer, false otherwise diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java index 7bacf864..c625f211 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java @@ -312,8 +312,9 @@ public CompletableFuture execute() { } @Override - public void inform(FDv2SourceResult sourceResult) { + public boolean inform(FDv2SourceResult sourceResult) { // Manually controlled; no auto-trigger from inform. + return false; } @Override diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java index a234069d..a48d960a 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java @@ -77,14 +77,15 @@ public void interruptedStatusStartsTimerThatCompletesResultFuture() throws Excep assertFalse(resultFuture.isDone()); // Inform with INTERRUPTED status - condition.inform( + boolean replace = condition.inform( FDv2SourceResult.interrupted( new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), false ) ); - // Future should still not be done immediately + // The condition does not ask to be replaced, and the future should still not be done immediately + assertFalse(replace); assertFalse(resultFuture.isDone()); // Wait for timeout to fire @@ -96,12 +97,10 @@ public void interruptedStatusStartsTimerThatCompletesResultFuture() throws Excep } @Test - public void changeSetCancelsActiveTimer() throws Exception { + public void changeSetWithActiveTimerRequestsReplacement() throws Exception { executor = Executors.newScheduledThreadPool(1); FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - CompletableFuture resultFuture = condition.execute(); - // Start timer with INTERRUPTED condition.inform( FDv2SourceResult.interrupted( @@ -110,19 +109,16 @@ public void changeSetCancelsActiveTimer() throws Exception { ) ); - // Cancel timer with CHANGE_SET + // Inform with CHANGE_SET while the timer is running ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // Wait longer than the timeout period - Thread.sleep(1500); + boolean replace = condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - // Future should still not be complete (timer was cancelled) - assertFalse(resultFuture.isDone()); + // The condition asks to be replaced + assertTrue(replace); } @Test - public void changeSetWithoutActiveTimerDoesNothing() throws Exception { + public void changeSetWithoutActiveTimerDoesNotRequestReplacement() throws Exception { executor = Executors.newScheduledThreadPool(1); FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); @@ -130,12 +126,10 @@ public void changeSetWithoutActiveTimerDoesNothing() throws Exception { // Inform with CHANGE_SET without starting a timer first ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); + boolean replace = condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - // Wait to ensure nothing happens - Thread.sleep(100); - - // Future should still not be complete + // The condition does not ask to be replaced, and is not met + assertFalse(replace); assertFalse(resultFuture.isDone()); } @@ -272,41 +266,7 @@ public void closeWithoutActiveTimerDoesNotFail() throws Exception { } @Test - public void timerCanBeStartedAfterBeingCancelled() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Start timer - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Cancel timer with CHANGE_SET - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // Start timer again - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Wait for second timer to fire - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void changeSetAfterTimerFiresDoesNotAffectCompletedFuture() throws Exception { + public void changeSetAfterTimerFiresRequestsReplacement() throws Exception { executor = Executors.newScheduledThreadPool(1); FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); @@ -327,9 +287,10 @@ public void changeSetAfterTimerFiresDoesNotAffectCompletedFuture() throws Except // Inform with CHANGE_SET after timer has fired ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); + boolean replace = condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - // Future should remain complete + // The condition asks to be replaced, and its future remains complete + assertTrue(replace); assertTrue(resultFuture.isDone()); } @@ -367,63 +328,6 @@ public void executeReturnsTheSameFutureOnMultipleCalls() { assertSame(first, second); } - @Test - public void changeSetDuringTimerExecutionCancelsTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Start timer - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Wait partway through timeout period - Thread.sleep(500); - - // Cancel with CHANGE_SET - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // Wait past the original timeout - Thread.sleep(1000); - - // Future should still not be complete - assertFalse(resultFuture.isDone()); - } - - @Test - public void multipleChangeSetCallsWithActiveTimerAreHandled() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Start timer - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Cancel with multiple CHANGE_SETs - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // Wait longer than timeout - Thread.sleep(1500); - - // Future should still not be complete - assertFalse(resultFuture.isDone()); - } - @Test public void closeCanBeCalledMultipleTimes() throws Exception { executor = Executors.newScheduledThreadPool(1); diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index 4a1b7a71..e18e4610 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -118,7 +118,11 @@ public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -167,7 +171,11 @@ public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() thro ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -217,7 +225,11 @@ public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() th ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -263,7 +275,11 @@ public void allInitializersFailSwitchesToSynchronizers() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -306,7 +322,11 @@ public void allThreeInitializersFailWithNoSynchronizers() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -347,7 +367,11 @@ public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -383,7 +407,11 @@ public void noInitializersOneSynchronizerIsWellBehaved() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -418,7 +446,11 @@ public void oneInitializerOneSynchronizerIsWellBehaved() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -440,7 +472,11 @@ public void noInitializersAndNoSynchronizersIsWellBehaved() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -480,7 +516,11 @@ public void errorWithFDv1FallbackTriggersFallback() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -525,7 +565,11 @@ public void fallbackAndRecoveryTasksWellBehaved() throws Exception { // Use short timeouts for testing FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -579,7 +623,11 @@ public void canDisposeWhenSynchronizersFallingBack() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -627,7 +675,11 @@ public void terminalErrorFailsOverToNextSynchronizer() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -680,7 +732,11 @@ public void terminalErrorParksSynchronizerAndRestartsItAfterUnparkTime() throws ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMillis(100), Duration.ofSeconds(1), Duration.ofMillis(10)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10)); resourcesToClose.add(dataSource); // Start the data system with its only synchronizer. @@ -710,7 +766,11 @@ public void failedNextParksSynchronizer() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, testLogger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMillis(100), Duration.ofSeconds(1), Duration.ofMillis(10)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10)); resourcesToClose.add(dataSource); // Start the data system with its only synchronizer. @@ -749,7 +809,11 @@ public void close() { ); FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMillis(100), Duration.ofSeconds(1), Duration.ofMillis(10)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10)); resourcesToClose.add(dataSource); // Start the data system with its only synchronizer. @@ -779,7 +843,11 @@ public void constructionFailureParksSynchronizer() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMillis(100), Duration.ofSeconds(1), Duration.ofMillis(10)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10)); resourcesToClose.add(dataSource); // Start the data system with its only synchronizer. @@ -812,7 +880,11 @@ public void resetThresholdMetReturnsParkDelayToBase() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMillis(100), Duration.ofSeconds(10), Duration.ofMillis(10)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(10), + /* failoverDelay= */ Duration.ofMillis(10)); resourcesToClose.add(dataSource); // Start the data system, and record when the synchronizer is built for the third and fourth times. @@ -828,6 +900,266 @@ public void resetThresholdMetReturnsParkDelayToBase() throws Exception { assertTrue("park lasted " + parkMillis + "ms", parkMillis < 200); } + @Test + public void fallbackWithNoOtherAvailableSynchronizerKeepsCurrentSynchronizer() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger secondaryBuildCount = new AtomicInteger(); + IterableAsyncQueue secondaryResults = new IterableAsyncQueue<>(); + secondaryResults.put(makeInterruptedResult()); + + ImmutableList> synchronizers = ImmutableList.of( + // The primary synchronizer fails, so it is parked for the rest of the test. + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + }, + // The first instance of the secondary synchronizer reports that it is interrupted. + () -> secondaryBuildCount.incrementAndGet() == 1 + ? new MockQueuedSynchronizer(secondaryResults) + : new MockQueuedSynchronizer(new IterableAsyncQueue<>()) + ); + + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofMillis(10), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofMillis(10)); + resourcesToClose.add(dataSource); + + // Start the data system. After the fallback condition of the secondary synchronizer is met, send a change set + // to the instance of the secondary synchronizer that was already running. + dataSource.start(); + executor.schedule( + () -> secondaryResults.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 60, + TimeUnit.MILLISECONDS); + + // The data system applies the change set, so it kept that instance. + sink.awaitApplyCount(1, 2, TimeUnit.SECONDS); + assertEquals(1, sink.getApplyCount()); + assertEquals(1, secondaryBuildCount.get()); + } + + @Test + public void unparkWhileCurrentSynchronizerIsHealthyKeepsIt() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger primaryBuildCount = new AtomicInteger(); + IterableAsyncQueue secondaryResults = new IterableAsyncQueue<>(); + secondaryResults.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + + ImmutableList> synchronizers = ImmutableList.of( + // The primary synchronizer fails, so it is parked briefly. + () -> { + primaryBuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + }, + // The secondary synchronizer delivers data and stays healthy. + () -> new MockQueuedSynchronizer(secondaryResults) + ); + + // The primary synchronizer is parked for 25-50ms. The fallback and recovery timeouts are too long to matter. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(50), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10)); + resourcesToClose.add(dataSource); + + // Start the data system, and send another change set after the primary synchronizer unparks. + dataSource.start(); + executor.schedule( + () -> secondaryResults.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 100, + TimeUnit.MILLISECONDS); + + // The data system applies both change sets, and does not start the primary synchronizer again. + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); + assertEquals(1, primaryBuildCount.get()); + } + + @Test + public void fallbackSwitchesWhenParkedSynchronizerUnparks() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger primaryBuildCount = new AtomicInteger(); + AtomicInteger secondaryBuildCount = new AtomicInteger(); + + ImmutableList> synchronizers = ImmutableList.of( + // The first instance of the primary synchronizer fails, and the second instance delivers data. + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(primaryBuildCount.incrementAndGet() == 1 + ? makeTerminalErrorResult() + : FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); + }, + // The secondary synchronizer reports that it is interrupted. + () -> { + secondaryBuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeInterruptedResult()); + return new MockQueuedSynchronizer(results); + } + ); + + // The fallback condition is met after 10ms, and the primary synchronizer unparks after 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofMillis(10), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // When the primary synchronizer unparks, the data system falls back to it and applies its data. + sink.awaitApplyCount(1, 2, TimeUnit.SECONDS); + assertEquals(2, primaryBuildCount.get()); + assertEquals(1, secondaryBuildCount.get()); + } + + @Test + public void changeSetAfterFallbackIsMetPreventsSwitchAtUnpark() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger primaryBuildCount = new AtomicInteger(); + IterableAsyncQueue secondaryResults = new IterableAsyncQueue<>(); + secondaryResults.put(makeInterruptedResult()); + + ImmutableList> synchronizers = ImmutableList.of( + // The primary synchronizer fails, so it is parked briefly. + () -> { + primaryBuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + }, + // The secondary synchronizer reports that it is interrupted. + () -> new MockQueuedSynchronizer(secondaryResults) + ); + + // The fallback condition is met after 10ms, and the primary synchronizer unparks after 80-160ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofMillis(10), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(160), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10)); + resourcesToClose.add(dataSource); + + // Start the data system, send a change set after the fallback is met, and another after the unpark. + dataSource.start(); + executor.schedule( + () -> secondaryResults.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 50, + TimeUnit.MILLISECONDS); + executor.schedule( + () -> secondaryResults.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 210, + TimeUnit.MILLISECONDS); + + // The data system applies both change sets, and does not start the primary synchronizer again. + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); + assertEquals(1, primaryBuildCount.get()); + } + + @Test + public void recoveryWithPreferredSynchronizerParkedWaitsForItsUnpark() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger primaryBuildCount = new AtomicInteger(); + AtomicInteger secondaryBuildCount = new AtomicInteger(); + + ImmutableList> synchronizers = ImmutableList.of( + // The first instance of the primary synchronizer fails, and the second instance delivers data. + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(primaryBuildCount.incrementAndGet() == 1 + ? makeTerminalErrorResult() + : FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); + }, + // The secondary synchronizer delivers data and stays healthy. + () -> { + secondaryBuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); + } + ); + + // The recovery condition is met after 50ms, and the primary synchronizer unparks after 100-200ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofMillis(50), + /* parkInitialDelay= */ Duration.ofMillis(200), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // When the primary synchronizer unparks, the data system recovers to it and applies its data. + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertEquals(2, primaryBuildCount.get()); + assertEquals(1, secondaryBuildCount.get()); + } + + @Test + public void changeSetDuringInterruptionRestartsFallbackTimeout() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + IterableAsyncQueue primaryResults = new IterableAsyncQueue<>(); + primaryResults.put(makeInterruptedResult()); + BlockingQueue secondaryBuildTimes = new LinkedBlockingQueue<>(); + + ImmutableList> synchronizers = ImmutableList.of( + // The primary synchronizer reports that it is interrupted. + () -> new MockQueuedSynchronizer(primaryResults), + // The secondary synchronizer records when it is built. + () -> { + secondaryBuildTimes.offer(System.nanoTime()); + return new MockQueuedSynchronizer(new IterableAsyncQueue<>()); + } + ); + + // The fallback timeout is 100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofMillis(100), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofMillis(10)); + resourcesToClose.add(dataSource); + + // Start the data system. Before the fallback timeout, the primary delivers a change set and is interrupted again. + long start = System.nanoTime(); + dataSource.start(); + executor.schedule( + () -> primaryResults.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 50, + TimeUnit.MILLISECONDS); + executor.schedule(() -> primaryResults.put(makeInterruptedResult()), 90, TimeUnit.MILLISECONDS); + + // The data system falls back a full timeout after the second interruption, not at the original deadline. + Long secondaryBuild = secondaryBuildTimes.poll(2, TimeUnit.SECONDS); + assertNotNull(secondaryBuild); + long fallbackMillis = TimeUnit.NANOSECONDS.toMillis(secondaryBuild - start); + assertTrue("fell back after " + fallbackMillis + "ms", fallbackMillis >= 150); + } + @Test public void allSynchronizersParkedRestartsAfterEarliestUnparkTime() throws Exception { executor = Executors.newScheduledThreadPool(2); @@ -859,7 +1191,11 @@ public void allSynchronizersParkedRestartsAfterEarliestUnparkTime() throws Excep ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMillis(100), Duration.ofSeconds(1), Duration.ofMillis(10)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10)); resourcesToClose.add(dataSource); // Start the data system, and wait for each synchronizer to be built and then for one more build. @@ -914,7 +1250,11 @@ public void disabledDataSourceCannotTriggerActions() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -961,7 +1301,11 @@ public void disposeCompletesStartFuture() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -989,7 +1333,11 @@ public void noSourcesProvidedCompletesImmediately() throws Exception{ ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1030,7 +1378,11 @@ public void startFutureCompletesExactlyOnce() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1054,7 +1406,11 @@ public void concurrentCloseAndStartHandledSafely() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); Future startFuture = dataSource.start(); @@ -1079,7 +1435,11 @@ public void multipleStartCallsEventuallyComplete() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture1 = dataSource.start(); @@ -1108,7 +1468,11 @@ public void isInitializedThreadSafe() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); dataSource.start(); @@ -1144,7 +1508,11 @@ public void dataSourceUpdatesApplyThreadSafe() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1181,7 +1549,11 @@ public void initializerThrowsExecutionException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1214,7 +1586,11 @@ public void initializerThrowsInterruptedException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1245,7 +1621,11 @@ public void initializerThrowsCancellationException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1275,7 +1655,11 @@ public void synchronizerNextThrowsExecutionException() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1308,7 +1692,11 @@ public void synchronizerNextThrowsInterruptedException() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1339,7 +1727,11 @@ public void synchronizerNextThrowsCancellationException() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1362,7 +1754,11 @@ public void closeWithoutStartDoesNotHang() { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); dataSource.close(); @@ -1385,7 +1781,11 @@ public void closeAfterInitializersCompletesImmediately() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1417,7 +1817,11 @@ public void close() { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1443,7 +1847,11 @@ public void multipleCloseCallsAreIdempotent() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1470,7 +1878,11 @@ public void closingDataSourceDuringInitializationReportsOffWithoutErrors() throw ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1511,7 +1923,11 @@ public void dataSourceClosedDuringSynchronizationReportsOffWithoutError() throws ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1549,7 +1965,11 @@ public void closeInterruptsConditionWaiting() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1588,7 +2008,11 @@ public CompletableFuture run() { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); dataSource.close(); Future startFuture = dataSource.start(); @@ -1626,7 +2050,11 @@ public void close() { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1661,7 +2089,11 @@ public void close() { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1694,7 +2126,11 @@ public void setActiveSourceOnInitializerChecksShutdown() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); Future startFuture = dataSource.start(); @@ -1756,7 +2192,11 @@ public void parkedSynchronizerSkippedInRotation() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1798,7 +2238,11 @@ public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { // Short recovery timeout FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1839,7 +2283,11 @@ public void fallbackMovesToNextSynchronizer() throws Exception { // Short fallback timeout FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(1), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1870,7 +2318,11 @@ public void conditionsClosedAfterSynchronizerLoop() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1898,7 +2350,11 @@ public void conditionsInformedOfAllResults() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(10), Duration.ofSeconds(20), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(10), + /* recoveryTimeout= */ Duration.ofSeconds(20), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1927,7 +2383,11 @@ public void conditionsClosedOnException() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1953,7 +2413,11 @@ public void primeSynchronizerHasNoRecoveryCondition() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1985,7 +2449,11 @@ public void nonPrimeSynchronizerHasBothConditions() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2010,7 +2478,11 @@ public void singleSynchronizerHasNoConditions() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2036,7 +2508,11 @@ public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2067,7 +2543,11 @@ public void changeSetAppliedToDataSourceUpdates() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2094,7 +2574,11 @@ public void multipleChangeSetsAppliedInOrder() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2128,7 +2612,11 @@ public void selectorNonEmptyCompletesInitialization() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, ImmutableList.of(), null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2158,7 +2646,11 @@ public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() thro ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2192,7 +2684,11 @@ public void synchronizerChangeSetAlwaysCompletesStartFuture() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2222,7 +2718,11 @@ public void goodbyeStatusHandledGracefully() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2257,7 +2757,11 @@ public void shutdownStatusExitsImmediately() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2285,7 +2789,11 @@ public void fdv1FallbackFlagHonored() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2319,7 +2827,11 @@ public void emptyInitializerListSkipsToSynchronizers() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2348,7 +2860,11 @@ public void startedFlagPreventsMultipleRuns() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); Future startFuture1 = dataSource.start(); @@ -2379,7 +2895,11 @@ public void startBeforeRunCompletesAllComplete() throws Exception { ); FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), Duration.ofSeconds(300), Duration.ofMinutes(5), Duration.ofHours(1), Duration.ofSeconds(1)); + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1)); resourcesToClose.add(dataSource); // Call start multiple times before completion @@ -2418,11 +2938,11 @@ public void statusTransitionsToValidAfterInitialization() throws Exception { Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2467,11 +2987,11 @@ public void statusIncludesErrorInfoOnFailure() throws Exception { Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2511,11 +3031,11 @@ public void statusRemainsValidDuringSynchronizerOperation() throws Exception { Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2570,11 +3090,11 @@ public void statusStaysInterruptedWhileOnlySynchronizerIsParked() throws Excepti Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMillis(100), - Duration.ofSeconds(1), - Duration.ofMillis(10) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10) ); resourcesToClose.add(dataSource); @@ -2631,11 +3151,11 @@ public void fdv1FallbackActivatesWhenFlagSet() throws Exception { Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2682,11 +3202,11 @@ public void fdv1FallbackNotCalledWithoutFlag() throws Exception { Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2743,11 +3263,11 @@ public void fdv1FallbackWorksAfterInterruption() throws Exception { Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2803,11 +3323,11 @@ public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredHaltsDataSystem() thr Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2882,11 +3402,11 @@ public void fdv1FallbackBlocksOtherSynchronizers() throws Exception { Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -2937,11 +3457,11 @@ public void fdv1FallbackOnlyCalledOncePerDataSource() throws Exception { Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -3002,11 +3522,11 @@ public void fdv1FallbackOnInitializerSuccessAppliesPayloadAndSwitches() throws E Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -3072,11 +3592,11 @@ public void fdv1FallbackOnInitializerErrorSwitchesToFDv1() throws Exception { Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -3127,11 +3647,11 @@ public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredTransitionsToOff() thr Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -3185,11 +3705,11 @@ public void fdv1FallbackOnInitializerSuccessNoSelectorAppliesPayloadAndSwitches( Thread.NORM_PRIORITY, logger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); @@ -3220,11 +3740,11 @@ public void orchestrationLogging_warnsWhenNoInitializersOrSynchronizersConfigure Thread.NORM_PRIORITY, testLogger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3254,11 +3774,11 @@ public void orchestrationLogging_logsInitializerStartSuccessAndErrorOnException( Thread.NORM_PRIORITY, testLogger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3294,11 +3814,11 @@ public void orchestrationLogging_logsInitializerFailedOnStatusResult() throws Ex Thread.NORM_PRIORITY, testLogger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3333,11 +3853,11 @@ public void orchestrationLogging_logsSynchronizerStartingStatusInterruptedAndShu Thread.NORM_PRIORITY, testLogger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3368,11 +3888,11 @@ public void orchestrationLogging_logsSynchronizerParkedAndAllParked() throws Exc Thread.NORM_PRIORITY, testLogger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMillis(100), - Duration.ofSeconds(1), - Duration.ofMillis(10) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10) ); resourcesToClose.add(dataSource); @@ -3411,11 +3931,11 @@ public void orchestrationLogging_logsErrorWhenSynchronizerNextThrows() throws Ex Thread.NORM_PRIORITY, testLogger, executor, - Duration.ofSeconds(120), - Duration.ofSeconds(300), - Duration.ofMinutes(5), - Duration.ofHours(1), - Duration.ofSeconds(1) + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); From 243e5dc4096513edd229e98eb9e7dc42c82de513 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Wed, 4 Nov 2026 21:57:31 -0800 Subject: [PATCH 05/13] test: Cover unpark conditions and FDv1 fallback parking --- .../sdk/server/FDv2DataSourceTest.java | 45 +++++++ .../FDv2DataSourceUnparkConditionTest.java | 114 ++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceUnparkConditionTest.java diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index e18e4610..612c12c4 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -3171,6 +3171,51 @@ public void fdv1FallbackActivatesWhenFlagSet() throws Exception { assertTrue("Should have at least 2 changesets", sink.getApplyCount() >= 2); } + @Test + public void fdv1FallbackSynchronizerIsParkedAndRestartedDuringDirectedFallback() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger fdv1BuildCount = new AtomicInteger(); + + // The FDv2 synchronizer reports an interruption that carries the FDv1 fallback directive. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + true)); + return new MockQueuedSynchronizer(results); + } + ); + + // The first instance of the FDv1 fallback synchronizer fails, and the second instance delivers data. + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(fdv1BuildCount.incrementAndGet() == 1 + ? makeTerminalErrorResult() + : FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); + }; + + // The FDv1 fallback synchronizer is parked for 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // The FDv1 fallback synchronizer is built again after its unpark time, and its data is applied. + sink.awaitApplyCount(1, 2, TimeUnit.SECONDS); + assertEquals(1, sink.getApplyCount()); + assertEquals(2, fdv1BuildCount.get()); + } + @Test public void fdv1FallbackNotCalledWithoutFlag() throws Exception { executor = Executors.newScheduledThreadPool(2); diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceUnparkConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceUnparkConditionTest.java new file mode 100644 index 00000000..3d034a8b --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceUnparkConditionTest.java @@ -0,0 +1,114 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.fdv2.ChangeSet; +import com.launchdarkly.sdk.fdv2.ChangeSetType; +import com.launchdarkly.sdk.fdv2.Selector; +import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; +import com.launchdarkly.sdk.server.FDv2DataSourceConditions.UnparkCondition; +import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; +import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; + +import org.junit.After; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; + +@SuppressWarnings("javadoc") +public class FDv2DataSourceUnparkConditionTest extends BaseTest { + + private ScheduledExecutorService executor; + + @After + public void tearDown() { + if (executor != null && !executor.isShutdown()) { + executor.shutdownNow(); + } + } + + @Test + public void getTypeReturnsUnpark() { + executor = Executors.newScheduledThreadPool(1); + UnparkCondition condition = new UnparkCondition(executor, Duration.ofSeconds(120)); + + assertEquals(Condition.ConditionType.UNPARK, condition.getType()); + } + + @Test + public void conditionIsMetAfterItsDelay() throws Exception { + executor = Executors.newScheduledThreadPool(1); + + // Build a condition with a short delay. + UnparkCondition condition = new UnparkCondition(executor, Duration.ofMillis(50)); + + // The condition is met with itself after the delay. + assertSame(condition, condition.execute().get(2, TimeUnit.SECONDS)); + } + + @Test + public void pastUnparkTimeIsMetImmediately() throws Exception { + executor = Executors.newScheduledThreadPool(1); + + // Build a condition for an unpark time that has already passed. + UnparkCondition condition = new UnparkCondition(executor, Duration.ofMillis(-100)); + + // The condition is met without waiting. + assertSame(condition, condition.execute().get(1, TimeUnit.SECONDS)); + } + + @Test + public void informNeverRequestsReplacement() { + executor = Executors.newScheduledThreadPool(1); + UnparkCondition condition = new UnparkCondition(executor, Duration.ofSeconds(120)); + + // Inform the condition of a change set and of an interruption. + boolean replaceAfterChangeSet = condition.inform(FDv2SourceResult.changeSet( + new ChangeSet<>(ChangeSetType.None, Selector.EMPTY, null, null, true), + false)); + boolean replaceAfterInterruption = condition.inform(FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false)); + + // Neither result requires a new condition. + assertFalse(replaceAfterChangeSet); + assertFalse(replaceAfterInterruption); + } + + @Test + public void factoryBuildsConditionMetAtEarliestUnparkTime() throws Exception { + executor = Executors.newScheduledThreadPool(1); + Instant unparkTime = Instant.now().plusMillis(50); + UnparkCondition.Factory factory = new UnparkCondition.Factory(executor, () -> unparkTime); + + // Build a condition while a synchronizer is parked. + Condition condition = factory.build(); + + // The condition is met at the unpark time. + assertEquals(Condition.ConditionType.UNPARK, factory.getType()); + assertNotNull(condition); + CompletableFuture future = condition.execute(); + assertSame(condition, future.get(2, TimeUnit.SECONDS)); + } + + @Test + public void factoryReturnsNullWhenNoSynchronizerIsParked() { + executor = Executors.newScheduledThreadPool(1); + UnparkCondition.Factory factory = new UnparkCondition.Factory(executor, () -> null); + + // Build a condition while no synchronizer is parked. + Condition condition = factory.build(); + + // There is no condition. + assertNull(condition); + } +} From 0067868e1968514d0ff8e14f6f1b6c913d63c3bb Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Thu, 5 Nov 2026 15:03:22 -0800 Subject: [PATCH 06/13] feat: End and extend the FDv1 fallback after its TTL --- .../sdk/server/FDv2DataSource.java | 436 ++++++----- .../sdk/server/FDv2DataSourceConditions.java | 30 +- .../sdk/server/SourceManager.java | 81 +- ... FDv2DataSourceDeadlineConditionTest.java} | 22 +- .../sdk/server/FDv2DataSourceTest.java | 724 ++++++++++++++---- 5 files changed, 894 insertions(+), 399 deletions(-) rename lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/{FDv2DataSourceUnparkConditionTest.java => FDv2DataSourceDeadlineConditionTest.java} (76%) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 56ea7615..703f09f3 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -4,6 +4,10 @@ import com.launchdarkly.logging.LDLogger; import com.launchdarkly.logging.LogValues; import com.launchdarkly.sdk.fdv2.SourceSignal; +import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; +import com.launchdarkly.sdk.server.FDv2DataSourceConditions.ConditionFactory; +import com.launchdarkly.sdk.server.FDv2DataSourceConditions.FallbackCondition; +import com.launchdarkly.sdk.server.FDv2DataSourceConditions.RecoveryCondition; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; import com.launchdarkly.sdk.server.datasources.Initializer; import com.launchdarkly.sdk.server.datasources.Synchronizer; @@ -31,38 +35,43 @@ import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.ConditionFactory; import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.FallbackCondition; import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.RecoveryCondition; -import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.UnparkCondition; +import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.DeadlineCondition; class FDv2DataSource implements DataSource { /** - * Default fallback timeout of 2 minutes. The timeout is only configurable for testing. + * Default fallback timeout of 2 minutes. */ private static final Duration DEFAULT_FALLBACK_TIMEOUT = Duration.ofMinutes(2); /** - * Default recovery timeout of 5 minutes. The timeout is only configurable for testing. + * Default recovery timeout of 5 minutes. */ private static final Duration DEFAULT_RECOVERY_TIMEOUT = Duration.ofMinutes(5); /** - * Default base of the park delay that follows an unexpected failure. The delay is only configurable for testing. + * Default base of the park delay that follows an unexpected failure. */ private static final Duration DEFAULT_PARK_INITIAL_DELAY = Duration.ofMinutes(5); /** - * Default maximum of the park delay. The delay is only configurable for testing. + * Default maximum of the park delay. */ private static final Duration DEFAULT_PARK_MAX_DELAY = Duration.ofHours(1); /** - * Default base of the failover delay that follows a park. The delay is only configurable for testing. + * Default base of the failover delay that follows a park. */ private static final Duration DEFAULT_FAILOVER_DELAY = Duration.ofSeconds(1); + /** + * Default duration of a fallback to FDv1, when the directive does not give one. + */ + private static final Duration DEFAULT_FDV1_FALLBACK_TTL = Duration.ofHours(1); + private final SourceManager sourceManager; - private final Map conditionFactories = - new EnumMap<>(Condition.ConditionType.class); + private final Map conditionFactories = new EnumMap<>( + Condition.ConditionType.class); private final Duration parkInitialDelay; @@ -70,6 +79,8 @@ class FDv2DataSource implements DataSource { private final Duration failoverDelay; + private final Duration defaultFDv1FallbackTTL; + private final DataSourceUpdateSinkV2 dataSourceUpdates; private final CompletableFuture startFuture = new CompletableFuture<>(); @@ -98,54 +109,56 @@ public interface DataSourceFactory { } public FDv2DataSource( - ImmutableList> initializers, - ImmutableList> synchronizers, - DataSourceFactory fdv1DataSourceFactory, - DataSourceUpdateSinkV2 dataSourceUpdates, - int threadPriority, - LDLogger logger, - ScheduledExecutorService sharedExecutor - ) { + ImmutableList> initializers, + ImmutableList> synchronizers, + DataSourceFactory fdv1DataSourceFactory, + DataSourceUpdateSinkV2 dataSourceUpdates, + int threadPriority, + LDLogger logger, + ScheduledExecutorService sharedExecutor) { this(initializers, - synchronizers, - fdv1DataSourceFactory, - dataSourceUpdates, - threadPriority, - logger, - sharedExecutor, - DEFAULT_FALLBACK_TIMEOUT, - DEFAULT_RECOVERY_TIMEOUT, - DEFAULT_PARK_INITIAL_DELAY, - DEFAULT_PARK_MAX_DELAY, - DEFAULT_FAILOVER_DELAY - ); + synchronizers, + fdv1DataSourceFactory, + dataSourceUpdates, + threadPriority, + logger, + sharedExecutor, + DEFAULT_FALLBACK_TIMEOUT, + DEFAULT_RECOVERY_TIMEOUT, + DEFAULT_PARK_INITIAL_DELAY, + DEFAULT_PARK_MAX_DELAY, + DEFAULT_FAILOVER_DELAY, + DEFAULT_FDV1_FALLBACK_TTL); } + /** + * Creates a data source with the given timeouts and delays. Tests use this constructor to shorten them. + */ public FDv2DataSource( - ImmutableList> initializers, - ImmutableList> synchronizers, - DataSourceFactory fdv1DataSourceFactory, - DataSourceUpdateSinkV2 dataSourceUpdates, - int threadPriority, - LDLogger logger, - ScheduledExecutorService sharedExecutor, - Duration fallbackTimeout, - Duration recoveryTimeout, - Duration parkInitialDelay, - Duration parkMaxDelay, - Duration failoverDelay - ) { + ImmutableList> initializers, + ImmutableList> synchronizers, + DataSourceFactory fdv1DataSourceFactory, + DataSourceUpdateSinkV2 dataSourceUpdates, + int threadPriority, + LDLogger logger, + ScheduledExecutorService sharedExecutor, + Duration fallbackTimeout, + Duration recoveryTimeout, + Duration parkInitialDelay, + Duration parkMaxDelay, + Duration failoverDelay, + Duration defaultFDv1FallbackTTL) { List synchronizerFactories = synchronizers - .stream() - .map(SynchronizerFactoryWithState::new) - // Collect to an ArrayList to ensure mutability. - .collect(Collectors.toCollection(ArrayList::new)); + .stream() + .map(SynchronizerFactoryWithState::new) + // Collect to an ArrayList to ensure mutability. + .collect(Collectors.toCollection(ArrayList::new)); // If we have a fdv1 data source factory, then add that to the synchronizer factories in a blocked state. // If we receive a request to fallback, then we will unblock it and block all other synchronizers. if (fdv1DataSourceFactory != null) { SynchronizerFactoryWithState wrapped = new SynchronizerFactoryWithState(fdv1DataSourceFactory, - true); + true); wrapped.block(); synchronizerFactories.add(wrapped); @@ -158,14 +171,21 @@ public FDv2DataSource( this.threadPriority = threadPriority; this.logger = logger; this.conditionFactories.put( - Condition.ConditionType.FALLBACK, new FallbackCondition.Factory(sharedExecutor, fallbackTimeout)); + Condition.ConditionType.FALLBACK, new FallbackCondition.Factory(sharedExecutor, fallbackTimeout)); this.conditionFactories.put( - Condition.ConditionType.RECOVERY, new RecoveryCondition.Factory(sharedExecutor, recoveryTimeout)); + Condition.ConditionType.RECOVERY, new RecoveryCondition.Factory(sharedExecutor, recoveryTimeout)); this.conditionFactories.put( - Condition.ConditionType.UNPARK, new UnparkCondition.Factory(sharedExecutor, sourceManager::getEarliestUnparkTime)); + Condition.ConditionType.UNPARK, + new DeadlineCondition.Factory( + sharedExecutor, Condition.ConditionType.UNPARK, sourceManager::getEarliestUnparkTime)); + this.conditionFactories.put( + Condition.ConditionType.FDV1_FALLBACK_END, + new DeadlineCondition.Factory( + sharedExecutor, Condition.ConditionType.FDV1_FALLBACK_END, sourceManager::getFDv1FallbackEndTime)); this.parkInitialDelay = parkInitialDelay; this.parkMaxDelay = parkMaxDelay; this.failoverDelay = failoverDelay; + this.defaultFDv1FallbackTTL = defaultFDv1FallbackTTL; } private void run() { @@ -174,8 +194,7 @@ private void run() { // There are not any initializer or synchronizers, so we are at the best state that // can be achieved. logger.warn( - "LaunchDarkly client will not connect to LaunchDarkly for feature flag data due to no initializers or synchronizers configured." - ); + "LaunchDarkly client will not connect to LaunchDarkly for feature flag data due to no initializers or synchronizers configured."); dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); startFuture.complete(true); return; @@ -186,44 +205,25 @@ private void run() { initializerOutcome = runInitializers(); } - // If an initializer signalled FDv1 fallback, block every FDv2 synchronizer in - // one shot via fdv1Fallback() (which also unblocks the FDv1 fallback - // synchronizer, if one is configured). If FDv1 is configured we hand off to it; - // otherwise we halt the data system. This takes precedence over the standard - // synchronizer chain -- the FDv2 synchronizers are not given a chance to run. + // If an initializer signalled FDv1 fallback, fall back to FDv1 before any synchronizer runs. if (initializerOutcome.fallbackToFDv1) { - sourceManager.fdv1Fallback(); - if (sourceManager.hasFDv1Fallback()) { - logger.warn("Initializer requested fallback to FDv1; switching to FDv1 fallback synchronizer."); - } else { - logger.warn("Initializer requested fallback to FDv1, but no FDv1 fallback synchronizer is configured."); - dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.OFF, - initializerOutcome.errorInfo); - startFuture.complete(false); - return; - } + fallBackToFDv1(jittered(defaultFDv1FallbackTTL), initializerOutcome.errorInfo); } - if(!sourceManager.hasAvailableSynchronizers()) { + if (!sourceManager.hasFDv2Synchronizers() && !sourceManager.hasAvailableSynchronizers()) { // If already completed by the initializers, then this will have no effect. if (!isInitialized()) { // If we have no synchronizers, and we didn't manage to initialize, and we aren't shutting down, // then that was unexpected, and we will report it. - maybeReportUnexpectedExhaustion("All initializers exhausted and there are no available synchronizers."); + maybeReportUnexpectedExhaustion( + "All initializers exhausted and there are no available synchronizers."); } // If already completed has no effect. startFuture.complete(false); return; } - boolean haltedByDirective = runSynchronizers(); - - if (!haltedByDirective) { - // If we had synchronizers, and we ran out of them, and we aren't shutting down, then that was unexpected, - // and we will report it. - maybeReportUnexpectedExhaustion("All data source acquisition methods have been exhausted."); - } + runSynchronizers(); // If we had initialized at some point, then the future will already be complete and this will be ignored. startFuture.complete(false); @@ -280,16 +280,16 @@ private InitializerOutcome runInitializers() { case INTERRUPTED: case TERMINAL_ERROR: logger.warn("Initializer '{}' failed: {}", - initializerName, - detailForError(status.getErrorInfo())); + initializerName, + detailForError(status.getErrorInfo())); fallbackErrorInfo = status.getErrorInfo(); // The data source updates handler will filter the state during initializing, but this // will make the error information available. dataSourceUpdates.updateStatus( - // While the error was terminal to the individual initializer, it isn't terminal - // to the data source as a whole. - DataSourceStatusProvider.State.INTERRUPTED, - status.getErrorInfo()); + // While the error was terminal to the individual initializer, it isn't terminal + // to the data source as a whole. + DataSourceStatusProvider.State.INTERRUPTED, + status.getErrorInfo()); break; case SHUTDOWN: case GOODBYE: @@ -312,15 +312,15 @@ private InitializerOutcome runInitializers() { // The data source updates handler will filter the state during initializing, but this // will make the error information available. dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.INTERRUPTED, - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.UNKNOWN, - 0, - e.toString(), - new Date().toInstant())); + DataSourceStatusProvider.State.INTERRUPTED, + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + e.toString(), + new Date().toInstant())); Throwable root = e instanceof ExecutionException && e.getCause() != null ? e.getCause() : e; logger.error("Error running initializer '{}': {}", - initializerName, - root.getMessage() != null ? root.getMessage() : LogValues.exceptionSummary(root)); + initializerName, + root.getMessage() != null ? root.getMessage() : LogValues.exceptionSummary(root)); } initializer = sourceManager.getNextInitializerAndSetActive(); } @@ -370,18 +370,19 @@ private List getConditions() { boolean isPrimeSynchronizer = sourceManager.isPrimeSynchronizer(); if (availableSynchronizers == 1) { - // If there is only 1 synchronizer, then we cannot fall back or recover, so we don't need any conditions. - return Collections.emptyList(); + // With only 1 synchronizer, we cannot fall back or recover. Only the end of the FDv1 fallback applies. + Condition fallbackEnd = buildCondition(Condition.ConditionType.FDV1_FALLBACK_END); + return fallbackEnd == null ? Collections.emptyList() : Collections.singletonList(fallbackEnd); } if (isPrimeSynchronizer) { // If there isn't a synchronizer to recover to, then don't add and recovery conditions. return conditionFactories.values().stream() - .filter((ConditionFactory factory) -> factory.getType() != Condition.ConditionType.RECOVERY) - .map(ConditionFactory::build).filter(Objects::nonNull).collect(Collectors.toList()); + .filter((ConditionFactory factory) -> factory.getType() != Condition.ConditionType.RECOVERY) + .map(ConditionFactory::build).filter(Objects::nonNull).collect(Collectors.toList()); } // The synchronizer can both fall back and recover. return conditionFactories.values().stream() - .map(ConditionFactory::build).filter(Objects::nonNull).collect(Collectors.toList()); + .map(ConditionFactory::build).filter(Objects::nonNull).collect(Collectors.toList()); } /** @@ -397,15 +398,9 @@ private Condition buildCondition(Condition.ConditionType type) { /** * Runs the configured synchronizers, falling back / recovering as conditions allow, - * until the list is exhausted, the data source is closed, or a server-directed FDv1 - * fallback halts the data system. - * - * @return true when {@code runSynchronizers} halted in response to a server-directed - * FDv1 fallback directive that could not be satisfied (no FDv1 fallback - * synchronizer configured) -- the caller should NOT report exhaustion in - * that case because OFF has already been published with a specific error. + * until the data source is closed or a synchronizer reports that it shut down. */ - private boolean runSynchronizers() { + private void runSynchronizers() { // When runSynchronizers exists, no matter how it exits, the synchronizerStateManager will be closed. try { Synchronizer synchronizer = buildNextAvailableSynchronizer(); @@ -437,43 +432,49 @@ private boolean runSynchronizers() { if (metType == Condition.ConditionType.UNPARK) { // Replace the unpark condition with one for the next unpark. conditions.replace( - Condition.ConditionType.UNPARK, - buildCondition(Condition.ConditionType.UNPARK)); + Condition.ConditionType.UNPARK, + buildCondition(Condition.ConditionType.UNPARK)); + } + if (metType == Condition.ConditionType.FDV1_FALLBACK_END) { + if (sourceManager.endFDv1FallbackIfExpired()) { + logger.info("Fallback TTL elapsed, restarting FDv2 data sources."); + break; + } + // The timer fired before the end time. This shouldn't normally happen. Wait for the remaining time. + conditions.replace( + Condition.ConditionType.FDV1_FALLBACK_END, + buildCondition(Condition.ConditionType.FDV1_FALLBACK_END)); } // An unpark does not switch synchronizers by itself, but other conditions might already // be met. Check all of them before deciding whether to switch. Recovery takes precedence. if (conditions.isMet(Condition.ConditionType.RECOVERY) - && sourceManager.hasMorePreferredAvailableSynchronizer()) { + && sourceManager.hasMorePreferredAvailableSynchronizer()) { // For recovery, we will start at the first available synchronizer. // So we reset the source index, and finding the source will start at the beginning. sourceManager.resetSourceIndex(); logger.info( - "Recovery condition met, moving from synchronizer '{}' to primary synchronizer.", - synchronizer.name() - ); + "Recovery condition met, moving from synchronizer '{}' to primary synchronizer.", + synchronizer.name()); break; } if (conditions.isMet(Condition.ConditionType.FALLBACK) - && sourceManager.hasOtherAvailableSynchronizer()) { + && sourceManager.hasOtherAvailableSynchronizer()) { logger.info( - "Fallback condition met, falling back from synchronizer '{}'.", - synchronizer.name() - ); + "Fallback condition met, falling back from synchronizer '{}'.", + synchronizer.name()); break; } // Keep running this synchronizer. if (metType == Condition.ConditionType.FALLBACK) { logger.info( - "Fallback condition met, but no other synchronizer is available; continuing with synchronizer '{}'.", - synchronizer.name() - ); + "Fallback condition met, but no other synchronizer is available; continuing with synchronizer '{}'.", + synchronizer.name()); } else if (metType == Condition.ConditionType.RECOVERY) { logger.info( - "Recovery condition met, but no more-preferred synchronizer is available; continuing with synchronizer '{}'.", - synchronizer.name() - ); + "Recovery condition met, but no more-preferred synchronizer is available; continuing with synchronizer '{}'.", + synchronizer.name()); } continue; } @@ -511,33 +512,40 @@ private boolean runSynchronizers() { switch (status.getState()) { case INTERRUPTED: maybeLogSynchronizerStatusChange( - synchronizer.name(), - status.getState() - ); + synchronizer.name(), + status.getState()); // Handled by conditions. dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.INTERRUPTED, - status.getErrorInfo()); + DataSourceStatusProvider.State.INTERRUPTED, + status.getErrorInfo()); break; case SHUTDOWN: maybeLogSynchronizerStatusChange( - synchronizer.name(), - status.getState() - ); - // We should be overall shutting down. - logger.debug("Synchronizer shutdown."); - return false; + synchronizer.name(), + status.getState()); + if (closed) { + // Closing the data source shut down the synchronizer. + logger.debug("Synchronizer shutdown."); + return; + } + // The synchronizer shut down by itself, so park it. + parkActiveSynchronizer(synchronizer.name(), status.getErrorInfo()); + parked = true; + running = false; + dataSourceUpdates.updateStatus( + DataSourceStatusProvider.State.INTERRUPTED, + status.getErrorInfo()); + break; case TERMINAL_ERROR: maybeLogSynchronizerStatusChange( - synchronizer.name(), - status.getState() - ); + synchronizer.name(), + status.getState()); parkActiveSynchronizer(synchronizer.name(), status.getErrorInfo()); parked = true; running = false; dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.INTERRUPTED, - status.getErrorInfo()); + DataSourceStatusProvider.State.INTERRUPTED, + status.getErrorInfo()); break; case GOODBYE: // We let the synchronizer handle this internally. @@ -545,34 +553,23 @@ private boolean runSynchronizers() { } break; } - // We have been requested to fall back to FDv1. Block every FDv2 - // synchronizer in one shot via fdv1Fallback() (which also unblocks the - // FDv1 fallback synchronizer, if one is configured). If FDv1 is - // configured we hand off to it; otherwise we halt the data system. - // An FDv1 fallback synchronizer asking to fall back again is ignored - // -- shouldn't happen in practice. - if (result.isFdv1Fallback() - && !sourceManager.isCurrentSynchronizerFDv1Fallback()) { - sourceManager.fdv1Fallback(); - if (sourceManager.hasFDv1Fallback()) { - logger.info("Falling back to an FDv1 fallback synchronizer."); - running = false; + if (result.isFdv1Fallback()) { + if (sourceManager.isCurrentSynchronizerFDv1Fallback()) { + // A directive during the FDv1 fallback replaces its TTL, and FDv1 keeps running. + Duration fallbackTTL = jittered(defaultFDv1FallbackTTL); + sourceManager.startFDv1Fallback(Instant.now().plus(fallbackTTL)); + conditions.replace( + Condition.ConditionType.FDV1_FALLBACK_END, + buildCondition(Condition.ConditionType.FDV1_FALLBACK_END)); + logger.info("FDv2 retry scheduled in {}s.", fallbackTTL.getSeconds()); } else { - // When the directive is signalled but no FDv1 fallback synchronizer - // is configured, halt the data system entirely. Surface OFF with - // the most recent error info (if any) and exit the synchronizer - // loop terminally. - logger.warn( - "Synchronizer '{}' requested FDv1 fallback, but no FDv1 fallback synchronizer is configured; halting the data system.", - synchronizer.name() - ); - DataSourceStatusProvider.ErrorInfo offError = - result.getStatus() != null ? result.getStatus().getErrorInfo() : null; - dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.OFF, - offError); - startFuture.complete(false); - return true; + // A directive to fall back to FDv1 ends the run of an FDv2 synchronizer. + fallBackToFDv1( + jittered(defaultFDv1FallbackTTL), + result.getStatus() != null ? result.getStatus().getErrorInfo() : null); + // The directive takes over, so there is no failover delay after a park. + parked = false; + running = false; } } } @@ -580,19 +577,21 @@ private boolean runSynchronizers() { } } catch (ExecutionException | InterruptedException | RuntimeException e) { DataSourceStatusProvider.ErrorInfo errorInfo = new DataSourceStatusProvider.ErrorInfo( - DataSourceStatusProvider.ErrorKind.UNKNOWN, - 0, - e.toString(), - new Date().toInstant() - ); + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + e.toString(), + new Date().toInstant()); dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, errorInfo); Throwable root = e instanceof ExecutionException && e.getCause() != null ? e.getCause() : e; logger.error("Error running synchronizer '{}': {}", - synchronizer.name(), - root.getMessage() != null ? root.getMessage() : LogValues.exceptionSummary(root)); + synchronizer.name(), + root.getMessage() != null ? root.getMessage() : LogValues.exceptionSummary(root)); parkActiveSynchronizer(synchronizer.name(), errorInfo); parked = true; } + // The run of this synchronizer has ended, so stop it. + sourceManager.closeActiveSource(); + if (parked && !waitUnlessClosed(jittered(failoverDelay))) { break; } @@ -605,7 +604,6 @@ private boolean runSynchronizers() { } finally { sourceManager.close(); } - return false; } private static String detailForError(DataSourceStatusProvider.ErrorInfo errorInfo) { @@ -656,8 +654,8 @@ private static Duration jittered(Duration delay) { */ private Duration parkDelay(int parkCount) { double delayMillis = Math.min( - parkInitialDelay.toMillis() * Math.pow(2, parkCount - 1), - parkMaxDelay.toMillis()); + parkInitialDelay.toMillis() * Math.pow(2, parkCount - 1), + parkMaxDelay.toMillis()); return jittered(Duration.ofMillis((long) delayMillis)); } @@ -671,32 +669,53 @@ private void parkActiveSynchronizer(String synchronizerName, DataSourceStatusPro Duration parkDelay = parkDelay(sourceManager.incrementCurrentSynchronizerParkCount()); sourceManager.parkCurrentSynchronizer(Instant.now().plus(parkDelay)); logger.warn( - "{} is backing off for {}s after an unexpected error ({}).", - synchronizerName != null ? "Synchronizer '" + synchronizerName + "'" : "A synchronizer that failed to build", - parkDelay.getSeconds(), - detailForError(errorInfo) - ); + "{} is backing off for {}s after an unexpected error ({}).", + synchronizerName != null ? "Synchronizer '" + synchronizerName + "'" + : "A synchronizer that failed to build", + parkDelay.getSeconds(), + detailForError(errorInfo)); } /** - * Gets the next available synchronizer, builds it, and sets it active. While no synchronizer is available, this - * waits for the earliest unpark time. A synchronizer that fails to build is parked, and the next available - * synchronizer is tried after the failover delay. + * Falls back to FDv1 for the given duration. If no FDv1 fallback synchronizer is configured, no synchronizer runs + * until the fallback ends. + * + * @param fallbackTTL how long the fallback lasts + * @param errorInfo the error that came with the directive, or null + */ + private void fallBackToFDv1(Duration fallbackTTL, DataSourceStatusProvider.ErrorInfo errorInfo) { + sourceManager.startFDv1Fallback(Instant.now().plus(fallbackTTL)); + if (sourceManager.hasFDv1Fallback()) { + logger.info("Falling back to FDv1 fallback synchronizer."); + } else { + logger.warn("Fallback to FDv1 was requested, but no FDv1 fallback synchronizer is configured."); + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, errorInfo); + } + logger.info("FDv2 retry scheduled in {}s.", fallbackTTL.getSeconds()); + } + + /** + * Gets the next available synchronizer, builds it, and sets it active. If the FDv1 fallback has expired, this + * first makes the FDv2 synchronizers available again. While no synchronizer is available, this waits for the + * earliest unpark time or the end of the FDv1 fallback, whichever comes first. A synchronizer that fails to build + * is parked, and the next available synchronizer is tried after the failover delay. * * @return the active synchronizer, or null if the data source was closed */ private Synchronizer buildNextAvailableSynchronizer() { while (!closed) { + if (sourceManager.endFDv1FallbackIfExpired()) { + logger.info("Fallback TTL elapsed, restarting FDv2 data sources."); + } Synchronizer synchronizer; try { synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); } catch (RuntimeException e) { DataSourceStatusProvider.ErrorInfo errorInfo = new DataSourceStatusProvider.ErrorInfo( - DataSourceStatusProvider.ErrorKind.UNKNOWN, - 0, - e.toString(), - Instant.now() - ); + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + e.toString(), + Instant.now()); dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, errorInfo); parkActiveSynchronizer(null, errorInfo); if (!waitUnlessClosed(jittered(failoverDelay))) { @@ -709,14 +728,23 @@ private Synchronizer buildNextAvailableSynchronizer() { } Instant earliestUnparkTime = sourceManager.getEarliestUnparkTime(); + Instant fallbackEndTime = sourceManager.getFDv1FallbackEndTime(); Duration wait; - if (earliestUnparkTime == null) { + if (earliestUnparkTime == null && fallbackEndTime == null) { // This should never happen. wait = parkInitialDelay; logger.error("No synchronizer can be started; checking again in {}s.", wait.getSeconds()); - } else { + } else if (earliestUnparkTime == null) { + wait = Duration.between(Instant.now(), fallbackEndTime); + logger.warn("No synchronizer is available until the FDv1 fallback ends in {}s.", + Math.max(0, wait.getSeconds())); + } else if (fallbackEndTime == null) { wait = Duration.between(Instant.now(), earliestUnparkTime); logger.warn("All synchronizers are backing off for at least {}s.", Math.max(0, wait.getSeconds())); + } else { + Instant earliest = fallbackEndTime.isBefore(earliestUnparkTime) ? fallbackEndTime : earliestUnparkTime; + wait = Duration.between(Instant.now(), earliest); + logger.warn("All synchronizers are backing off for at least {}s.", Math.max(0, wait.getSeconds())); } if (!waitUnlessClosed(wait)) { return null; @@ -737,8 +765,8 @@ private void maybeLogSynchronizerStatusChange(String sourceName, SourceSignal st return; } if (sourceName != null - && sourceName.equals(lastLoggedSynchronizerDedupeName) - && state == lastLoggedSynchronizerDedupeStatus) { + && sourceName.equals(lastLoggedSynchronizerDedupeName) + && state == lastLoggedSynchronizerDedupeStatus) { return; } lastLoggedSynchronizerDedupeName = sourceName; @@ -780,17 +808,16 @@ public void close() { } private void maybeReportUnexpectedExhaustion(String message) { - if(!closed) { + if (!closed) { dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.OFF, - // If the data source was closed, then we just report we are OFF without an - // associated error. - new DataSourceStatusProvider.ErrorInfo( - DataSourceStatusProvider.ErrorKind.UNKNOWN, - 0, - message, - new Date().toInstant()) - ); + DataSourceStatusProvider.State.OFF, + // If the data source was closed, then we just report we are OFF without an + // associated error. + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + message, + new Date().toInstant())); } } @@ -829,8 +856,7 @@ static class Conditions implements AutoCloseable { /** * The conditions, with at most one of each type. Accessed only under {@code lock}. */ - private final Map conditions = - new EnumMap<>(Condition.ConditionType.class); + private final Map conditions = new EnumMap<>(Condition.ConditionType.class); /** * The types of the conditions that the aggregate has completed with. Accessed only under {@code lock}. @@ -866,12 +892,12 @@ public Conditions(List conditions) { private void arm() { synchronized (lock) { CompletableFuture[] unreported = conditions.values().stream() - .filter(c -> !reported.contains(c.getType())) - .map(Condition::execute) - .toArray(CompletableFuture[]::new); + .filter(c -> !reported.contains(c.getType())) + .map(Condition::execute) + .toArray(CompletableFuture[]::new); CompletableFuture newAggregate = unreported.length == 0 - ? new CompletableFuture<>() // Never completes if no conditions are unreported - : CompletableFuture.anyOf(unreported); + ? new CompletableFuture<>() // Never completes if no conditions are unreported + : CompletableFuture.anyOf(unreported); aggregate = newAggregate; // Start a new set only if the previous aggregate drained it. Otherwise, the futures in it are still // waiting, and the new aggregate completes them. diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java index f042678a..0b9b2049 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java @@ -35,6 +35,7 @@ enum ConditionType { FALLBACK, RECOVERY, UNPARK, + FDV1_FALLBACK_END, } CompletableFuture execute(); @@ -199,36 +200,41 @@ public ConditionType getType() { /** * This condition is met when its delay has passed. */ - static class UnparkCondition extends TimedCondition { + static class DeadlineCondition extends TimedCondition { /** - * Builds a condition that is met at the earliest unpark time, or no condition if no synchronizer is parked. + * Builds a condition that is met at the deadline, or no condition if there is no deadline. */ static class Factory implements ConditionFactory { private final ScheduledExecutorService sharedExecutor; - private final Supplier earliestUnparkTime; + private final ConditionType type; + private final Supplier deadline; - public Factory(ScheduledExecutorService sharedExecutor, Supplier earliestUnparkTime) { + public Factory(ScheduledExecutorService sharedExecutor, ConditionType type, Supplier deadline) { this.sharedExecutor = sharedExecutor; - this.earliestUnparkTime = earliestUnparkTime; + this.type = type; + this.deadline = deadline; } @Override public Condition build() { - Instant unparkTime = earliestUnparkTime.get(); - if (unparkTime == null) { + Instant time = deadline.get(); + if (time == null) { return null; } - return new UnparkCondition(sharedExecutor, Duration.between(Instant.now(), unparkTime)); + return new DeadlineCondition(sharedExecutor, type, Duration.between(Instant.now(), time)); } @Override public ConditionType getType() { - return ConditionType.UNPARK; + return type; } } - public UnparkCondition(ScheduledExecutorService sharedExecutor, Duration delay) { + private final ConditionType type; + + public DeadlineCondition(ScheduledExecutorService sharedExecutor, ConditionType type, Duration delay) { super(sharedExecutor, delay); + this.type = type; timerFuture = sharedExecutor.schedule(() -> { resultFuture.complete(this); return null; @@ -237,13 +243,13 @@ public UnparkCondition(ScheduledExecutorService sharedExecutor, Duration delay) @Override public boolean inform(FDv2SourceResult sourceResult) { - // An unpark is time-based, so a result never requires a new condition. + // A deadline is time-based, so a result never requires a new condition. return false; } @Override public ConditionType getType() { - return ConditionType.UNPARK; + return type; } } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java index 859056d9..effc699b 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java @@ -37,6 +37,12 @@ class SourceManager implements Closeable { */ private SynchronizerFactoryWithState currentSynchronizerFactory; + /** + * The time at which the fallback to FDv1 ends, or null if the SDK is not falling back to FDv1. Accessed only + * under activeSourceLock. + */ + private Instant fdv1FallbackEndTime; + public SourceManager(List synchronizers, List> initializers) { this.synchronizers = synchronizers; this.initializers = initializers; @@ -61,15 +67,29 @@ public boolean hasFDv1Fallback() { return false; } + public boolean hasFDv2Synchronizers() { + for (SynchronizerFactoryWithState factory : synchronizers) { + if (!factory.isFDv1Fallback()) { + return true; + } + } + return false; + } + /** - * Block all synchronizers aside from the fdv1 fallback and unblock the fdv1 fallback. + * Block all synchronizers aside from the fdv1 fallback and unblock the fdv1 fallback, until the given time. + * + * @param endTime the time at which the fallback to FDv1 ends */ - public void fdv1Fallback() { - for (SynchronizerFactoryWithState factory : synchronizers) { - if(factory.isFDv1Fallback()) { - factory.unblock(); - } else { - factory.block(); + public void startFDv1Fallback(Instant endTime) { + synchronized (activeSourceLock) { + fdv1FallbackEndTime = endTime; + for (SynchronizerFactoryWithState factory : synchronizers) { + if (factory.isFDv1Fallback()) { + factory.unblock(); + } else { + factory.block(); + } } } } @@ -86,6 +106,41 @@ public void unparkExpiredSynchronizers() { } } + /** + * The time at which the fallback to FDv1 ends. + * + * @return the end time, or null if the SDK is not falling back to FDv1 + */ + public Instant getFDv1FallbackEndTime() { + synchronized (activeSourceLock) { + return fdv1FallbackEndTime; + } + } + + /** + * Ends the fallback to FDv1 if its end time has arrived. This unblocks the FDv2 synchronizers, blocks the fdv1 + * fallback, and starts the search for the next synchronizer from the most-preferred one. + * + * @return true if the fallback ended + */ + public boolean endFDv1FallbackIfExpired() { + synchronized (activeSourceLock) { + if (fdv1FallbackEndTime == null || Instant.now().isBefore(fdv1FallbackEndTime)) { + return false; + } + fdv1FallbackEndTime = null; + for (SynchronizerFactoryWithState factory : synchronizers) { + if (factory.isFDv1Fallback()) { + factory.block(); + } else { + factory.unblock(); + } + } + synchronizerIndex = -1; + return true; + } + } + /** * Get the next synchronizer to use. This operates based on tracking the index of the currently active synchronizer, * which will loop through all available synchronizers handling interruptions. Then a non-prime synchronizer recovers @@ -341,6 +396,18 @@ public int getAvailableSynchronizerCount() { } + /** + * Closes the active source, if there is one. + */ + public void closeActiveSource() { + synchronized (activeSourceLock) { + if (activeSource != null) { + safeClose(activeSource); + activeSource = null; + } + } + } + /** * Close the state manager and shut down any active source. * Implements AutoCloseable to enable try-with-resources usage. diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceUnparkConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceDeadlineConditionTest.java similarity index 76% rename from lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceUnparkConditionTest.java rename to lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceDeadlineConditionTest.java index 3d034a8b..a8cfaa34 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceUnparkConditionTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceDeadlineConditionTest.java @@ -4,7 +4,8 @@ import com.launchdarkly.sdk.fdv2.ChangeSetType; import com.launchdarkly.sdk.fdv2.Selector; import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.UnparkCondition; +import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition.ConditionType; +import com.launchdarkly.sdk.server.FDv2DataSourceConditions.DeadlineCondition; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; @@ -25,7 +26,7 @@ import static org.junit.Assert.assertSame; @SuppressWarnings("javadoc") -public class FDv2DataSourceUnparkConditionTest extends BaseTest { +public class FDv2DataSourceDeadlineConditionTest extends BaseTest { private ScheduledExecutorService executor; @@ -39,9 +40,9 @@ public void tearDown() { @Test public void getTypeReturnsUnpark() { executor = Executors.newScheduledThreadPool(1); - UnparkCondition condition = new UnparkCondition(executor, Duration.ofSeconds(120)); + DeadlineCondition condition = new DeadlineCondition(executor, ConditionType.UNPARK, Duration.ofSeconds(120)); - assertEquals(Condition.ConditionType.UNPARK, condition.getType()); + assertEquals(ConditionType.UNPARK, condition.getType()); } @Test @@ -49,7 +50,7 @@ public void conditionIsMetAfterItsDelay() throws Exception { executor = Executors.newScheduledThreadPool(1); // Build a condition with a short delay. - UnparkCondition condition = new UnparkCondition(executor, Duration.ofMillis(50)); + DeadlineCondition condition = new DeadlineCondition(executor, ConditionType.UNPARK, Duration.ofMillis(50)); // The condition is met with itself after the delay. assertSame(condition, condition.execute().get(2, TimeUnit.SECONDS)); @@ -60,7 +61,7 @@ public void pastUnparkTimeIsMetImmediately() throws Exception { executor = Executors.newScheduledThreadPool(1); // Build a condition for an unpark time that has already passed. - UnparkCondition condition = new UnparkCondition(executor, Duration.ofMillis(-100)); + DeadlineCondition condition = new DeadlineCondition(executor, ConditionType.UNPARK, Duration.ofMillis(-100)); // The condition is met without waiting. assertSame(condition, condition.execute().get(1, TimeUnit.SECONDS)); @@ -69,7 +70,7 @@ public void pastUnparkTimeIsMetImmediately() throws Exception { @Test public void informNeverRequestsReplacement() { executor = Executors.newScheduledThreadPool(1); - UnparkCondition condition = new UnparkCondition(executor, Duration.ofSeconds(120)); + DeadlineCondition condition = new DeadlineCondition(executor, ConditionType.UNPARK, Duration.ofSeconds(120)); // Inform the condition of a change set and of an interruption. boolean replaceAfterChangeSet = condition.inform(FDv2SourceResult.changeSet( @@ -88,13 +89,14 @@ public void informNeverRequestsReplacement() { public void factoryBuildsConditionMetAtEarliestUnparkTime() throws Exception { executor = Executors.newScheduledThreadPool(1); Instant unparkTime = Instant.now().plusMillis(50); - UnparkCondition.Factory factory = new UnparkCondition.Factory(executor, () -> unparkTime); + DeadlineCondition.Factory factory = + new DeadlineCondition.Factory(executor, ConditionType.UNPARK, () -> unparkTime); // Build a condition while a synchronizer is parked. Condition condition = factory.build(); // The condition is met at the unpark time. - assertEquals(Condition.ConditionType.UNPARK, factory.getType()); + assertEquals(ConditionType.UNPARK, factory.getType()); assertNotNull(condition); CompletableFuture future = condition.execute(); assertSame(condition, future.get(2, TimeUnit.SECONDS)); @@ -103,7 +105,7 @@ public void factoryBuildsConditionMetAtEarliestUnparkTime() throws Exception { @Test public void factoryReturnsNullWhenNoSynchronizerIsParked() { executor = Executors.newScheduledThreadPool(1); - UnparkCondition.Factory factory = new UnparkCondition.Factory(executor, () -> null); + DeadlineCondition.Factory factory = new DeadlineCondition.Factory(executor, ConditionType.UNPARK, () -> null); // Build a condition while no synchronizer is parked. Condition condition = factory.build(); diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index 612c12c4..a68137ed 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -122,7 +122,8 @@ public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -175,7 +176,8 @@ public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() thro /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -229,7 +231,8 @@ public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() th /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -279,7 +282,8 @@ public void allInitializersFailSwitchesToSynchronizers() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -326,7 +330,8 @@ public void allThreeInitializersFailWithNoSynchronizers() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -371,7 +376,8 @@ public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -411,7 +417,8 @@ public void noInitializersOneSynchronizerIsWellBehaved() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -450,7 +457,8 @@ public void oneInitializerOneSynchronizerIsWellBehaved() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -476,7 +484,8 @@ public void noInitializersAndNoSynchronizersIsWellBehaved() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -520,7 +529,8 @@ public void errorWithFDv1FallbackTriggersFallback() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -569,7 +579,8 @@ public void fallbackAndRecoveryTasksWellBehaved() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -627,7 +638,8 @@ public void canDisposeWhenSynchronizersFallingBack() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -679,7 +691,8 @@ public void terminalErrorFailsOverToNextSynchronizer() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -736,7 +749,8 @@ public void terminalErrorParksSynchronizerAndRestartsItAfterUnparkTime() throws /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), /* parkMaxDelay= */ Duration.ofSeconds(1), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system with its only synchronizer. @@ -770,7 +784,8 @@ public void failedNextParksSynchronizer() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), /* parkMaxDelay= */ Duration.ofSeconds(1), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system with its only synchronizer. @@ -813,7 +828,8 @@ public void close() { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), /* parkMaxDelay= */ Duration.ofSeconds(1), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system with its only synchronizer. @@ -847,7 +863,8 @@ public void constructionFailureParksSynchronizer() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), /* parkMaxDelay= */ Duration.ofSeconds(1), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system with its only synchronizer. @@ -884,7 +901,8 @@ public void resetThresholdMetReturnsParkDelayToBase() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), /* parkMaxDelay= */ Duration.ofSeconds(10), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system, and record when the synchronizer is built for the third and fourth times. @@ -926,7 +944,8 @@ public void fallbackWithNoOtherAvailableSynchronizerKeepsCurrentSynchronizer() t /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system. After the fallback condition of the secondary synchronizer is met, send a change set @@ -969,7 +988,8 @@ public void unparkWhileCurrentSynchronizerIsHealthyKeepsIt() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(50), /* parkMaxDelay= */ Duration.ofSeconds(1), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system, and send another change set after the primary synchronizer unparks. @@ -1016,7 +1036,8 @@ public void fallbackSwitchesWhenParkedSynchronizerUnparks() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), /* parkMaxDelay= */ Duration.ofSeconds(1), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system. @@ -1054,7 +1075,8 @@ public void changeSetAfterFallbackIsMetPreventsSwitchAtUnpark() throws Exception /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(160), /* parkMaxDelay= */ Duration.ofSeconds(1), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system, send a change set after the fallback is met, and another after the unpark. @@ -1105,7 +1127,8 @@ public void recoveryWithPreferredSynchronizerParkedWaitsForItsUnpark() throws Ex /* recoveryTimeout= */ Duration.ofMillis(50), /* parkInitialDelay= */ Duration.ofMillis(200), /* parkMaxDelay= */ Duration.ofSeconds(1), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system. @@ -1141,7 +1164,8 @@ public void changeSetDuringInterruptionRestartsFallbackTimeout() throws Exceptio /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system. Before the fallback timeout, the primary delivers a change set and is interrupted again. @@ -1160,6 +1184,81 @@ public void changeSetDuringInterruptionRestartsFallbackTimeout() throws Exceptio assertTrue("fell back after " + fallbackMillis + "ms", fallbackMillis >= 150); } + // A synchronizer that shuts down without being closed is parked, like one with any other unexpected failure. + @Test + public void unpromptedShutdownParksSynchronizer() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + + // The first instance shuts down, and the second instance delivers data. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(buildCount.incrementAndGet() == 1 + ? FDv2SourceResult.shutdown() + : FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); + } + ); + + // The synchronizer is parked for 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start().get(2, TimeUnit.SECONDS); + + // The synchronizer is built again after its unpark time, and its data initializes the data system. + assertEquals(2, buildCount.get()); + assertTrue(dataSource.isInitialized()); + } + + // While every synchronizer is parked, the synchronizer that was parked last is stopped. + @Test + public void parkedSynchronizerIsClosedWhileAllSynchronizersAreParked() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + CountDownLatch closed = new CountDownLatch(1); + + // The only synchronizer fails with a terminal error, and records when it is closed. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results) { + @Override + public void close() { + closed.countDown(); + super.close(); + } + }; + } + ); + + // The synchronizer is parked for much longer than the test. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // The parked synchronizer is closed while it waits for its unpark time. + assertTrue(closed.await(1, TimeUnit.SECONDS)); + } + @Test public void allSynchronizersParkedRestartsAfterEarliestUnparkTime() throws Exception { executor = Executors.newScheduledThreadPool(2); @@ -1195,7 +1294,8 @@ public void allSynchronizersParkedRestartsAfterEarliestUnparkTime() throws Excep /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), /* parkMaxDelay= */ Duration.ofSeconds(1), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system, and wait for each synchronizer to be built and then for one more build. @@ -1254,7 +1354,8 @@ public void disabledDataSourceCannotTriggerActions() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1305,7 +1406,8 @@ public void disposeCompletesStartFuture() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1337,7 +1439,8 @@ public void noSourcesProvidedCompletesImmediately() throws Exception{ /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1382,7 +1485,8 @@ public void startFutureCompletesExactlyOnce() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1410,7 +1514,8 @@ public void concurrentCloseAndStartHandledSafely() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); @@ -1439,7 +1544,8 @@ public void multipleStartCallsEventuallyComplete() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture1 = dataSource.start(); @@ -1472,7 +1578,8 @@ public void isInitializedThreadSafe() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); dataSource.start(); @@ -1512,7 +1619,8 @@ public void dataSourceUpdatesApplyThreadSafe() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1553,7 +1661,8 @@ public void initializerThrowsExecutionException() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1590,7 +1699,8 @@ public void initializerThrowsInterruptedException() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1625,7 +1735,8 @@ public void initializerThrowsCancellationException() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1659,7 +1770,8 @@ public void synchronizerNextThrowsExecutionException() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1696,7 +1808,8 @@ public void synchronizerNextThrowsInterruptedException() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1731,7 +1844,8 @@ public void synchronizerNextThrowsCancellationException() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1758,7 +1872,8 @@ public void closeWithoutStartDoesNotHang() { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); dataSource.close(); @@ -1785,7 +1900,8 @@ public void closeAfterInitializersCompletesImmediately() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1821,7 +1937,8 @@ public void close() { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1851,7 +1968,8 @@ public void multipleCloseCallsAreIdempotent() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1882,7 +2000,8 @@ public void closingDataSourceDuringInitializationReportsOffWithoutErrors() throw /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1927,7 +2046,8 @@ public void dataSourceClosedDuringSynchronizationReportsOffWithoutError() throws /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1969,7 +2089,8 @@ public void closeInterruptsConditionWaiting() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -2012,7 +2133,8 @@ public CompletableFuture run() { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); dataSource.close(); Future startFuture = dataSource.start(); @@ -2054,7 +2176,8 @@ public void close() { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2093,7 +2216,8 @@ public void close() { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -2130,7 +2254,8 @@ public void setActiveSourceOnInitializerChecksShutdown() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); @@ -2196,7 +2321,8 @@ public void parkedSynchronizerSkippedInRotation() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2242,7 +2368,8 @@ public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2287,7 +2414,8 @@ public void fallbackMovesToNextSynchronizer() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2322,7 +2450,8 @@ public void conditionsClosedAfterSynchronizerLoop() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2354,7 +2483,8 @@ public void conditionsInformedOfAllResults() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(20), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2387,7 +2517,8 @@ public void conditionsClosedOnException() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2417,7 +2548,8 @@ public void primeSynchronizerHasNoRecoveryCondition() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2453,7 +2585,8 @@ public void nonPrimeSynchronizerHasBothConditions() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2482,7 +2615,8 @@ public void singleSynchronizerHasNoConditions() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2512,7 +2646,8 @@ public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2547,7 +2682,8 @@ public void changeSetAppliedToDataSourceUpdates() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2578,7 +2714,8 @@ public void multipleChangeSetsAppliedInOrder() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2616,7 +2753,8 @@ public void selectorNonEmptyCompletesInitialization() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2650,7 +2788,8 @@ public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() thro /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2688,7 +2827,8 @@ public void synchronizerChangeSetAlwaysCompletesStartFuture() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2722,7 +2862,8 @@ public void goodbyeStatusHandledGracefully() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2761,7 +2902,8 @@ public void shutdownStatusExitsImmediately() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2793,7 +2935,8 @@ public void fdv1FallbackFlagHonored() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2831,7 +2974,8 @@ public void emptyInitializerListSkipsToSynchronizers() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2864,7 +3008,8 @@ public void startedFlagPreventsMultipleRuns() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture1 = dataSource.start(); @@ -2899,7 +3044,8 @@ public void startBeforeRunCompletesAllComplete() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1)); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Call start multiple times before completion @@ -2942,7 +3088,8 @@ public void statusTransitionsToValidAfterInitialization() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -2991,7 +3138,8 @@ public void statusIncludesErrorInfoOnFailure() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -3035,7 +3183,8 @@ public void statusRemainsValidDuringSynchronizerOperation() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -3094,7 +3243,8 @@ public void statusStaysInterruptedWhileOnlySynchronizerIsParked() throws Excepti /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), /* parkMaxDelay= */ Duration.ofSeconds(1), - /* failoverDelay= */ Duration.ofMillis(10) + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -3155,7 +3305,8 @@ public void fdv1FallbackActivatesWhenFlagSet() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -3204,7 +3355,8 @@ public void fdv1FallbackSynchronizerIsParkedAndRestartedDuringDirectedFallback() /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), /* parkMaxDelay= */ Duration.ofSeconds(1), - /* failoverDelay= */ Duration.ofMillis(10)); + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Start the data system. @@ -3251,7 +3403,8 @@ public void fdv1FallbackNotCalledWithoutFlag() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -3312,7 +3465,8 @@ public void fdv1FallbackWorksAfterInterruption() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -3328,42 +3482,184 @@ public void fdv1FallbackWorksAfterInterruption() throws Exception { assertTrue("Should have at least 2 changesets", sink.getApplyCount() >= 2); } - // Synchronizer-phase analogue of the initializer halt path: when a synchronizer signals - // FDv1 fallback but no FDv1 fallback synchronizer is configured, the data system must halt - // (transition to OFF, stop building further synchronizers). + // Synchronizer-phase analogue of the initializer recovery path: when a synchronizer signals FDv1 fallback but no + // FDv1 fallback synchronizer is configured, the data system reports INTERRUPTED, runs no synchronizer until the + // fallback TTL ends, and then restarts the FDv2 synchronizers. @Test - public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredHaltsDataSystem() throws Exception { + public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredRecoversAfterTTL() throws Exception { executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - // First synchronizer sends a normal payload, then a payload with the directive. - BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); - firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), true)); - - AtomicInteger firstSyncBuildCount = new AtomicInteger(0); + AtomicInteger firstSyncBuildCount = new AtomicInteger(); AtomicBoolean secondSyncCalled = new AtomicBoolean(false); - // Two synchronizers configured; once the directive halts the data system, neither - // should be re-built. The second synchronizer in particular must never run. ImmutableList> synchronizers = ImmutableList.of( + // The first instance sends a payload, then one with the directive. The second sends a payload. () -> { - firstSyncBuildCount.incrementAndGet(); - return new MockQueuedSynchronizer(firstSyncResults); + BlockingQueue results = new LinkedBlockingQueue<>(); + if (firstSyncBuildCount.incrementAndGet() == 1) { + results.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + results.add(FDv2SourceResult.changeSet(makeChangeSet(false), true)); + } else { + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + } + return new MockQueuedSynchronizer(results); }, + // The second synchronizer must never run. () -> { secondSyncCalled.set(true); return new MockQueuedSynchronizer(new LinkedBlockingQueue<>()); } ); - // No FDv1 fallback configured. + // No FDv1 fallback synchronizer is configured, and the fallback TTL ends after 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofMillis(100)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start().get(2, TimeUnit.SECONDS); + + // The data system reports INTERRUPTED, and restarts the first synchronizer after the TTL. + List statuses = sink.awaitStatuses(4, 2, TimeUnit.SECONDS); + assertEquals(Arrays.asList( + DataSourceStatusProvider.State.VALID, + DataSourceStatusProvider.State.VALID, + DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.State.VALID), statuses); + assertEquals(2, firstSyncBuildCount.get()); + assertFalse(secondSyncCalled.get()); + } + + // While no FDv1 fallback synchronizer is configured, the FDv2 synchronizer that sent the directive is stopped. + @Test + public void directiveWithoutFDv1ClosesFDv2SynchronizerDuringFallback() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + CountDownLatch closed = new CountDownLatch(1); + + // The synchronizer sends a payload with the directive, and records when it is closed. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), true)); + return new MockQueuedSynchronizer(results) { + @Override + public void close() { + closed.countDown(); + super.close(); + } + }; + } + ); + + // No FDv1 fallback synchronizer is configured, and the fallback lasts much longer than the test. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // The synchronizer is closed during the fallback. + assertTrue(closed.await(1, TimeUnit.SECONDS)); + } + + // When the fallback TTL ends, the data system stops the FDv1 synchronizer and restarts the FDv2 synchronizers. + @Test + public void fdv1FallbackEndsAfterTTL() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger fdv2BuildCount = new AtomicInteger(); + AtomicBoolean fdv1Closed = new AtomicBoolean(false); + + ImmutableList> synchronizers = ImmutableList.of( + // The first instance sends a payload with the directive. The second sends a payload. + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), fdv2BuildCount.incrementAndGet() == 1)); + return new MockQueuedSynchronizer(results); + } + ); + + // The FDv1 fallback synchronizer sends a payload, and records when it is closed. + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results) { + @Override + public void close() { + fdv1Closed.set(true); + super.close(); + } + }; + }; + + // The fallback TTL ends after 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofMillis(100)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // After the TTL, the data system has stopped the FDv1 fallback synchronizer and applied FDv2 data again. + sink.awaitApplyCount(3, 2, TimeUnit.SECONDS); + assertEquals(2, fdv2BuildCount.get()); + assertTrue(fdv1Closed.get()); + } + + // When the fallback TTL ends before the FDv1 synchronizer's unpark time, the FDv2 synchronizers restart at the TTL. + @Test + public void fdv1FallbackEndsWhileFDv1SynchronizerIsParked() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger fdv2BuildCount = new AtomicInteger(); + AtomicInteger fdv1BuildCount = new AtomicInteger(); + + // The first instance reports an interruption with the directive. The second sends a payload. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + if (fdv2BuildCount.incrementAndGet() == 1) { + results.add(FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + true)); + } else { + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + } + return new MockQueuedSynchronizer(results); + } + ); + + // The FDv1 fallback synchronizer fails, so the data system parks it. + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + fdv1BuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + }; + + // The FDv1 fallback synchronizer is parked for minutes, and the fallback TTL ends after 100-200ms. FDv2DataSource dataSource = new FDv2DataSource( - initializers, + ImmutableList.of(), synchronizers, - null, + fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, @@ -3372,38 +3668,129 @@ public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredHaltsDataSystem() thr /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofMillis(200) ); resourcesToClose.add(dataSource); - dataSource.start().get(2, TimeUnit.SECONDS); + // Start the data system. + dataSource.start(); - // First payload is applied; the directive-bearing payload is also applied (the SDK - // surfaces the data) but no further synchronizers are dialled. - sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + // The FDv2 synchronizer is built again after the TTL, and its data is applied. + sink.awaitApplyCount(1, 2, TimeUnit.SECONDS); + assertEquals(1, sink.getApplyCount()); + assertEquals(2, fdv2BuildCount.get()); + assertEquals(1, fdv1BuildCount.get()); + } + + // A directive during the FDv1 fallback replaces the TTL, and the FDv1 synchronizer keeps running. + @Test + public void directiveDuringFDv1FallbackExtendsTTL() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger fdv2BuildCount = new AtomicInteger(); + AtomicInteger fdv1BuildCount = new AtomicInteger(); + BlockingQueue fdv2RestartTimes = new LinkedBlockingQueue<>(); + IterableAsyncQueue fdv1Results = new IterableAsyncQueue<>(); - // Status must end up OFF because the directive could not be satisfied. - // Poll briefly for the OFF state in case the final transition is observed slightly - // after the apply count reaches 2; this keeps the assertion deterministic without - // relying on real-time delays. - DataSourceStatusProvider.State finalState = null; - for (int i = 0; i < 20 && finalState != DataSourceStatusProvider.State.OFF; i++) { - DataSourceStatusProvider.State next = sink.awaitStatus(100, TimeUnit.MILLISECONDS); - if (next != null) { - finalState = next; - } else if (sink.getLastState() == DataSourceStatusProvider.State.OFF) { - finalState = DataSourceStatusProvider.State.OFF; + // The first instance sends a payload with the directive. The second records when it is built. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + if (fdv2BuildCount.incrementAndGet() == 1) { + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), true)); + } else { + fdv2RestartTimes.offer(System.nanoTime()); + } + return new MockQueuedSynchronizer(results); } + ); + + // The FDv1 fallback synchronizer sends the results that the test puts in its queue. + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + fdv1BuildCount.incrementAndGet(); + return new MockQueuedSynchronizer(fdv1Results); + }; + + // The fallback TTL ends after 100-200ms. + FDv2DataSource dataSource = new FDv2DataSource( + ImmutableList.of(), + synchronizers, + fdv1Fallback, + sink, + Thread.NORM_PRIORITY, + logger, + executor, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofMillis(200) + ); + resourcesToClose.add(dataSource); + + // Start the data system. FDv1 sends a directive every 50ms until 300ms, which is longer than the TTL. + long start = System.nanoTime(); + dataSource.start(); + for (int delay = 50; delay <= 300; delay += 50) { + executor.schedule( + () -> fdv1Results.put(FDv2SourceResult.changeSet(makeChangeSet(false), true)), + delay, + TimeUnit.MILLISECONDS); } - assertEquals("Status should be OFF after directive halts the data system", - DataSourceStatusProvider.State.OFF, sink.getLastState()); - - // The second synchronizer must never be built. - assertFalse("Subsequent synchronizers must not run after directive-induced halt", - secondSyncCalled.get()); - // The first synchronizer should also not be re-built (we should stay halted). - assertEquals("First synchronizer should be built exactly once", - 1, firstSyncBuildCount.get()); + + // FDv2 restarts at least the shortest TTL after the last directive, and FDv1 never restarts. + Long fdv2Restart = fdv2RestartTimes.poll(2, TimeUnit.SECONDS); + assertNotNull(fdv2Restart); + long restartMillis = TimeUnit.NANOSECONDS.toMillis(fdv2Restart - start); + assertTrue("restarted FDv2 after " + restartMillis + "ms", restartMillis >= 400); + assertEquals(1, fdv1BuildCount.get()); + } + + // When the FDv2 synchronizers send the directive again after the fallback TTL ends, the data system falls back again. + @Test + public void directiveDuringRecoveryFallsBackToFDv1Again() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger fdv2BuildCount = new AtomicInteger(); + AtomicInteger fdv1BuildCount = new AtomicInteger(); + + // Every instance of the FDv2 synchronizer sends a payload with the directive. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + fdv2BuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), true)); + return new MockQueuedSynchronizer(results); + } + ); + + // Every instance of the FDv1 fallback synchronizer sends a payload. + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + fdv1BuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); + }; + + // The fallback TTL ends after 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, executor, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofMillis(100)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // After the TTL, the FDv2 synchronizer sends the directive again, and the data system starts FDv1 again. + sink.awaitApplyCount(4, 2, TimeUnit.SECONDS); + assertEquals(2, fdv2BuildCount.get()); + assertEquals(2, fdv1BuildCount.get()); } @Test @@ -3451,7 +3838,8 @@ public void fdv1FallbackBlocksOtherSynchronizers() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -3506,7 +3894,8 @@ public void fdv1FallbackOnlyCalledOncePerDataSource() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -3571,7 +3960,8 @@ public void fdv1FallbackOnInitializerSuccessAppliesPayloadAndSwitches() throws E /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -3641,7 +4031,8 @@ public void fdv1FallbackOnInitializerErrorSwitchesToFDv1() throws Exception { /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -3654,14 +4045,15 @@ public void fdv1FallbackOnInitializerErrorSwitchesToFDv1() throws Exception { fdv2SyncCalled.get()); } - // When an initializer signals FDv1 fallback but no FDv1 synchronizer is configured, the - // SDK must transition the data source status to OFF -- not stay stuck at INITIALIZING -- - // and surface the underlying initializer error so monitors can see why. + // When an initializer signals FDv1 fallback but no FDv1 synchronizer is configured, the data system reports + // INTERRUPTED, runs no synchronizer until the fallback TTL ends, and then starts the FDv2 synchronizers. @Test - public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredTransitionsToOff() throws Exception { - executor = Executors.newScheduledThreadPool(1); + public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredRecoversAfterTTL() throws Exception { + executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger syncBuildCount = new AtomicInteger(); + // The initializer fails with the directive. DataSourceStatusProvider.ErrorInfo initError = new DataSourceStatusProvider.ErrorInfo( DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, 500, @@ -3675,41 +4067,34 @@ public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredTransitionsToOff() thr () -> new MockInitializer(initFuture) ); - AtomicBoolean fdv2SyncCalled = new AtomicBoolean(false); + // The synchronizer delivers data. ImmutableList> synchronizers = ImmutableList.of( () -> { - fdv2SyncCalled.set(true); - return new MockQueuedSynchronizer(new LinkedBlockingQueue<>()); + syncBuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); } ); - // No FDv1 fallback configured. - FDv2DataSource dataSource = new FDv2DataSource( - initializers, - synchronizers, - null, - sink, - Thread.NORM_PRIORITY, - logger, - executor, + // No FDv1 fallback synchronizer is configured, and the fallback TTL ends after 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) - ); + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofMillis(100)); resourcesToClose.add(dataSource); + // Start the data system, and wait for it to be initialized by the synchronizer. dataSource.start().get(2, TimeUnit.SECONDS); - // Status must end up OFF (not INITIALIZING) so callers can observe the terminal state. - assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); - assertNotNull("Initializer error should be preserved on the OFF status", sink.getLastError()); - assertEquals(initError.getKind(), sink.getLastError().getKind()); - assertEquals(initError.getStatusCode(), sink.getLastError().getStatusCode()); - - // FDv2 synchronizers should not run. - assertFalse(fdv2SyncCalled.get()); + // The data system never reported OFF, and started the synchronizer after the TTL. + List statuses = sink.awaitStatuses(3, 2, TimeUnit.SECONDS); + assertFalse(statuses.contains(DataSourceStatusProvider.State.OFF)); + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); + assertEquals(1, syncBuildCount.get()); } // When an initializer returns a successful payload-without-selector AND the FDv1 fallback @@ -3754,7 +4139,8 @@ public void fdv1FallbackOnInitializerSuccessNoSelectorAppliesPayloadAndSwitches( /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -3789,7 +4175,8 @@ public void orchestrationLogging_warnsWhenNoInitializersOrSynchronizersConfigure /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3823,7 +4210,8 @@ public void orchestrationLogging_logsInitializerStartSuccessAndErrorOnException( /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3863,7 +4251,8 @@ public void orchestrationLogging_logsInitializerFailedOnStatusResult() throws Ex /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3902,10 +4291,13 @@ public void orchestrationLogging_logsSynchronizerStartingStatusInterruptedAndShu /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); - dataSource.start().get(2, TimeUnit.SECONDS); + dataSource.start(); + // Wait for the status from the interruption, and the status from parking the synchronizer after it shuts down. + sink.awaitStatuses(2, 2, TimeUnit.SECONDS); assertTrue(logTextContains(logCapture, LDLogLevel.INFO, "Synchronizer 'primary-sync' is starting.")); assertTrue(logTextContains(logCapture, LDLogLevel.INFO, "Synchronizer 'primary-sync' reported status: INTERRUPTED.")); assertTrue(logTextContains(logCapture, LDLogLevel.INFO, "Synchronizer 'primary-sync' reported status: SHUTDOWN.")); @@ -3937,7 +4329,8 @@ public void orchestrationLogging_logsSynchronizerParkedAndAllParked() throws Exc /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), /* parkMaxDelay= */ Duration.ofSeconds(1), - /* failoverDelay= */ Duration.ofMillis(10) + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -3958,9 +4351,9 @@ public void orchestrationLogging_logsSynchronizerParkedAndAllParked() throws Exc public void orchestrationLogging_logsErrorWhenSynchronizerNextThrows() throws Exception { executor = Executors.newScheduledThreadPool(1); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - // The second synchronizer shuts down at once. That ends the run, so start() completes. + // The second synchronizer delivers data, so start() completes. IterableAsyncQueue followUpQueue = new IterableAsyncQueue<>(); - followUpQueue.put(FDv2SourceResult.shutdown()); + followUpQueue.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)); ImmutableList> synchronizers = ImmutableList.of( // The first synchronizer fails from next(). () -> new MockSynchronizer(() -> { @@ -3980,7 +4373,8 @@ public void orchestrationLogging_logsErrorWhenSynchronizerNextThrows() throws Ex /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1) + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); From 78908ac029d988b50a532229cdcb9d7016fe21d5 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Thu, 5 Nov 2026 15:39:33 -0800 Subject: [PATCH 07/13] refactor: Replace synchronizer conditions with deadlines checked on each wake --- .../sdk/server/FDv2DataSource.java | 472 +++++------------- .../sdk/server/FDv2DataSourceConditions.java | 255 ---------- ...FDv2DataSourceConditionsAggregateTest.java | 331 ------------ .../FDv2DataSourceDeadlineConditionTest.java | 116 ----- .../FDv2DataSourceFallbackConditionTest.java | 351 ------------- .../FDv2DataSourceRecoveryConditionTest.java | 330 ------------ 6 files changed, 119 insertions(+), 1736 deletions(-) delete mode 100644 lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java delete mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java delete mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceDeadlineConditionTest.java delete mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java delete mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 703f09f3..17ab88a3 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -4,10 +4,6 @@ import com.launchdarkly.logging.LDLogger; import com.launchdarkly.logging.LogValues; import com.launchdarkly.sdk.fdv2.SourceSignal; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.ConditionFactory; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.FallbackCondition; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.RecoveryCondition; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; import com.launchdarkly.sdk.server.datasources.Initializer; import com.launchdarkly.sdk.server.datasources.Synchronizer; @@ -18,25 +14,12 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.Date; -import java.util.EnumMap; -import java.util.EnumSet; import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.WeakHashMap; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; -import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.ConditionFactory; -import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.FallbackCondition; -import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.RecoveryCondition; -import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.DeadlineCondition; - class FDv2DataSource implements DataSource { /** * Default fallback timeout of 2 minutes. @@ -70,15 +53,11 @@ class FDv2DataSource implements DataSource { private final SourceManager sourceManager; - private final Map conditionFactories = new EnumMap<>( - Condition.ConditionType.class); - + private final Duration fallbackTimeout; + private final Duration recoveryTimeout; private final Duration parkInitialDelay; - private final Duration parkMaxDelay; - private final Duration failoverDelay; - private final Duration defaultFDv1FallbackTTL; private final DataSourceUpdateSinkV2 dataSourceUpdates; @@ -170,18 +149,8 @@ public FDv2DataSource( this.dataSourceUpdates = dataSourceUpdates; this.threadPriority = threadPriority; this.logger = logger; - this.conditionFactories.put( - Condition.ConditionType.FALLBACK, new FallbackCondition.Factory(sharedExecutor, fallbackTimeout)); - this.conditionFactories.put( - Condition.ConditionType.RECOVERY, new RecoveryCondition.Factory(sharedExecutor, recoveryTimeout)); - this.conditionFactories.put( - Condition.ConditionType.UNPARK, - new DeadlineCondition.Factory( - sharedExecutor, Condition.ConditionType.UNPARK, sourceManager::getEarliestUnparkTime)); - this.conditionFactories.put( - Condition.ConditionType.FDV1_FALLBACK_END, - new DeadlineCondition.Factory( - sharedExecutor, Condition.ConditionType.FDV1_FALLBACK_END, sourceManager::getFDv1FallbackEndTime)); + this.fallbackTimeout = fallbackTimeout; + this.recoveryTimeout = recoveryTimeout; this.parkInitialDelay = parkInitialDelay; this.parkMaxDelay = parkMaxDelay; this.failoverDelay = failoverDelay; @@ -359,43 +328,6 @@ static InitializerOutcome fallbackToFDv1(DataSourceStatusProvider.ErrorInfo erro } } - /** - * Determine conditions for the current synchronizer. Synchronizers require different conditions depending on if - * they are the 'prime' synchronizer or if there are other available synchronizers to use. - * - * @return a list of conditions to apply to the synchronizer - */ - private List getConditions() { - int availableSynchronizers = sourceManager.getAvailableSynchronizerCount(); - boolean isPrimeSynchronizer = sourceManager.isPrimeSynchronizer(); - - if (availableSynchronizers == 1) { - // With only 1 synchronizer, we cannot fall back or recover. Only the end of the FDv1 fallback applies. - Condition fallbackEnd = buildCondition(Condition.ConditionType.FDV1_FALLBACK_END); - return fallbackEnd == null ? Collections.emptyList() : Collections.singletonList(fallbackEnd); - } - if (isPrimeSynchronizer) { - // If there isn't a synchronizer to recover to, then don't add and recovery conditions. - return conditionFactories.values().stream() - .filter((ConditionFactory factory) -> factory.getType() != Condition.ConditionType.RECOVERY) - .map(ConditionFactory::build).filter(Objects::nonNull).collect(Collectors.toList()); - } - // The synchronizer can both fall back and recover. - return conditionFactories.values().stream() - .map(ConditionFactory::build).filter(Objects::nonNull).collect(Collectors.toList()); - } - - /** - * Builds the condition of the given type. - * - * @param type the type of condition - * @return the condition, or null if it does not apply - */ - private Condition buildCondition(Condition.ConditionType type) { - ConditionFactory factory = conditionFactories.get(type); - return factory == null ? null : factory.build(); - } - /** * Runs the configured synchronizers, falling back / recovering as conditions allow, * until the data source is closed or a synchronizer reports that it shut down. @@ -413,85 +345,31 @@ private void runSynchronizers() { boolean parked = false; try { boolean running = true; + // Fallback and recovery apply only when another synchronizer can take over. + boolean canFallBack = sourceManager.getAvailableSynchronizerCount() > 1; + boolean canRecover = canFallBack && !sourceManager.isPrimeSynchronizer(); + Instant recoveryTime = canRecover ? Instant.now().plus(recoveryTimeout) : null; + Instant fallbackTime = null; + // True once the loop has seen the deadline pass. + boolean recoveryMet = false; + boolean fallbackMet = false; + // The result the loop is waiting for, or null if it has not asked the synchronizer for one. + CompletableFuture nextResultFuture = null; + while (running) { + if (nextResultFuture == null) { + nextResultFuture = synchronizer.next(); + } - try (Conditions conditions = new Conditions(getConditions())) { - // A condition doesn't necessarily stop the current synchronizer, so keep track of its pending result. - CompletableFuture nextResultFuture = null; - while (running) { - if (nextResultFuture == null) { - nextResultFuture = synchronizer.next(); - } - - // The conditions future completes when a condition is met. - Object res = CompletableFuture.anyOf(conditions.getFuture(), nextResultFuture).get(); - - if (res instanceof Condition) { - Condition.ConditionType metType = ((Condition) res).getType(); - // Clear expired unpark times before checking which synchronizers are available. - sourceManager.unparkExpiredSynchronizers(); - if (metType == Condition.ConditionType.UNPARK) { - // Replace the unpark condition with one for the next unpark. - conditions.replace( - Condition.ConditionType.UNPARK, - buildCondition(Condition.ConditionType.UNPARK)); - } - if (metType == Condition.ConditionType.FDV1_FALLBACK_END) { - if (sourceManager.endFDv1FallbackIfExpired()) { - logger.info("Fallback TTL elapsed, restarting FDv2 data sources."); - break; - } - // The timer fired before the end time. This shouldn't normally happen. Wait for the remaining time. - conditions.replace( - Condition.ConditionType.FDV1_FALLBACK_END, - buildCondition(Condition.ConditionType.FDV1_FALLBACK_END)); - } - - // An unpark does not switch synchronizers by itself, but other conditions might already - // be met. Check all of them before deciding whether to switch. Recovery takes precedence. - if (conditions.isMet(Condition.ConditionType.RECOVERY) - && sourceManager.hasMorePreferredAvailableSynchronizer()) { - // For recovery, we will start at the first available synchronizer. - // So we reset the source index, and finding the source will start at the beginning. - sourceManager.resetSourceIndex(); - logger.info( - "Recovery condition met, moving from synchronizer '{}' to primary synchronizer.", - synchronizer.name()); - break; - } - if (conditions.isMet(Condition.ConditionType.FALLBACK) - && sourceManager.hasOtherAvailableSynchronizer()) { - logger.info( - "Fallback condition met, falling back from synchronizer '{}'.", - synchronizer.name()); - break; - } - - // Keep running this synchronizer. - if (metType == Condition.ConditionType.FALLBACK) { - logger.info( - "Fallback condition met, but no other synchronizer is available; continuing with synchronizer '{}'.", - synchronizer.name()); - } else if (metType == Condition.ConditionType.RECOVERY) { - logger.info( - "Recovery condition met, but no more-preferred synchronizer is available; continuing with synchronizer '{}'.", - synchronizer.name()); - } - continue; - } - - // The pending result has arrived. - nextResultFuture = null; - - if (!(res instanceof FDv2SourceResult)) { - logger.error("Unexpected result type from synchronizer: {}", res.getClass().getName()); - continue; - } - - try (FDv2SourceResult result = (FDv2SourceResult) res) { - Set typesToReplace = conditions.inform(result); - for (Condition.ConditionType type : typesToReplace) { - conditions.replace(type, buildCondition(type)); - } + // Wait for the next result, or for the next deadline that the loop has not seen pass. + Instant wakeTime = earliest( + recoveryMet ? null : recoveryTime, + fallbackMet ? null : fallbackTime, + sourceManager.getEarliestUnparkTime(), + sourceManager.getFDv1FallbackEndTime()); + if (awaitUntil(nextResultFuture, wakeTime)) { + try (FDv2SourceResult result = nextResultFuture.get()) { + // The pending result has arrived. + nextResultFuture = null; if (result.isResetThresholdMet()) { sourceManager.resetCurrentSynchronizerParkCount(); @@ -501,6 +379,9 @@ private void runSynchronizers() { case CHANGE_SET: // A data update breaks the "in a row" streak for status deduplication. resetSynchronizerStatusDedupe(); + // The synchronizer is healthy, so it no longer needs to fall back. + fallbackTime = null; + fallbackMet = false; dataSourceUpdates.apply(result.getChangeSet()); dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); // This could have been completed by any data source. But if it has not been completed before @@ -514,7 +395,10 @@ private void runSynchronizers() { maybeLogSynchronizerStatusChange( synchronizer.name(), status.getState()); - // Handled by conditions. + // The fallback timeout runs from the first interruption. + if (canFallBack && fallbackTime == null) { + fallbackTime = Instant.now().plus(fallbackTimeout); + } dataSourceUpdates.updateStatus( DataSourceStatusProvider.State.INTERRUPTED, status.getErrorInfo()); @@ -558,9 +442,6 @@ private void runSynchronizers() { // A directive during the FDv1 fallback replaces its TTL, and FDv1 keeps running. Duration fallbackTTL = jittered(defaultFDv1FallbackTTL); sourceManager.startFDv1Fallback(Instant.now().plus(fallbackTTL)); - conditions.replace( - Condition.ConditionType.FDV1_FALLBACK_END, - buildCondition(Condition.ConditionType.FDV1_FALLBACK_END)); logger.info("FDv2 retry scheduled in {}s.", fallbackTTL.getSeconds()); } else { // A directive to fall back to FDv1 ends the run of an FDv2 synchronizer. @@ -574,6 +455,50 @@ private void runSynchronizers() { } } } + if (!running) { + break; + } + + // Check every deadline against the current time. + Instant now = Instant.now(); + sourceManager.unparkExpiredSynchronizers(); + if (sourceManager.endFDv1FallbackIfExpired()) { + logger.info("Fallback TTL elapsed, restarting FDv2 data sources."); + break; + } + boolean recoveryJustMet = !recoveryMet && recoveryTime != null && !now.isBefore(recoveryTime); + boolean fallbackJustMet = !fallbackMet && fallbackTime != null && !now.isBefore(fallbackTime); + recoveryMet = recoveryMet || recoveryJustMet; + fallbackMet = fallbackMet || fallbackJustMet; + + // Recovery takes precedence over fallback. + if (recoveryMet && sourceManager.hasMorePreferredAvailableSynchronizer()) { + // For recovery, we will start at the first available synchronizer. + // So we reset the source index, and finding the source will start at the beginning. + sourceManager.resetSourceIndex(); + logger.info( + "Recovery condition met, moving from synchronizer '{}' to primary synchronizer.", + synchronizer.name()); + break; + } + if (fallbackMet && sourceManager.hasOtherAvailableSynchronizer()) { + logger.info( + "Fallback condition met, falling back from synchronizer '{}'.", + synchronizer.name()); + break; + } + + // Keep running this synchronizer. + if (fallbackJustMet) { + logger.info( + "Fallback condition met, but no other synchronizer is available; continuing with synchronizer '{}'.", + synchronizer.name()); + } + if (recoveryJustMet) { + logger.info( + "Recovery condition met, but no more-preferred synchronizer is available; continuing with synchronizer '{}'.", + synchronizer.name()); + } } } catch (ExecutionException | InterruptedException | RuntimeException e) { DataSourceStatusProvider.ErrorInfo errorInfo = new DataSourceStatusProvider.ErrorInfo( @@ -636,6 +561,46 @@ private boolean waitUnlessClosed(Duration duration) { } } + /** + * Waits for the future to complete, or until the given time. + * + * @param future the future to wait for + * @param time the time to stop waiting, or null to wait until the future completes + * @return true if the future completed + * @throws InterruptedException if the thread is interrupted while waiting + */ + private static boolean awaitUntil(CompletableFuture future, Instant time) throws InterruptedException { + try { + if (time == null) { + future.get(); + } else { + future.get(Math.max(0, Duration.between(Instant.now(), time).toNanos()), TimeUnit.NANOSECONDS); + } + return true; + } catch (TimeoutException e) { + return false; + } catch (ExecutionException e) { + // The future completed with an error, which the caller gets from the future. + return true; + } + } + + /** + * Gets the earliest of the given times. + * + * @param times the times, any of which may be null + * @return the earliest time, or null if every time is null + */ + private static Instant earliest(Instant... times) { + Instant earliest = null; + for (Instant time : times) { + if (time != null && (earliest == null || time.isBefore(earliest))) { + earliest = time; + } + } + return earliest; + } + /** * Subtracts a random jitter of up to half of the given delay. * @@ -820,203 +785,4 @@ private void maybeReportUnexpectedExhaustion(String message) { new Date().toInstant())); } } - - /** - * Helper class to manage the lifecycle of conditions with automatic cleanup. - * - *

Before the aggregate completes, {@link #getFuture()} returns a - * fresh {@link CompletableFuture} per call. This matters because - * the run loop calls {@code CompletableFuture.anyOf(getFuture(), - * synchronizerNext)} on every iteration: if {@code getFuture()} returned - * the shared underlying aggregate while it was still pending, each - * {@code anyOf} call would permanently attach an {@code OrRelay} - * {@code Completion} to its {@code stack}. On a healthy primary - * synchronizer that streams ChangeSets without ever arming the fallback - * timer, the aggregate never completes, so those Completion nodes would - * accumulate monotonically for the synchronizer's full tenure -- a real - * memory leak proportional to event rate. - * - *

After the aggregate completes, {@link #getFuture()} returns the - * aggregate directly: any continuation registered on an already-completed - * CompletableFuture fires synchronously at registration time and is - * removed from the stack immediately by {@code cleanStack}, so the same - * accumulation cannot happen. - * - *

Fresh pre-completion futures are tracked in a {@link WeakHashMap}-backed - * set, so a fresh future whose only strong references were in the caller's - * loop iteration becomes garbage-collectable -- and automatically removed - * from {@code pending} -- once that iteration ends. - * - *

Package-private (rather than private) so that direct unit tests can - * exercise the API surface and assert per-call distinctness. - */ - static class Conditions implements AutoCloseable { - private final Object lock = new Object(); - - /** - * The conditions, with at most one of each type. Accessed only under {@code lock}. - */ - private final Map conditions = new EnumMap<>(Condition.ConditionType.class); - - /** - * The types of the conditions that the aggregate has completed with. Accessed only under {@code lock}. - */ - private final Set reported = EnumSet.noneOf(Condition.ConditionType.class); - - /** - * A future over the futures of all the conditions. It completes with the first condition that is met. - * Accessed only under {@code lock}. - */ - private CompletableFuture aggregate; - - /** - * Tracks futures previously returned by {@link #getFuture()} that have - * not yet been completed. Held weakly via {@link WeakHashMap} so that - * fresh futures abandoned by the caller (the typical end-of-iteration - * case) become GC-collectable. Set to {@code null} once the aggregate - * has fired and the entries have been drained. Accessed only under - * {@code lock}. - */ - private Set> pending; - - public Conditions(List conditions) { - for (Condition condition : conditions) { - this.conditions.put(condition.getType(), condition); - } - arm(); - } - - /** - * Builds the aggregate future over the conditions. - */ - private void arm() { - synchronized (lock) { - CompletableFuture[] unreported = conditions.values().stream() - .filter(c -> !reported.contains(c.getType())) - .map(Condition::execute) - .toArray(CompletableFuture[]::new); - CompletableFuture newAggregate = unreported.length == 0 - ? new CompletableFuture<>() // Never completes if no conditions are unreported - : CompletableFuture.anyOf(unreported); - aggregate = newAggregate; - // Start a new set only if the previous aggregate drained it. Otherwise, the futures in it are still - // waiting, and the new aggregate completes them. - if (pending == null) { - pending = Collections.newSetFromMap(new WeakHashMap<>()); - } - - // Single permanent listener. This is the only Completion node ever - // attached to aggregate.stack while the aggregate is still pending - // -- subsequent pre-completion getFuture() calls do not touch the - // aggregate at all. - newAggregate.whenComplete((result, throwable) -> { - List> snapshot; - synchronized (lock) { - // A newer aggregate replaced this one, and it completes the pending futures instead. - if (aggregate != newAggregate || pending == null) { - return; - } - if (throwable == null) { - reported.add(((Condition) result).getType()); - } - // Copy under the lock: the ArrayList holds strong - // references so entries that survived GC to this point - // stay alive until we complete them below. - snapshot = new ArrayList<>(pending); - pending = null; - } - for (CompletableFuture cf : snapshot) { - if (throwable != null) { - cf.completeExceptionally(throwable); - } else { - cf.complete(result); - } - } - }); - } - } - - /** - * Returns a future that completes with a met condition that this method has not reported before. - */ - public CompletableFuture getFuture() { - synchronized (lock) { - if (pending == null) { - // The aggregate has fired. - if (aggregate.isCompletedExceptionally()) { - return aggregate; - } - // Wait for the conditions that have not been reported. - arm(); - if (pending == null) { - // Another condition was already met, so the new aggregate has fired too. - return aggregate; - } - } - CompletableFuture fresh = new CompletableFuture<>(); - pending.add(fresh); - return fresh; - } - } - - /** - * True if the condition of the given type has been reported as met. - * - * @param type the type of condition - * @return true if the condition is met - */ - public boolean isMet(Condition.ConditionType type) { - synchronized (lock) { - return reported.contains(type); - } - } - - /** - * Closes the condition of the given type, if there is one, and puts the replacement in its place. - * - * @param type the type of condition - * @param replacement the new condition, or null to leave no condition of that type - */ - public void replace(Condition.ConditionType type, Condition replacement) { - synchronized (lock) { - Condition old = conditions.remove(type); - if (old != null) { - old.close(); - } - reported.remove(type); - if (replacement != null) { - conditions.put(type, replacement); - } - arm(); - } - } - - /** - * Informs the conditions of a result from the synchronizer. - * - * @param result the result - * @return the types of the conditions that should be replaced with new ones - */ - public Set inform(FDv2SourceResult result) { - synchronized (lock) { - Set typesToReplace = EnumSet.noneOf(Condition.ConditionType.class); - for (Condition condition : conditions.values()) { - if (condition.inform(result)) { - typesToReplace.add(condition.getType()); - } - } - return typesToReplace; - } - } - - @Override - public void close() { - synchronized (lock) { - conditions.values().forEach(Condition::close); - if (pending != null) { - pending.clear(); - } - } - } - } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java deleted file mode 100644 index 0b9b2049..00000000 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java +++ /dev/null @@ -1,255 +0,0 @@ -package com.launchdarkly.sdk.server; - -import com.launchdarkly.sdk.fdv2.SourceResultType; -import com.launchdarkly.sdk.fdv2.SourceSignal; -import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; - -import java.io.Closeable; -import java.io.IOException; -import java.time.Duration; -import java.time.Instant; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; - -/** - * Container class for FDv2 data source conditions and related types. - *

- * This class is non-constructable and serves only as a namespace for condition-related types. - * Package-private for internal use and testing. - */ -class FDv2DataSourceConditions { - /** - * Private constructor to prevent instantiation. - */ - private FDv2DataSourceConditions() { - } - - /** - * Package-private for testing. - */ - interface Condition extends Closeable { - enum ConditionType { - FALLBACK, - RECOVERY, - UNPARK, - FDV1_FALLBACK_END, - } - - CompletableFuture execute(); - - /** - * Informs the condition of a result from the synchronizer. - * - * @param sourceResult the result - * @return true if the condition should be replaced with a new one - */ - boolean inform(FDv2SourceResult sourceResult); - - void close(); - - ConditionType getType(); - } - - interface ConditionFactory { - /** - * Builds the condition. - * - * @return the condition, or null if the condition does not apply - */ - Condition build(); - - Condition.ConditionType getType(); - } - - static abstract class TimedCondition implements Condition { - protected final CompletableFuture resultFuture = new CompletableFuture<>(); - - protected final ScheduledExecutorService sharedExecutor; - - /** - * Future for the timeout task, if any. Will be null when no timeout is active. - */ - protected ScheduledFuture timerFuture; - - /** - * Timeout duration for the fallback operation. - */ - protected final Duration timeout; - - public TimedCondition(ScheduledExecutorService sharedExecutor, Duration timeout) { - this.sharedExecutor = sharedExecutor; - this.timeout = timeout; - } - - @Override - public CompletableFuture execute() { - return resultFuture; - } - - @Override - public void close() { - if (timerFuture != null) { - timerFuture.cancel(false); - timerFuture = null; - } - } - - static abstract class Factory implements ConditionFactory { - protected final ScheduledExecutorService sharedExecutor; - protected final Duration timeout; - - public Factory(ScheduledExecutorService sharedExecutor, Duration timeout) { - this.sharedExecutor = sharedExecutor; - this.timeout = timeout; - } - } - } - - /** - * This condition is used to determine if a fallback should be performed. It is informed of each data source result - * via {@link #inform(FDv2SourceResult)}. Based on the results, it updates its internal state. When the fallback - * condition is met, then the {@link java.util.concurrent.Future} returned by {@link #execute()} will complete. - *

- * This is package-private, instead of private, for ease of testing. - */ - static class FallbackCondition extends TimedCondition { - static class Factory extends TimedCondition.Factory { - public Factory(ScheduledExecutorService sharedExecutor, Duration timeout) { - super(sharedExecutor, timeout); - } - - @Override - public Condition build() { - return new FallbackCondition(sharedExecutor, timeout); - } - - @Override - public ConditionType getType() { - return ConditionType.FALLBACK; - } - } - - public FallbackCondition(ScheduledExecutorService sharedExecutor, Duration timeout) { - super(sharedExecutor, timeout); - } - - @Override - public boolean inform(FDv2SourceResult sourceResult) { - if (sourceResult.getResultType() == SourceResultType.CHANGE_SET) { - // The synchronizer is healthy. If the timer has started, the condition must start over. - return timerFuture != null; - } - if (sourceResult.getResultType() == SourceResultType.STATUS && sourceResult.getStatus().getState() == SourceSignal.INTERRUPTED) { - if (timerFuture == null) { - timerFuture = sharedExecutor.schedule(() -> { - resultFuture.complete(this); - return null; - }, timeout.toMillis(), TimeUnit.MILLISECONDS); - } - } - return false; - } - - @Override - public ConditionType getType() { - return ConditionType.FALLBACK; - } - } - - static class RecoveryCondition extends TimedCondition { - - static class Factory extends TimedCondition.Factory { - public Factory(ScheduledExecutorService sharedExecutor, Duration timeout) { - super(sharedExecutor, timeout); - } - - @Override - public Condition build() { - return new RecoveryCondition(sharedExecutor, timeout); - } - - @Override - public ConditionType getType() { - return ConditionType.RECOVERY; - } - } - - public RecoveryCondition(ScheduledExecutorService sharedExecutor, Duration timeout) { - super(sharedExecutor, timeout); - timerFuture = sharedExecutor.schedule(() -> { - resultFuture.complete(this); - return null; - }, timeout.toMillis(), TimeUnit.MILLISECONDS); - } - - @Override - public boolean inform(FDv2SourceResult sourceResult) { - // Recovery is time-based, so a result never requires a new condition. - return false; - } - - @Override - public ConditionType getType() { - return ConditionType.RECOVERY; - } - } - - /** - * This condition is met when its delay has passed. - */ - static class DeadlineCondition extends TimedCondition { - /** - * Builds a condition that is met at the deadline, or no condition if there is no deadline. - */ - static class Factory implements ConditionFactory { - private final ScheduledExecutorService sharedExecutor; - private final ConditionType type; - private final Supplier deadline; - - public Factory(ScheduledExecutorService sharedExecutor, ConditionType type, Supplier deadline) { - this.sharedExecutor = sharedExecutor; - this.type = type; - this.deadline = deadline; - } - - @Override - public Condition build() { - Instant time = deadline.get(); - if (time == null) { - return null; - } - return new DeadlineCondition(sharedExecutor, type, Duration.between(Instant.now(), time)); - } - - @Override - public ConditionType getType() { - return type; - } - } - - private final ConditionType type; - - public DeadlineCondition(ScheduledExecutorService sharedExecutor, ConditionType type, Duration delay) { - super(sharedExecutor, delay); - this.type = type; - timerFuture = sharedExecutor.schedule(() -> { - resultFuture.complete(this); - return null; - }, delay.toMillis(), TimeUnit.MILLISECONDS); - } - - @Override - public boolean inform(FDv2SourceResult sourceResult) { - // A deadline is time-based, so a result never requires a new condition. - return false; - } - - @Override - public ConditionType getType() { - return type; - } - } -} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java deleted file mode 100644 index c625f211..00000000 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java +++ /dev/null @@ -1,331 +0,0 @@ -package com.launchdarkly.sdk.server; - -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition.ConditionType; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.FallbackCondition; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.RecoveryCondition; -import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; -import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.Arrays; -import java.util.Collections; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.sameInstance; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -/** - * Direct tests for {@link FDv2DataSource.Conditions}. - * - *

The Conditions class is the aggregator that races fallback/recovery - * condition futures against synchronizer.next() in the FDv2DataSource run - * loop. Each iteration of that loop calls getFuture() and passes the result to - * CompletableFuture.anyOf(...) -- so getFuture() must not return a shared - * instance, or every anyOf call permanently attaches a Completion node to the - * shared instance's stack, leaking memory proportional to event rate during - * the synchronizer's tenure on a healthy primary. - */ -public class FDv2DataSourceConditionsAggregateTest { - private ScheduledExecutorService executor; - - @Before - public void setUp() { - executor = Executors.newScheduledThreadPool(1); - } - - @After - public void tearDown() { - executor.shutdownNow(); - } - - /** - * Bug-proving test: getFuture() must return a fresh instance per call. - * - *

If it returns the same instance (as it did before the fix), the run - * loop's per-iteration {@code anyOf(getFuture(), syncNext)} attaches a new - * OrRelay Completion to the shared future's stack every iteration, with no - * deregister path -- a monotonic leak for a non-firing aggregate. - */ - @Test - public void getFutureReturnsDistinctInstancesPerCall() { - Condition fallback = new FallbackCondition(executor, Duration.ofSeconds(60)); - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.singletonList(fallback))) { - CompletableFuture f1 = conditions.getFuture(); - CompletableFuture f2 = conditions.getFuture(); - CompletableFuture f3 = conditions.getFuture(); - assertThat(f1, not(sameInstance(f2))); - assertThat(f2, not(sameInstance(f3))); - assertThat(f1, not(sameInstance(f3))); - } - } - - /** - * Even with no underlying conditions (a single-synchronizer configuration), - * getFuture() must return fresh instances. The aggregate never completes - * in this case, which is exactly the scenario where any per-iteration - * accumulation would be most damaging. - */ - @Test - public void getFutureReturnsDistinctInstancesEvenWithNoConditions() { - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.emptyList())) { - CompletableFuture f1 = conditions.getFuture(); - CompletableFuture f2 = conditions.getFuture(); - assertThat(f1, not(sameInstance(f2))); - } - } - - /** - * Every fresh future returned by getFuture() must complete when the - * underlying aggregate fires. The fan-out via the single permanent listener - * is what makes the fresh-per-call pattern work; verify it actually - * delivers. - */ - @Test - public void allFreshFuturesCompleteWhenAggregateFires() throws Exception { - // 0-second timeout -> fires on first INTERRUPTED inform. - Condition fallback = new FallbackCondition(executor, Duration.ofSeconds(0)); - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.singletonList(fallback))) { - CompletableFuture f1 = conditions.getFuture(); - CompletableFuture f2 = conditions.getFuture(); - CompletableFuture f3 = conditions.getFuture(); - - conditions.inform(makeInterruptedResult()); - - Object r1 = f1.get(2, TimeUnit.SECONDS); - Object r2 = f2.get(2, TimeUnit.SECONDS); - Object r3 = f3.get(2, TimeUnit.SECONDS); - - assertNotNull(r1); - assertNotNull(r2); - assertNotNull(r3); - assertTrue(r1 instanceof Condition); - assertTrue(r2 instanceof Condition); - assertTrue(r3 instanceof Condition); - } - } - - /** - * Bug-proving test for the null-sentinel issue caught by Cursor Bugbot: - * if the underlying aggregate completes exceptionally, every - * future returned by getFuture() -- both those handed out before the - * exception and those requested after -- must complete exceptionally too. - * - *

Prior to this fix, the "fired" state was tracked via a {@code null} - * sentinel on a {@code completedValue} field, which also stayed - * {@code null} on exceptional completion. A subsequent getFuture() call - * would then return {@code CompletableFuture.completedFuture(null)} -- - * silently converting an exceptional completion into a normal - * {@code null} completion. The run loop's downstream - * {@code res.getClass().getName()} would then throw NPE. - */ - @Test - public void getFutureFailsExceptionallyWhenAggregateFailsExceptionally() - throws Exception { - ManualCondition manualCondition = new ManualCondition(ConditionType.FALLBACK); - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.singletonList(manualCondition))) { - // Future requested BEFORE the exceptional completion. - CompletableFuture before = conditions.getFuture(); - - RuntimeException boom = new RuntimeException("simulated condition failure"); - manualCondition.future.completeExceptionally(boom); - - // Future requested AFTER the exceptional completion (exercises the - // fast path through makeCompletedFuture). This is the case bugbot - // caught: pre-fix, it returned completedFuture(null). - CompletableFuture after = conditions.getFuture(); - - assertThrowsExecutionExceptionWithCause(before, boom); - assertThrowsExecutionExceptionWithCause(after, boom); - } - } - - /** - * After getFuture() reports a met condition, the next getFuture() waits for another condition. - */ - @Test - public void getFutureAfterConditionIsReportedWaitsForAnotherCondition() throws Exception { - ManualCondition fallback = new ManualCondition(ConditionType.FALLBACK); - ManualCondition recovery = new ManualCondition(ConditionType.RECOVERY); - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Arrays.asList(fallback, recovery))) { - // Meet the fallback condition. - CompletableFuture first = conditions.getFuture(); - fallback.future.complete(fallback); - - // The first future reports the fallback condition, and the next future waits. - assertSame(fallback, first.get(2, TimeUnit.SECONDS)); - CompletableFuture second = conditions.getFuture(); - assertFalse(second.isDone()); - - // Meet the recovery condition. - recovery.future.complete(recovery); - - // The second future reports the recovery condition. - assertSame(recovery, second.get(2, TimeUnit.SECONDS)); - } - } - - /** - * isMet() is true only for a condition that the aggregate has reported. - */ - @Test - public void isMetIsTrueOnlyForAReportedCondition() throws Exception { - ManualCondition fallback = new ManualCondition(ConditionType.FALLBACK); - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.singletonList(fallback))) { - // Before the condition is met, it is not reported. - assertFalse(conditions.isMet(ConditionType.FALLBACK)); - - // Meet the condition, and wait for the aggregate to report it. - CompletableFuture future = conditions.getFuture(); - fallback.future.complete(fallback); - future.get(2, TimeUnit.SECONDS); - - // The condition is reported, and a type with no condition is not. - assertTrue(conditions.isMet(ConditionType.FALLBACK)); - assertFalse(conditions.isMet(ConditionType.RECOVERY)); - } - } - - /** - * replace() closes the old condition, and the next future waits for the replacement. - */ - @Test - public void replaceWaitsForTheReplacementCondition() throws Exception { - ManualCondition oldFallback = new ManualCondition(ConditionType.FALLBACK); - ManualCondition newFallback = new ManualCondition(ConditionType.FALLBACK); - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.singletonList(oldFallback))) { - // Meet the old condition, and wait for the aggregate to report it. - CompletableFuture first = conditions.getFuture(); - oldFallback.future.complete(oldFallback); - first.get(2, TimeUnit.SECONDS); - - // Replace the old condition. - conditions.replace(ConditionType.FALLBACK, newFallback); - - // The old condition is closed, the type is no longer met, and the next future waits. - assertTrue(oldFallback.closed); - assertFalse(conditions.isMet(ConditionType.FALLBACK)); - CompletableFuture second = conditions.getFuture(); - assertFalse(second.isDone()); - - // Meet the replacement. - newFallback.future.complete(newFallback); - - // The next future reports the replacement. - assertSame(newFallback, second.get(2, TimeUnit.SECONDS)); - } - } - - /** - * replace() with null closes the old condition and leaves no condition of that type. - */ - @Test - public void replaceWithNullLeavesNoCondition() throws Exception { - ManualCondition fallback = new ManualCondition(ConditionType.FALLBACK); - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.singletonList(fallback))) { - // Meet the condition, and wait for the aggregate to report it. - CompletableFuture first = conditions.getFuture(); - fallback.future.complete(fallback); - first.get(2, TimeUnit.SECONDS); - - // Replace the condition with nothing. - conditions.replace(ConditionType.FALLBACK, null); - - // The condition is closed, and the next future waits. - assertTrue(fallback.closed); - assertFalse(conditions.getFuture().isDone()); - } - } - - private static FDv2SourceResult makeInterruptedResult() { - return FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo( - DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, - 0, - "simulated", - Instant.now()), - false); - } - - /** - * Asserts that {@code future.get()} throws {@link ExecutionException} - * wrapping the expected cause. {@code CompletableFuture#get} surfaces - * exceptional completion as ExecutionException with the original - * exception as its cause. - */ - private static void assertThrowsExecutionExceptionWithCause( - CompletableFuture future, - Throwable expectedCause) throws Exception { - try { - future.get(2, TimeUnit.SECONDS); - throw new AssertionError("expected ExecutionException, got normal completion"); - } catch (ExecutionException ee) { - if (ee.getCause() != expectedCause) { - throw new AssertionError( - "expected cause to be " + expectedCause + " but was " + ee.getCause(), ee); - } - } - } - - /** - * Test-only Condition with an externally-controllable future. The - * existing FallbackCondition/RecoveryCondition only resolve normally - * (with {@code this}); to exercise the exceptional path through the - * aggregate's whenComplete listener we need a Condition we can fail - * directly. - */ - private static final class ManualCondition - implements Condition { - final CompletableFuture future = new CompletableFuture<>(); - private final ConditionType type; - volatile boolean closed = false; - - ManualCondition(ConditionType type) { - this.type = type; - } - - @Override - public CompletableFuture execute() { - return future; - } - - @Override - public boolean inform(FDv2SourceResult sourceResult) { - // Manually controlled; no auto-trigger from inform. - return false; - } - - @Override - public void close() { - // No timer to cancel. - closed = true; - } - - @Override - public ConditionType getType() { - return type; - } - } -} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceDeadlineConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceDeadlineConditionTest.java deleted file mode 100644 index a8cfaa34..00000000 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceDeadlineConditionTest.java +++ /dev/null @@ -1,116 +0,0 @@ -package com.launchdarkly.sdk.server; - -import com.launchdarkly.sdk.fdv2.ChangeSet; -import com.launchdarkly.sdk.fdv2.ChangeSetType; -import com.launchdarkly.sdk.fdv2.Selector; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition.ConditionType; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.DeadlineCondition; -import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; -import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; - -import org.junit.After; -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; - -@SuppressWarnings("javadoc") -public class FDv2DataSourceDeadlineConditionTest extends BaseTest { - - private ScheduledExecutorService executor; - - @After - public void tearDown() { - if (executor != null && !executor.isShutdown()) { - executor.shutdownNow(); - } - } - - @Test - public void getTypeReturnsUnpark() { - executor = Executors.newScheduledThreadPool(1); - DeadlineCondition condition = new DeadlineCondition(executor, ConditionType.UNPARK, Duration.ofSeconds(120)); - - assertEquals(ConditionType.UNPARK, condition.getType()); - } - - @Test - public void conditionIsMetAfterItsDelay() throws Exception { - executor = Executors.newScheduledThreadPool(1); - - // Build a condition with a short delay. - DeadlineCondition condition = new DeadlineCondition(executor, ConditionType.UNPARK, Duration.ofMillis(50)); - - // The condition is met with itself after the delay. - assertSame(condition, condition.execute().get(2, TimeUnit.SECONDS)); - } - - @Test - public void pastUnparkTimeIsMetImmediately() throws Exception { - executor = Executors.newScheduledThreadPool(1); - - // Build a condition for an unpark time that has already passed. - DeadlineCondition condition = new DeadlineCondition(executor, ConditionType.UNPARK, Duration.ofMillis(-100)); - - // The condition is met without waiting. - assertSame(condition, condition.execute().get(1, TimeUnit.SECONDS)); - } - - @Test - public void informNeverRequestsReplacement() { - executor = Executors.newScheduledThreadPool(1); - DeadlineCondition condition = new DeadlineCondition(executor, ConditionType.UNPARK, Duration.ofSeconds(120)); - - // Inform the condition of a change set and of an interruption. - boolean replaceAfterChangeSet = condition.inform(FDv2SourceResult.changeSet( - new ChangeSet<>(ChangeSetType.None, Selector.EMPTY, null, null, true), - false)); - boolean replaceAfterInterruption = condition.inform(FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false)); - - // Neither result requires a new condition. - assertFalse(replaceAfterChangeSet); - assertFalse(replaceAfterInterruption); - } - - @Test - public void factoryBuildsConditionMetAtEarliestUnparkTime() throws Exception { - executor = Executors.newScheduledThreadPool(1); - Instant unparkTime = Instant.now().plusMillis(50); - DeadlineCondition.Factory factory = - new DeadlineCondition.Factory(executor, ConditionType.UNPARK, () -> unparkTime); - - // Build a condition while a synchronizer is parked. - Condition condition = factory.build(); - - // The condition is met at the unpark time. - assertEquals(ConditionType.UNPARK, factory.getType()); - assertNotNull(condition); - CompletableFuture future = condition.execute(); - assertSame(condition, future.get(2, TimeUnit.SECONDS)); - } - - @Test - public void factoryReturnsNullWhenNoSynchronizerIsParked() { - executor = Executors.newScheduledThreadPool(1); - DeadlineCondition.Factory factory = new DeadlineCondition.Factory(executor, ConditionType.UNPARK, () -> null); - - // Build a condition while no synchronizer is parked. - Condition condition = factory.build(); - - // There is no condition. - assertNull(condition); - } -} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java deleted file mode 100644 index a48d960a..00000000 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java +++ /dev/null @@ -1,351 +0,0 @@ -package com.launchdarkly.sdk.server; - -import com.launchdarkly.sdk.fdv2.Selector; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.FallbackCondition; -import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; -import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; -import com.launchdarkly.sdk.fdv2.ChangeSet; -import com.launchdarkly.sdk.fdv2.ChangeSetType; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; - -import org.junit.After; -import java.util.Map; - -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -public class FDv2DataSourceFallbackConditionTest extends BaseTest { - - private ScheduledExecutorService executor; - - @After - public void tearDown() { - if (executor != null && !executor.isShutdown()) { - executor.shutdownNow(); - } - } - - private ChangeSet>>> makeChangeSet() { - return new ChangeSet<>( - ChangeSetType.None, - Selector.EMPTY, - null, - null, - true - ); - } - - @Test - public void executeReturnsCompletableFuture() { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(120)); - - CompletableFuture result = condition.execute(); - - assertFalse(result.isDone()); - } - - @Test - public void getTypeReturnsFallback() { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(120)); - - assertEquals(Condition.ConditionType.FALLBACK, condition.getType()); - } - - @Test - public void interruptedStatusStartsTimerThatCompletesResultFuture() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - assertFalse(resultFuture.isDone()); - - // Inform with INTERRUPTED status - boolean replace = condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // The condition does not ask to be replaced, and the future should still not be done immediately - assertFalse(replace); - assertFalse(resultFuture.isDone()); - - // Wait for timeout to fire - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - - // Now it should be done and return the condition instance - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void changeSetWithActiveTimerRequestsReplacement() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - - // Start timer with INTERRUPTED - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Inform with CHANGE_SET while the timer is running - ChangeSet>>> changeSet = makeChangeSet(); - boolean replace = condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // The condition asks to be replaced - assertTrue(replace); - } - - @Test - public void changeSetWithoutActiveTimerDoesNotRequestReplacement() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with CHANGE_SET without starting a timer first - ChangeSet>>> changeSet = makeChangeSet(); - boolean replace = condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // The condition does not ask to be replaced, and is not met - assertFalse(replace); - assertFalse(resultFuture.isDone()); - } - - @Test - public void multipleInterruptedStatusesDoNotStartMultipleTimers() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(2)); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with INTERRUPTED multiple times - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - Thread.sleep(100); - - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - Thread.sleep(100); - - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Wait for the timer (should only fire once) - Condition result = resultFuture.get(3, TimeUnit.SECONDS); - - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void terminalErrorStatusDoesNotStartTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with TERMINAL_ERROR status - condition.inform( - FDv2SourceResult.terminalError( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, 401, null, Instant.now()), - false - ) - ); - - // Wait longer than timeout - Thread.sleep(1500); - - // Future should still not be complete (no timer started) - assertFalse(resultFuture.isDone()); - } - - @Test - public void shutdownStatusDoesNotStartTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with SHUTDOWN status - condition.inform(FDv2SourceResult.shutdown()); - - // Wait longer than timeout - Thread.sleep(1500); - - // Future should still not be complete (no timer started) - assertFalse(resultFuture.isDone()); - } - - @Test - public void goodbyeStatusDoesNotStartTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with GOODBYE status - condition.inform(FDv2SourceResult.goodbye("server-requested", false)); - - // Wait longer than timeout - Thread.sleep(1500); - - // Future should still not be complete (no timer started) - assertFalse(resultFuture.isDone()); - } - - @Test - public void closeCancelsActiveTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Start timer with INTERRUPTED - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Close the condition - condition.close(); - - // Wait longer than the timeout period - Thread.sleep(1500); - - // Future should still not be complete (timer was cancelled) - assertFalse(resultFuture.isDone()); - } - - @Test - public void closeWithoutActiveTimerDoesNotFail() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(120)); - - // Close without starting a timer - condition.close(); - - // Should not throw exception - } - - @Test - public void changeSetAfterTimerFiresRequestsReplacement() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Start timer - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Wait for timer to fire - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - - // Inform with CHANGE_SET after timer has fired - ChangeSet>>> changeSet = makeChangeSet(); - boolean replace = condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // The condition asks to be replaced, and its future remains complete - assertTrue(replace); - assertTrue(resultFuture.isDone()); - } - - @Test - public void factoryCreatesFallbackCondition() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition.Factory factory = new FallbackCondition.Factory(executor, Duration.ofSeconds(1)); - - FallbackCondition condition = (FallbackCondition) factory.build(); - - // Verify it works by using it - CompletableFuture resultFuture = condition.execute(); - assertFalse(resultFuture.isDone()); - - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void executeReturnsTheSameFutureOnMultipleCalls() { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(120)); - - CompletableFuture first = condition.execute(); - CompletableFuture second = condition.execute(); - - assertSame(first, second); - } - - @Test - public void closeCanBeCalledMultipleTimes() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, Duration.ofSeconds(1)); - - // Start timer - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Close multiple times - condition.close(); - condition.close(); - condition.close(); - - // Should not throw exception - } -} \ No newline at end of file diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java deleted file mode 100644 index 0d6765f3..00000000 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java +++ /dev/null @@ -1,330 +0,0 @@ -package com.launchdarkly.sdk.server; - -import com.launchdarkly.sdk.fdv2.Selector; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.RecoveryCondition; -import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; -import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; -import com.launchdarkly.sdk.fdv2.ChangeSet; -import com.launchdarkly.sdk.fdv2.ChangeSetType; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; - -import org.junit.After; -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -@SuppressWarnings("javadoc") -public class FDv2DataSourceRecoveryConditionTest extends BaseTest { - - private ScheduledExecutorService executor; - - @After - public void tearDown() { - if (executor != null && !executor.isShutdown()) { - executor.shutdownNow(); - } - } - - private ChangeSet>>> makeChangeSet() { - return new ChangeSet<>( - ChangeSetType.None, - Selector.EMPTY, - null, - null, - true - ); - } - - @Test - public void getTypeReturnsRecovery() { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(120)); - - assertEquals(Condition.ConditionType.RECOVERY, condition.getType()); - } - - @Test - public void timerStartsImmediatelyAndCompletesResultFuture() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Future should not be done immediately - assertFalse(resultFuture.isDone()); - - // Wait for timeout to fire - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - - // Now it should be done and return the condition instance - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void closeCancelsActiveTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Close the condition before timeout - condition.close(); - - // Wait longer than the timeout period - Thread.sleep(1500); - - // Future should still not be complete (timer was cancelled) - assertFalse(resultFuture.isDone()); - } - - @Test - public void closeAfterTimerFiresDoesNotCauseError() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Wait for timer to fire - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - - // Close after timer has fired - condition.close(); - - // Should not throw exception - } - - @Test - public void closeCanBeCalledMultipleTimes() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - // Close multiple times before timer fires - condition.close(); - condition.close(); - condition.close(); - - // Should not throw exception - } - - @Test - public void informWithChangeSetDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with CHANGE_SET - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // Timer should still fire after timeout (inform does nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void informWithInterruptedStatusDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with INTERRUPTED status - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Timer should still fire after timeout (inform does nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void informWithTerminalErrorStatusDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with TERMINAL_ERROR status - condition.inform( - FDv2SourceResult.terminalError( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, 401, null, Instant.now()), - false - ) - ); - - // Timer should still fire after timeout (inform does nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void informWithShutdownStatusDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with SHUTDOWN status - condition.inform(FDv2SourceResult.shutdown()); - - // Timer should still fire after timeout (inform does nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void informWithGoodbyeStatusDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with GOODBYE status - condition.inform(FDv2SourceResult.goodbye("server-requested", false)); - - // Timer should still fire after timeout (inform does nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void informWithNullDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with null - condition.inform(null); - - // Timer should still fire after timeout (inform does nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void multipleInformCallsDoNotAffectTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - CompletableFuture resultFuture = condition.execute(); - - // Multiple inform calls - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - condition.inform(FDv2SourceResult.shutdown()); - - // Timer should still fire after timeout (all inform calls do nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void factoryCreatesRecoveryCondition() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition.Factory factory = new RecoveryCondition.Factory(executor, Duration.ofSeconds(1)); - - RecoveryCondition condition = (RecoveryCondition) factory.build(); - - // Verify it works by using it - CompletableFuture resultFuture = condition.execute(); - assertFalse(resultFuture.isDone()); - - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void factoryGetTypeReturnsRecovery() { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition.Factory factory = new RecoveryCondition.Factory(executor, Duration.ofSeconds(1)); - - assertEquals(Condition.ConditionType.RECOVERY, factory.getType()); - } - - @Test - public void executeReturnsTheSameFutureOnMultipleCalls() { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(120)); - - CompletableFuture first = condition.execute(); - CompletableFuture second = condition.execute(); - - assertSame(first, second); - } - - @Test - public void timerStartsImmediatelyOnConstruction() throws Exception { - executor = Executors.newScheduledThreadPool(1); - - // Create condition with very short timeout - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - // Get the future - CompletableFuture resultFuture = condition.execute(); - - // Verify it's not done yet - assertFalse(resultFuture.isDone()); - - // Wait for it to complete - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - - // Verify it completed with the condition - assertSame(condition, result); - } - - @Test - public void closeBeforeExecuteDoesNotPreventFutureAccess() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, Duration.ofSeconds(1)); - - // Close immediately - condition.close(); - - // Should still be able to get the future - CompletableFuture resultFuture = condition.execute(); - - // Wait to ensure timer doesn't fire - Thread.sleep(1500); - - // Future should not be complete (timer was cancelled) - assertFalse(resultFuture.isDone()); - } -} From 76e2ca267dd242883672bc080990ee836709d051 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Thu, 5 Nov 2026 16:03:28 -0800 Subject: [PATCH 08/13] refactor: Remove the unused executor from FDv2DataSource --- .../sdk/server/FDv2DataSource.java | 5 +- .../sdk/server/FDv2DataSystem.java | 3 +- .../sdk/server/FDv2DataSourceTest.java | 283 +++++------------- 3 files changed, 83 insertions(+), 208 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 17ab88a3..bb74c12a 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -93,15 +93,13 @@ public FDv2DataSource( DataSourceFactory fdv1DataSourceFactory, DataSourceUpdateSinkV2 dataSourceUpdates, int threadPriority, - LDLogger logger, - ScheduledExecutorService sharedExecutor) { + LDLogger logger) { this(initializers, synchronizers, fdv1DataSourceFactory, dataSourceUpdates, threadPriority, logger, - sharedExecutor, DEFAULT_FALLBACK_TIMEOUT, DEFAULT_RECOVERY_TIMEOUT, DEFAULT_PARK_INITIAL_DELAY, @@ -120,7 +118,6 @@ public FDv2DataSource( DataSourceUpdateSinkV2 dataSourceUpdates, int threadPriority, LDLogger logger, - ScheduledExecutorService sharedExecutor, Duration fallbackTimeout, Duration recoveryTimeout, Duration parkInitialDelay, diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java index 5053625b..2e44afba 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java @@ -176,8 +176,7 @@ static FDv2DataSystem create( fdv1FallbackFactory, dataSourceUpdates, config.threadPriority, - clientContext.getBaseLogger().subLogger(Loggers.DATA_SOURCE_LOGGER_NAME), - clientContext.sharedExecutor + clientContext.getBaseLogger().subLogger(Loggers.DATA_SOURCE_LOGGER_NAME) ); } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index a68137ed..2b625f85 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -40,15 +40,13 @@ @SuppressWarnings("javadoc") public class FDv2DataSourceTest extends BaseTest { - private ScheduledExecutorService executor; + private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(2); private final LDLogger logger = LDLogger.withAdapter(Logs.none(), ""); private final List resourcesToClose = new ArrayList<>(); @After public void tearDown() { - if (executor != null && !executor.isShutdown()) { - executor.shutdownNow(); - } + executor.shutdownNow(); for (AutoCloseable resource : resourcesToClose) { try { resource.close(); @@ -100,7 +98,6 @@ private FDv2SourceResult makeTerminalErrorResult() { @Test public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture firstInitializerFuture = new CompletableFuture<>(); @@ -117,7 +114,7 @@ public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -144,7 +141,6 @@ public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws @Test public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture firstInitializerFuture = new CompletableFuture<>(); @@ -171,7 +167,7 @@ public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() thro } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -205,7 +201,6 @@ public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() thro @Test public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture firstInitializerFuture = CompletableFuture.completedFuture( @@ -226,7 +221,7 @@ public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() th ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -251,7 +246,6 @@ public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() th @Test public void allInitializersFailSwitchesToSynchronizers() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture firstInitializerFuture = new CompletableFuture<>(); @@ -277,7 +271,7 @@ public void allInitializersFailSwitchesToSynchronizers() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -302,7 +296,6 @@ public void allInitializersFailSwitchesToSynchronizers() throws Exception { @Test public void allThreeInitializersFailWithNoSynchronizers() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of( @@ -325,7 +318,7 @@ public void allThreeInitializersFailWithNoSynchronizers() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -358,7 +351,6 @@ public void allThreeInitializersFailWithNoSynchronizers() throws Exception { @Test public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -371,7 +363,7 @@ public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -399,7 +391,6 @@ public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { @Test public void noInitializersOneSynchronizerIsWellBehaved() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -412,7 +403,7 @@ public void noInitializersOneSynchronizerIsWellBehaved() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -433,7 +424,6 @@ public void noInitializersOneSynchronizerIsWellBehaved() throws Exception { @Test public void oneInitializerOneSynchronizerIsWellBehaved() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -452,7 +442,7 @@ public void oneInitializerOneSynchronizerIsWellBehaved() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -473,13 +463,12 @@ public void oneInitializerOneSynchronizerIsWellBehaved() throws Exception { @Test public void noInitializersAndNoSynchronizersIsWellBehaved() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -507,7 +496,6 @@ public void noInitializersAndNoSynchronizersIsWellBehaved() throws Exception { @Test public void errorWithFDv1FallbackTriggersFallback() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -524,7 +512,7 @@ public void errorWithFDv1FallbackTriggersFallback() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -543,7 +531,6 @@ public void errorWithFDv1FallbackTriggersFallback() throws Exception { @Test public void fallbackAndRecoveryTasksWellBehaved() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -574,7 +561,7 @@ public void fallbackAndRecoveryTasksWellBehaved() throws Exception { ); // Use short timeouts for testing - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(1), /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -619,7 +606,6 @@ public void fallbackAndRecoveryTasksWellBehaved() throws Exception { @Test public void canDisposeWhenSynchronizersFallingBack() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -633,7 +619,7 @@ public void canDisposeWhenSynchronizersFallingBack() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(1), /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -659,7 +645,6 @@ public void canDisposeWhenSynchronizersFallingBack() throws Exception { @Test public void terminalErrorFailsOverToNextSynchronizer() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -686,7 +671,7 @@ public void terminalErrorFailsOverToNextSynchronizer() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -726,7 +711,6 @@ public void terminalErrorFailsOverToNextSynchronizer() throws Exception { @Test public void terminalErrorParksSynchronizerAndRestartsItAfterUnparkTime() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -744,7 +728,7 @@ public void terminalErrorParksSynchronizerAndRestartsItAfterUnparkTime() throws : new MockQueuedSynchronizer(secondInstanceResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), @@ -764,7 +748,6 @@ public void terminalErrorParksSynchronizerAndRestartsItAfterUnparkTime() throws @Test public void failedNextParksSynchronizer() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger buildCount = new AtomicInteger(); @@ -779,7 +762,7 @@ public void failedNextParksSynchronizer() throws Exception { "throwing-sync") ); - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, testLogger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, testLogger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), @@ -798,7 +781,6 @@ public void failedNextParksSynchronizer() throws Exception { @Test public void exceptionThrownFromNextParksSynchronizer() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger buildCount = new AtomicInteger(); @@ -823,7 +805,7 @@ public void close() { } ); - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), @@ -842,7 +824,6 @@ public void close() { @Test public void constructionFailureParksSynchronizer() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger buildCount = new AtomicInteger(); @@ -858,7 +839,7 @@ public void constructionFailureParksSynchronizer() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), @@ -877,7 +858,6 @@ public void constructionFailureParksSynchronizer() throws Exception { @Test public void resetThresholdMetReturnsParkDelayToBase() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger buildCount = new AtomicInteger(); BlockingQueue buildTimes = new LinkedBlockingQueue<>(); @@ -896,7 +876,7 @@ public void resetThresholdMetReturnsParkDelayToBase() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), @@ -920,7 +900,6 @@ public void resetThresholdMetReturnsParkDelayToBase() throws Exception { @Test public void fallbackWithNoOtherAvailableSynchronizerKeepsCurrentSynchronizer() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger secondaryBuildCount = new AtomicInteger(); IterableAsyncQueue secondaryResults = new IterableAsyncQueue<>(); @@ -939,7 +918,7 @@ public void fallbackWithNoOtherAvailableSynchronizerKeepsCurrentSynchronizer() t : new MockQueuedSynchronizer(new IterableAsyncQueue<>()) ); - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofMillis(10), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -964,7 +943,6 @@ public void fallbackWithNoOtherAvailableSynchronizerKeepsCurrentSynchronizer() t @Test public void unparkWhileCurrentSynchronizerIsHealthyKeepsIt() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger primaryBuildCount = new AtomicInteger(); IterableAsyncQueue secondaryResults = new IterableAsyncQueue<>(); @@ -983,7 +961,7 @@ public void unparkWhileCurrentSynchronizerIsHealthyKeepsIt() throws Exception { ); // The primary synchronizer is parked for 25-50ms. The fallback and recovery timeouts are too long to matter. - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(50), @@ -1007,7 +985,6 @@ public void unparkWhileCurrentSynchronizerIsHealthyKeepsIt() throws Exception { @Test public void fallbackSwitchesWhenParkedSynchronizerUnparks() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger primaryBuildCount = new AtomicInteger(); AtomicInteger secondaryBuildCount = new AtomicInteger(); @@ -1031,7 +1008,7 @@ public void fallbackSwitchesWhenParkedSynchronizerUnparks() throws Exception { ); // The fallback condition is met after 10ms, and the primary synchronizer unparks after 50-100ms. - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofMillis(10), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), @@ -1051,7 +1028,6 @@ public void fallbackSwitchesWhenParkedSynchronizerUnparks() throws Exception { @Test public void changeSetAfterFallbackIsMetPreventsSwitchAtUnpark() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger primaryBuildCount = new AtomicInteger(); IterableAsyncQueue secondaryResults = new IterableAsyncQueue<>(); @@ -1070,7 +1046,7 @@ public void changeSetAfterFallbackIsMetPreventsSwitchAtUnpark() throws Exception ); // The fallback condition is met after 10ms, and the primary synchronizer unparks after 80-160ms. - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofMillis(10), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(160), @@ -1098,7 +1074,6 @@ public void changeSetAfterFallbackIsMetPreventsSwitchAtUnpark() throws Exception @Test public void recoveryWithPreferredSynchronizerParkedWaitsForItsUnpark() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger primaryBuildCount = new AtomicInteger(); AtomicInteger secondaryBuildCount = new AtomicInteger(); @@ -1122,7 +1097,7 @@ public void recoveryWithPreferredSynchronizerParkedWaitsForItsUnpark() throws Ex ); // The recovery condition is met after 50ms, and the primary synchronizer unparks after 100-200ms. - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofMillis(50), /* parkInitialDelay= */ Duration.ofMillis(200), @@ -1142,7 +1117,6 @@ public void recoveryWithPreferredSynchronizerParkedWaitsForItsUnpark() throws Ex @Test public void changeSetDuringInterruptionRestartsFallbackTimeout() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); IterableAsyncQueue primaryResults = new IterableAsyncQueue<>(); primaryResults.put(makeInterruptedResult()); @@ -1159,7 +1133,7 @@ public void changeSetDuringInterruptionRestartsFallbackTimeout() throws Exceptio ); // The fallback timeout is 100ms. - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofMillis(100), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1187,7 +1161,6 @@ public void changeSetDuringInterruptionRestartsFallbackTimeout() throws Exceptio // A synchronizer that shuts down without being closed is parked, like one with any other unexpected failure. @Test public void unpromptedShutdownParksSynchronizer() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger buildCount = new AtomicInteger(); @@ -1203,7 +1176,7 @@ public void unpromptedShutdownParksSynchronizer() throws Exception { ); // The synchronizer is parked for 50-100ms. - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), @@ -1223,7 +1196,6 @@ public void unpromptedShutdownParksSynchronizer() throws Exception { // While every synchronizer is parked, the synchronizer that was parked last is stopped. @Test public void parkedSynchronizerIsClosedWhileAllSynchronizersAreParked() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CountDownLatch closed = new CountDownLatch(1); @@ -1243,7 +1215,7 @@ public void close() { ); // The synchronizer is parked for much longer than the test. - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1261,7 +1233,6 @@ public void close() { @Test public void allSynchronizersParkedRestartsAfterEarliestUnparkTime() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1289,7 +1260,7 @@ public void allSynchronizersParkedRestartsAfterEarliestUnparkTime() throws Excep } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), @@ -1325,7 +1296,6 @@ public void allSynchronizersParkedRestartsAfterEarliestUnparkTime() throws Excep @Test public void disabledDataSourceCannotTriggerActions() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1349,7 +1319,7 @@ public void disabledDataSourceCannotTriggerActions() throws Exception { () -> new MockQueuedSynchronizer(secondSyncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1388,7 +1358,6 @@ public void disabledDataSourceCannotTriggerActions() throws Exception { @Test public void disposeCompletesStartFuture() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1401,7 +1370,7 @@ public void disposeCompletesStartFuture() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1428,13 +1397,12 @@ public void disposeCompletesStartFuture() throws Exception { @Test public void noSourcesProvidedCompletesImmediately() throws Exception{ - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1461,7 +1429,6 @@ public void noSourcesProvidedCompletesImmediately() throws Exception{ @Test public void startFutureCompletesExactlyOnce() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -1480,7 +1447,7 @@ public void startFutureCompletesExactlyOnce() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1498,7 +1465,6 @@ public void startFutureCompletesExactlyOnce() throws Exception { @Test public void concurrentCloseAndStartHandledSafely() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); BlockingQueue syncResults = new LinkedBlockingQueue<>(); @@ -1509,7 +1475,7 @@ public void concurrentCloseAndStartHandledSafely() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1528,7 +1494,6 @@ public void concurrentCloseAndStartHandledSafely() throws Exception { @Test public void multipleStartCallsEventuallyComplete() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); BlockingQueue syncResults = new LinkedBlockingQueue<>(); @@ -1539,7 +1504,7 @@ public void multipleStartCallsEventuallyComplete() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1562,7 +1527,6 @@ public void multipleStartCallsEventuallyComplete() throws Exception { @Test public void isInitializedThreadSafe() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); BlockingQueue syncResults = new LinkedBlockingQueue<>(); @@ -1573,7 +1537,7 @@ public void isInitializedThreadSafe() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1601,7 +1565,6 @@ public void isInitializedThreadSafe() throws Exception { @Test public void dataSourceUpdatesApplyThreadSafe() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); BlockingQueue syncResults = new LinkedBlockingQueue<>(); @@ -1614,7 +1577,7 @@ public void dataSourceUpdatesApplyThreadSafe() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1639,7 +1602,6 @@ public void dataSourceUpdatesApplyThreadSafe() throws Exception { @Test public void initializerThrowsExecutionException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture badFuture = new CompletableFuture<>(); @@ -1656,7 +1618,7 @@ public void initializerThrowsExecutionException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1674,7 +1636,6 @@ public void initializerThrowsExecutionException() throws Exception { @Test public void initializerThrowsInterruptedException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicBoolean firstCalled = new AtomicBoolean(false); @@ -1694,7 +1655,7 @@ public void initializerThrowsInterruptedException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1713,7 +1674,6 @@ public void initializerThrowsInterruptedException() throws Exception { @Test public void initializerThrowsCancellationException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture cancelledFuture = new CompletableFuture<>(); @@ -1730,7 +1690,7 @@ public void initializerThrowsCancellationException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1748,7 +1708,6 @@ public void initializerThrowsCancellationException() throws Exception { @Test public void synchronizerNextThrowsExecutionException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1765,7 +1724,7 @@ public void synchronizerNextThrowsExecutionException() throws Exception { () -> new MockSynchronizer(goodFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1783,7 +1742,6 @@ public void synchronizerNextThrowsExecutionException() throws Exception { @Test public void synchronizerNextThrowsInterruptedException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1803,7 +1761,7 @@ public void synchronizerNextThrowsInterruptedException() throws Exception { () -> new MockSynchronizer(goodFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1822,7 +1780,6 @@ public void synchronizerNextThrowsInterruptedException() throws Exception { @Test public void synchronizerNextThrowsCancellationException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1839,7 +1796,7 @@ public void synchronizerNextThrowsCancellationException() throws Exception { () -> new MockSynchronizer(goodFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1861,13 +1818,12 @@ public void synchronizerNextThrowsCancellationException() throws Exception { @Test public void closeWithoutStartDoesNotHang() { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1882,7 +1838,6 @@ public void closeWithoutStartDoesNotHang() { @Test public void closeAfterInitializersCompletesImmediately() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -1895,7 +1850,7 @@ public void closeAfterInitializersCompletesImmediately() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1912,7 +1867,6 @@ public void closeAfterInitializersCompletesImmediately() throws Exception { @Test public void closeWhileSynchronizerRunningShutdownsSource() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1932,7 +1886,7 @@ public void close() { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1950,7 +1904,6 @@ public void close() { @Test public void multipleCloseCallsAreIdempotent() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -1963,7 +1916,7 @@ public void multipleCloseCallsAreIdempotent() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -1983,7 +1936,6 @@ public void multipleCloseCallsAreIdempotent() throws Exception { @Test public void closingDataSourceDuringInitializationReportsOffWithoutErrors() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = new CompletableFuture<>(); @@ -1995,7 +1947,7 @@ public void closingDataSourceDuringInitializationReportsOffWithoutErrors() throw ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2027,7 +1979,6 @@ public void closingDataSourceDuringInitializationReportsOffWithoutErrors() throw @Test public void dataSourceClosedDuringSynchronizationReportsOffWithoutError() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2041,7 +1992,7 @@ public void dataSourceClosedDuringSynchronizationReportsOffWithoutError() throws } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2070,7 +2021,6 @@ public void dataSourceClosedDuringSynchronizationReportsOffWithoutError() throws @Test public void closeInterruptsConditionWaiting() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2084,7 +2034,7 @@ public void closeInterruptsConditionWaiting() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2107,7 +2057,6 @@ public void closeInterruptsConditionWaiting() throws Exception { @Test public void setActiveSourceReturnsShutdownStatus() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -2128,7 +2077,7 @@ public CompletableFuture run() { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2146,7 +2095,6 @@ public CompletableFuture run() { @Test public void activeSourceClosedWhenSwitchingSynchronizers() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2171,7 +2119,7 @@ public void close() { () -> new MockQueuedSynchronizer(secondSyncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2191,7 +2139,6 @@ public void close() { @Test public void activeSourceClosedOnShutdown() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2211,7 +2158,7 @@ public void close() { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2229,7 +2176,6 @@ public void close() { @Test public void setActiveSourceOnInitializerChecksShutdown() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CountDownLatch initializerStarted = new CountDownLatch(1); @@ -2249,7 +2195,7 @@ public void setActiveSourceOnInitializerChecksShutdown() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2280,7 +2226,6 @@ public void setActiveSourceOnInitializerChecksShutdown() throws Exception { @Test public void parkedSynchronizerSkippedInRotation() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2316,7 +2261,7 @@ public void parkedSynchronizerSkippedInRotation() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2334,7 +2279,6 @@ public void parkedSynchronizerSkippedInRotation() throws Exception { @Test public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2363,7 +2307,7 @@ public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { ); // Short recovery timeout - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(1), /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2384,7 +2328,6 @@ public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { @Test public void fallbackMovesToNextSynchronizer() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2409,7 +2352,7 @@ public void fallbackMovesToNextSynchronizer() throws Exception { ); // Short fallback timeout - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(1), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2432,7 +2375,6 @@ public void fallbackMovesToNextSynchronizer() throws Exception { @Test public void conditionsClosedAfterSynchronizerLoop() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2445,7 +2387,7 @@ public void conditionsClosedAfterSynchronizerLoop() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(1), /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2464,7 +2406,6 @@ public void conditionsClosedAfterSynchronizerLoop() throws Exception { @Test public void conditionsInformedOfAllResults() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2478,7 +2419,7 @@ public void conditionsInformedOfAllResults() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(10), /* recoveryTimeout= */ Duration.ofSeconds(20), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2497,7 +2438,6 @@ public void conditionsInformedOfAllResults() throws Exception { @Test public void conditionsClosedOnException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2512,7 +2452,7 @@ public void conditionsClosedOnException() throws Exception { )) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(1), /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2529,7 +2469,6 @@ public void conditionsClosedOnException() throws Exception { @Test public void primeSynchronizerHasNoRecoveryCondition() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2543,7 +2482,7 @@ public void primeSynchronizerHasNoRecoveryCondition() throws Exception { () -> new MockQueuedSynchronizer(new LinkedBlockingQueue<>()) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(1), /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2561,7 +2500,6 @@ public void primeSynchronizerHasNoRecoveryCondition() throws Exception { @Test public void nonPrimeSynchronizerHasBothConditions() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2580,7 +2518,7 @@ public void nonPrimeSynchronizerHasBothConditions() throws Exception { () -> new MockQueuedSynchronizer(secondSyncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(1), /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2598,7 +2536,6 @@ public void nonPrimeSynchronizerHasBothConditions() throws Exception { @Test public void singleSynchronizerHasNoConditions() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2610,7 +2547,7 @@ public void singleSynchronizerHasNoConditions() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(1), /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2628,7 +2565,6 @@ public void singleSynchronizerHasNoConditions() throws Exception { @Test public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2641,7 +2577,7 @@ public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(1), /* recoveryTimeout= */ Duration.ofSeconds(2), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2664,7 +2600,6 @@ public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { @Test public void changeSetAppliedToDataSourceUpdates() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -2677,7 +2612,7 @@ public void changeSetAppliedToDataSourceUpdates() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2695,7 +2630,6 @@ public void changeSetAppliedToDataSourceUpdates() throws Exception { @Test public void multipleChangeSetsAppliedInOrder() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2709,7 +2643,7 @@ public void multipleChangeSetsAppliedInOrder() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2729,7 +2663,6 @@ public void multipleChangeSetsAppliedInOrder() throws Exception { @Test public void selectorNonEmptyCompletesInitialization() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture firstInitializerFuture = CompletableFuture.completedFuture( @@ -2748,7 +2681,7 @@ public void selectorNonEmptyCompletesInitialization() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, ImmutableList.of(), null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, ImmutableList.of(), null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2770,7 +2703,6 @@ public void selectorNonEmptyCompletesInitialization() throws Exception { @Test public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -2783,7 +2715,7 @@ public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() thro ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2809,7 +2741,6 @@ public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() thro @Test public void synchronizerChangeSetAlwaysCompletesStartFuture() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2822,7 +2753,7 @@ public void synchronizerChangeSetAlwaysCompletesStartFuture() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2843,7 +2774,6 @@ public void synchronizerChangeSetAlwaysCompletesStartFuture() throws Exception { @Test public void goodbyeStatusHandledGracefully() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2857,7 +2787,7 @@ public void goodbyeStatusHandledGracefully() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2878,7 +2808,6 @@ public void goodbyeStatusHandledGracefully() throws Exception { @Test public void shutdownStatusExitsImmediately() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2897,7 +2826,7 @@ public void shutdownStatusExitsImmediately() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2918,7 +2847,6 @@ public void shutdownStatusExitsImmediately() throws Exception { @Test public void fdv1FallbackFlagHonored() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2930,7 +2858,7 @@ public void fdv1FallbackFlagHonored() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2952,7 +2880,6 @@ public void fdv1FallbackFlagHonored() throws Exception { @Test public void emptyInitializerListSkipsToSynchronizers() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2969,7 +2896,7 @@ public void emptyInitializerListSkipsToSynchronizers() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -2987,7 +2914,6 @@ public void emptyInitializerListSkipsToSynchronizers() throws Exception { @Test public void startedFlagPreventsMultipleRuns() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger runCount = new AtomicInteger(0); @@ -3003,7 +2929,7 @@ public void startedFlagPreventsMultipleRuns() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3028,7 +2954,6 @@ public void startedFlagPreventsMultipleRuns() throws Exception { @Test public void startBeforeRunCompletesAllComplete() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); BlockingQueue syncResults = new LinkedBlockingQueue<>(); @@ -3039,7 +2964,7 @@ public void startBeforeRunCompletesAllComplete() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3065,7 +2990,6 @@ public void startBeforeRunCompletesAllComplete() throws Exception { @Test public void statusTransitionsToValidAfterInitialization() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -3083,7 +3007,6 @@ public void statusTransitionsToValidAfterInitialization() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3105,7 +3028,6 @@ public void statusTransitionsToValidAfterInitialization() throws Exception { @Test public void statusIncludesErrorInfoOnFailure() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -3133,7 +3055,6 @@ public void statusIncludesErrorInfoOnFailure() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3156,7 +3077,6 @@ public void statusIncludesErrorInfoOnFailure() throws Exception { @Test public void statusRemainsValidDuringSynchronizerOperation() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -3178,7 +3098,6 @@ public void statusRemainsValidDuringSynchronizerOperation() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3201,7 +3120,6 @@ public void statusRemainsValidDuringSynchronizerOperation() throws Exception { @Test public void statusStaysInterruptedWhileOnlySynchronizerIsParked() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -3238,7 +3156,6 @@ public void statusStaysInterruptedWhileOnlySynchronizerIsParked() throws Excepti sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), @@ -3269,7 +3186,6 @@ public void statusStaysInterruptedWhileOnlySynchronizerIsParked() throws Excepti @Test public void fdv1FallbackActivatesWhenFlagSet() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -3300,7 +3216,6 @@ public void fdv1FallbackActivatesWhenFlagSet() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3324,7 +3239,6 @@ public void fdv1FallbackActivatesWhenFlagSet() throws Exception { @Test public void fdv1FallbackSynchronizerIsParkedAndRestartedDuringDirectedFallback() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger fdv1BuildCount = new AtomicInteger(); @@ -3350,7 +3264,7 @@ public void fdv1FallbackSynchronizerIsParkedAndRestartedDuringDirectedFallback() }; // The FDv1 fallback synchronizer is parked for 50-100ms. - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), @@ -3370,7 +3284,6 @@ public void fdv1FallbackSynchronizerIsParkedAndRestartedDuringDirectedFallback() @Test public void fdv1FallbackNotCalledWithoutFlag() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -3398,7 +3311,6 @@ public void fdv1FallbackNotCalledWithoutFlag() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3421,7 +3333,6 @@ public void fdv1FallbackNotCalledWithoutFlag() throws Exception { @Test public void fdv1FallbackWorksAfterInterruption() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -3460,7 +3371,6 @@ public void fdv1FallbackWorksAfterInterruption() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3487,7 +3397,6 @@ public void fdv1FallbackWorksAfterInterruption() throws Exception { // fallback TTL ends, and then restarts the FDv2 synchronizers. @Test public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredRecoversAfterTTL() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger firstSyncBuildCount = new AtomicInteger(); AtomicBoolean secondSyncCalled = new AtomicBoolean(false); @@ -3512,7 +3421,7 @@ public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredRecoversAfterTTL() th ); // No FDv1 fallback synchronizer is configured, and the fallback TTL ends after 50-100ms. - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3538,7 +3447,6 @@ public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredRecoversAfterTTL() th // While no FDv1 fallback synchronizer is configured, the FDv2 synchronizer that sent the directive is stopped. @Test public void directiveWithoutFDv1ClosesFDv2SynchronizerDuringFallback() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CountDownLatch closed = new CountDownLatch(1); @@ -3558,7 +3466,7 @@ public void close() { ); // No FDv1 fallback synchronizer is configured, and the fallback lasts much longer than the test. - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3577,7 +3485,6 @@ public void close() { // When the fallback TTL ends, the data system stops the FDv1 synchronizer and restarts the FDv2 synchronizers. @Test public void fdv1FallbackEndsAfterTTL() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger fdv2BuildCount = new AtomicInteger(); AtomicBoolean fdv1Closed = new AtomicBoolean(false); @@ -3605,7 +3512,7 @@ public void close() { }; // The fallback TTL ends after 50-100ms. - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3626,7 +3533,6 @@ public void close() { // When the fallback TTL ends before the FDv1 synchronizer's unpark time, the FDv2 synchronizers restart at the TTL. @Test public void fdv1FallbackEndsWhileFDv1SynchronizerIsParked() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger fdv2BuildCount = new AtomicInteger(); AtomicInteger fdv1BuildCount = new AtomicInteger(); @@ -3663,7 +3569,6 @@ public void fdv1FallbackEndsWhileFDv1SynchronizerIsParked() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3686,7 +3591,6 @@ public void fdv1FallbackEndsWhileFDv1SynchronizerIsParked() throws Exception { // A directive during the FDv1 fallback replaces the TTL, and the FDv1 synchronizer keeps running. @Test public void directiveDuringFDv1FallbackExtendsTTL() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger fdv2BuildCount = new AtomicInteger(); AtomicInteger fdv1BuildCount = new AtomicInteger(); @@ -3720,7 +3624,6 @@ public void directiveDuringFDv1FallbackExtendsTTL() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3751,7 +3654,6 @@ public void directiveDuringFDv1FallbackExtendsTTL() throws Exception { // When the FDv2 synchronizers send the directive again after the fallback TTL ends, the data system falls back again. @Test public void directiveDuringRecoveryFallsBackToFDv1Again() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger fdv2BuildCount = new AtomicInteger(); AtomicInteger fdv1BuildCount = new AtomicInteger(); @@ -3775,7 +3677,7 @@ public void directiveDuringRecoveryFallsBackToFDv1Again() throws Exception { }; // The fallback TTL ends after 50-100ms. - FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3795,7 +3697,6 @@ public void directiveDuringRecoveryFallsBackToFDv1Again() throws Exception { @Test public void fdv1FallbackBlocksOtherSynchronizers() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -3833,7 +3734,6 @@ public void fdv1FallbackBlocksOtherSynchronizers() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3857,7 +3757,6 @@ public void fdv1FallbackBlocksOtherSynchronizers() throws Exception { @Test public void fdv1FallbackOnlyCalledOncePerDataSource() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -3889,7 +3788,6 @@ public void fdv1FallbackOnlyCalledOncePerDataSource() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3920,7 +3818,6 @@ public void fdv1FallbackOnlyCalledOncePerDataSource() throws Exception { // without giving any of the configured FDv2 synchronizers a chance to run. @Test public void fdv1FallbackOnInitializerSuccessAppliesPayloadAndSwitches() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); // Initializer returns a payload with a selector AND the FDv1 fallback signal. @@ -3955,7 +3852,6 @@ public void fdv1FallbackOnInitializerSuccessAppliesPayloadAndSwitches() throws E sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -3984,7 +3880,6 @@ public void fdv1FallbackOnInitializerSuccessAppliesPayloadAndSwitches() throws E // to the FDv1 fallback synchronizer without trying FDv2 synchronizers. @Test public void fdv1FallbackOnInitializerErrorSwitchesToFDv1() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initFuture = CompletableFuture.completedFuture( @@ -4026,7 +3921,6 @@ public void fdv1FallbackOnInitializerErrorSwitchesToFDv1() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -4049,7 +3943,6 @@ public void fdv1FallbackOnInitializerErrorSwitchesToFDv1() throws Exception { // INTERRUPTED, runs no synchronizer until the fallback TTL ends, and then starts the FDv2 synchronizers. @Test public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredRecoversAfterTTL() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger syncBuildCount = new AtomicInteger(); @@ -4078,7 +3971,7 @@ public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredRecoversAfterTTL() thr ); // No FDv1 fallback synchronizer is configured, and the fallback TTL ends after 50-100ms. - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -4105,7 +3998,6 @@ public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredRecoversAfterTTL() thr // Ruby behavior on initialization gating. @Test public void fdv1FallbackOnInitializerSuccessNoSelectorAppliesPayloadAndSwitches() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initFuture = CompletableFuture.completedFuture( @@ -4134,7 +4026,6 @@ public void fdv1FallbackOnInitializerSuccessNoSelectorAppliesPayloadAndSwitches( sink, Thread.NORM_PRIORITY, logger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -4161,7 +4052,6 @@ public void fdv1FallbackOnInitializerSuccessNoSelectorAppliesPayloadAndSwitches( @Test public void orchestrationLogging_warnsWhenNoInitializersOrSynchronizersConfigured() throws Exception { - executor = Executors.newScheduledThreadPool(1); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); FDv2DataSource dataSource = new FDv2DataSource( ImmutableList.of(), @@ -4170,7 +4060,6 @@ public void orchestrationLogging_warnsWhenNoInitializersOrSynchronizersConfigure sink, Thread.NORM_PRIORITY, testLogger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -4187,7 +4076,6 @@ public void orchestrationLogging_warnsWhenNoInitializersOrSynchronizersConfigure @Test public void orchestrationLogging_logsInitializerStartSuccessAndErrorOnException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture first = new CompletableFuture<>(); first.completeExceptionally(new RuntimeException("first-init-failure")); @@ -4205,7 +4093,6 @@ public void orchestrationLogging_logsInitializerStartSuccessAndErrorOnException( sink, Thread.NORM_PRIORITY, testLogger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -4223,7 +4110,6 @@ public void orchestrationLogging_logsInitializerStartSuccessAndErrorOnException( @Test public void orchestrationLogging_logsInitializerFailedOnStatusResult() throws Exception { - executor = Executors.newScheduledThreadPool(1); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initFuture = CompletableFuture.completedFuture( FDv2SourceResult.interrupted( @@ -4246,7 +4132,6 @@ public void orchestrationLogging_logsInitializerFailedOnStatusResult() throws Ex sink, Thread.NORM_PRIORITY, testLogger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -4263,7 +4148,6 @@ public void orchestrationLogging_logsInitializerFailedOnStatusResult() throws Ex @Test public void orchestrationLogging_logsSynchronizerStartingStatusInterruptedAndShutdown() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); IterableAsyncQueue q = new IterableAsyncQueue<>(); q.put(FDv2SourceResult.interrupted( @@ -4286,7 +4170,6 @@ public void orchestrationLogging_logsSynchronizerStartingStatusInterruptedAndShu sink, Thread.NORM_PRIORITY, testLogger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), @@ -4305,7 +4188,6 @@ public void orchestrationLogging_logsSynchronizerStartingStatusInterruptedAndShu @Test public void orchestrationLogging_logsSynchronizerParkedAndAllParked() throws Exception { - executor = Executors.newScheduledThreadPool(1); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); BlockingQueue builds = new LinkedBlockingQueue<>(); // Every instance of the only synchronizer fails with a terminal error. Each build is recorded. @@ -4324,7 +4206,6 @@ public void orchestrationLogging_logsSynchronizerParkedAndAllParked() throws Exc sink, Thread.NORM_PRIORITY, testLogger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMillis(100), @@ -4349,7 +4230,6 @@ public void orchestrationLogging_logsSynchronizerParkedAndAllParked() throws Exc @Test public void orchestrationLogging_logsErrorWhenSynchronizerNextThrows() throws Exception { - executor = Executors.newScheduledThreadPool(1); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); // The second synchronizer delivers data, so start() completes. IterableAsyncQueue followUpQueue = new IterableAsyncQueue<>(); @@ -4368,7 +4248,6 @@ public void orchestrationLogging_logsErrorWhenSynchronizerNextThrows() throws Ex sink, Thread.NORM_PRIORITY, testLogger, - executor, /* fallbackTimeout= */ Duration.ofSeconds(120), /* recoveryTimeout= */ Duration.ofSeconds(300), /* parkInitialDelay= */ Duration.ofMinutes(5), From fb3e5232338d429a224028247bd6e57bb39b4115 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Thu, 5 Nov 2026 16:47:54 -0800 Subject: [PATCH 09/13] test: Stop the adapter test from holding a common-pool thread --- .../sdk/server/DataSourceSynchronizerAdapterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java index 1eeb5053..eb042484 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java @@ -373,7 +373,7 @@ public void interruptedStatusIsTerminalOnlyForUnexpectedHttpErrors() throws Exce DataSourceSynchronizerAdapter adapter = new DataSourceSynchronizerAdapter(sink -> { capturedSink.set(sink); - return new MockDataSource(new CountDownLatch(1), null); + return new MockDataSource(null, null); }); resourcesToClose.add(adapter); From ef226272aa05641f918cfa6ef2a40a690deecd87 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Thu, 5 Nov 2026 18:10:08 -0800 Subject: [PATCH 10/13] fix: Report INTERRUPTED while waiting for a synchronizer to unpark --- .../sdk/server/FDv2DataSource.java | 2 ++ .../sdk/server/FDv2DataSourceTest.java | 22 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index bb74c12a..00ac2602 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -708,6 +708,8 @@ private Synchronizer buildNextAvailableSynchronizer() { wait = Duration.between(Instant.now(), earliest); logger.warn("All synchronizers are backing off for at least {}s.", Math.max(0, wait.getSeconds())); } + // No synchronizer is running until the wait ends. + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, null); if (!waitUnlessClosed(wait)) { return null; } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index 2b625f85..a24cc246 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -3434,9 +3434,8 @@ public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredRecoversAfterTTL() th dataSource.start().get(2, TimeUnit.SECONDS); // The data system reports INTERRUPTED, and restarts the first synchronizer after the TTL. - List statuses = sink.awaitStatuses(4, 2, TimeUnit.SECONDS); + List statuses = sink.awaitStatuses(3, 2, TimeUnit.SECONDS); assertEquals(Arrays.asList( - DataSourceStatusProvider.State.VALID, DataSourceStatusProvider.State.VALID, DataSourceStatusProvider.State.INTERRUPTED, DataSourceStatusProvider.State.VALID), statuses); @@ -3537,7 +3536,7 @@ public void fdv1FallbackEndsWhileFDv1SynchronizerIsParked() throws Exception { AtomicInteger fdv2BuildCount = new AtomicInteger(); AtomicInteger fdv1BuildCount = new AtomicInteger(); - // The first instance reports an interruption with the directive. The second sends a payload. + // The first instance reports an interruption with the directive. The second sends a payload with the directive. ImmutableList> synchronizers = ImmutableList.of( () -> { BlockingQueue results = new LinkedBlockingQueue<>(); @@ -3547,7 +3546,7 @@ public void fdv1FallbackEndsWhileFDv1SynchronizerIsParked() throws Exception { DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), true)); } else { - results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), true)); } return new MockQueuedSynchronizer(results); } @@ -3585,6 +3584,14 @@ public void fdv1FallbackEndsWhileFDv1SynchronizerIsParked() throws Exception { sink.awaitApplyCount(1, 2, TimeUnit.SECONDS); assertEquals(1, sink.getApplyCount()); assertEquals(2, fdv2BuildCount.get()); + + // Each directive falls back to the FDv1 synchronizer, which is still parked, so nothing runs until the TTL. + List statuses = sink.awaitStatuses(4, 2, TimeUnit.SECONDS); + assertEquals(Arrays.asList( + DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.State.VALID, + DataSourceStatusProvider.State.INTERRUPTED), statuses); assertEquals(1, fdv1BuildCount.get()); } @@ -4284,9 +4291,12 @@ public boolean apply(ChangeSet Date: Thu, 5 Nov 2026 18:36:13 -0800 Subject: [PATCH 11/13] fix: Report the reset threshold from the FDv1 fallback adapter --- .../server/DataSourceSynchronizerAdapter.java | 22 +++++++++- .../DataSourceSynchronizerAdapterTest.java | 43 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java index 81a35189..97acc1e2 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java @@ -132,9 +132,14 @@ public void close() { * This sink doesn't delegate to any other sink - it exists solely to convert FDv1 updates to FDv2 results. */ private static class ConvertingUpdateSink implements DataSourceUpdateSink { + // The number of VALID statuses in a row that meet the reset threshold, the same as for a polling synchronizer. + private static final int RESET_THRESHOLD_SUCCESSFUL_RESULTS = 2; + private final IterableAsyncQueue resultQueue; // Only a full data set carries an environment ID; it is retained for subsequent partial updates. private volatile String environmentId = null; + // The number of VALID statuses the data source has reported since its last failure. + private int consecutiveSuccessfulResults = 0; public ConvertingUpdateSink(IterableAsyncQueue resultQueue) { this.resultQueue = resultQueue; @@ -191,6 +196,7 @@ public void updateStatus(DataSourceStatusProvider.State newState, DataSourceStat // Convert state changes to FDv2SourceResult status events switch (newState) { case INTERRUPTED: + consecutiveSuccessfulResults = 0; if (newError != null && newError.getKind() == DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE && HttpErrors.classifyHttpFailure(newError.getStatusCode()) == FailureClass.UNEXPECTED) { @@ -200,13 +206,27 @@ public void updateStatus(DataSourceStatusProvider.State newState, DataSourceStat } break; case OFF: + consecutiveSuccessfulResults = 0; if (newError != null) { resultQueue.put(FDv2SourceResult.terminalError(newError, false)); } break; case VALID: + consecutiveSuccessfulResults++; + if (consecutiveSuccessfulResults == RESET_THRESHOLD_SUCCESSFUL_RESULTS) { + // Any data that came with this status is already queued, so report the reset with no changes. + ChangeSet>>> noChanges = + new ChangeSet<>( + ChangeSetType.None, + Selector.EMPTY, + Collections.emptyList(), + environmentId, + /* shouldPersist= */ true); + resultQueue.put(FDv2SourceResult.changeSet(noChanges, false).withResetThresholdMet()); + } + break; case INITIALIZING: - // These states don't map to FDv2SourceResult status events + // This state doesn't map to an FDv2SourceResult status event break; } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java index eb042484..a68b781c 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java @@ -398,4 +398,47 @@ public void interruptedStatusIsTerminalOnlyForUnexpectedHttpErrors() throws Exce adapter.close(); } + + /** Test that two VALID statuses in a row meet the reset threshold, and a failure restarts the count. */ + @Test + public void resetThresholdIsMetAfterTwoValidStatusesInARow() throws Exception { + AtomicReference capturedSink = new AtomicReference<>(); + + DataSourceSynchronizerAdapter adapter = new DataSourceSynchronizerAdapter(sink -> { + capturedSink.set(sink); + return new MockDataSource(null, null); + }); + resourcesToClose.add(adapter); + + // Report a successful poll, a failed poll, and two more successful polls, as the FDv1 polling data source does. + CompletableFuture firstFuture = adapter.next(); + DataSourceUpdateSink sink = capturedSink.get(); + sink.init(new FullDataSet<>(DataBuilder.forStandardTypes().build().getData(), true, null)); + sink.updateStatus(DataSourceStatusProvider.State.VALID, null); + sink.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.ErrorInfo.fromHttpError(503)); + sink.init(new FullDataSet<>(DataBuilder.forStandardTypes().build().getData(), true, null)); + sink.updateStatus(DataSourceStatusProvider.State.VALID, null); + sink.init(new FullDataSet<>(DataBuilder.forStandardTypes().build().getData(), true, null)); + sink.updateStatus(DataSourceStatusProvider.State.VALID, null); + List results = new ArrayList<>(); + results.add(firstFuture.get(2, TimeUnit.SECONDS)); + for (int i = 0; i < 4; i++) { + results.add(adapter.next().get(2, TimeUnit.SECONDS)); + } + + // Only the last poll meets the reset threshold, and it reports the reset after its data. + assertEquals(SourceResultType.CHANGE_SET, results.get(0).getResultType()); + assertFalse(results.get(0).isResetThresholdMet()); + assertEquals(SourceSignal.INTERRUPTED, results.get(1).getStatus().getState()); + assertFalse(results.get(1).isResetThresholdMet()); + assertEquals(SourceResultType.CHANGE_SET, results.get(2).getResultType()); + assertFalse(results.get(2).isResetThresholdMet()); + assertEquals(SourceResultType.CHANGE_SET, results.get(3).getResultType()); + assertFalse(results.get(3).isResetThresholdMet()); + assertEquals(ChangeSetType.None, results.get(4).getChangeSet().getType()); + assertTrue(results.get(4).isResetThresholdMet()); + + adapter.close(); + } } From 198207ab3ecf6552117674c7f9fdc52b903441bd Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Thu, 5 Nov 2026 18:48:28 -0800 Subject: [PATCH 12/13] fix: Report a terminal error when the stream thread ends unexpectedly --- .../sdk/server/StreamingSynchronizerImpl.java | 5 ++- .../server/StreamingSynchronizerImplTest.java | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java index 62613929..c9600888 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java @@ -219,9 +219,8 @@ private Thread getRunThread() { e.toString(), Instant.now() ); - // We aren't restarting the event source here. We aren't going to automatically recover, so the - // data system will move to the next source when it determined this source is unhealthy. - resultQueue.put(FDv2SourceResult.interrupted(errorInfo, getFallback(e))); + // This synchronizer cannot recover, so the failure is terminal. The data system will start a new one. + shutdownFuture.complete(FDv2SourceResult.terminalError(errorInfo, getFallback(e))); } finally { eventSource.close(); } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java index 8e32d5c0..0a40b23c 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java @@ -2177,4 +2177,37 @@ public void certificateFailureIsTerminal() throws Exception { synchronizer.close(); } } + + @Test + public void unexpectedExceptionOnStreamThreadIsTerminal() throws Exception { + try (HttpServer server = HttpServer.start(Handlers.SSE.start())) { + HttpProperties httpProperties = toHttpProperties(clientContext("sdk-key", baseConfig().build()).getHttp()); + // The selector source throws when the stream connects, which ends the stream thread. + SelectorSource selectorSource = mock(SelectorSource.class); + when(selectorSource.getSelector()).thenThrow(new IllegalStateException("unexpected")); + + StreamingSynchronizerImpl synchronizer = new StreamingSynchronizerImpl( + httpProperties, + server.getUri(), + "/stream", + testLogger, + selectorSource, + executor, + null, + Duration.ofMillis(100), + StreamingSynchronizerImpl.RESET_THRESHOLD, + Thread.NORM_PRIORITY, + null + ); + + // Start the stream. + FDv2SourceResult result = synchronizer.next().get(5, TimeUnit.SECONDS); + + // The stream thread cannot recover, so the failure is terminal for this synchronizer. + assertEquals(SourceResultType.STATUS, result.getResultType()); + assertEquals(SourceSignal.TERMINAL_ERROR, result.getStatus().getState()); + + synchronizer.close(); + } + } } From 0dafa429e422c57defaa59ad2c2cce0913a63b5c Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Thu, 5 Nov 2026 19:34:48 -0800 Subject: [PATCH 13/13] test: Remove condition tests that assert nothing --- .../sdk/server/FDv2DataSourceTest.java | 158 ------------------ 1 file changed, 158 deletions(-) diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index a24cc246..399afcf8 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -2373,37 +2373,6 @@ public void fallbackMovesToNextSynchronizer() throws Exception { // Condition Lifecycle // ============================================================================ - @Test - public void conditionsClosedAfterSynchronizerLoop() throws Exception { - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - BlockingQueue syncResults = new LinkedBlockingQueue<>(); - syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - syncResults.add(makeTerminalErrorResult()); - - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(syncResults) - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, - /* fallbackTimeout= */ Duration.ofSeconds(1), - /* recoveryTimeout= */ Duration.ofSeconds(2), - /* parkInitialDelay= */ Duration.ofMinutes(5), - /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1), - /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - dataSource.close(); - - // If conditions weren't closed properly, we might see issues - } - @Test public void conditionsInformedOfAllResults() throws Exception { MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); @@ -2436,133 +2405,6 @@ public void conditionsInformedOfAllResults() throws Exception { assertTrue(sink.getApplyCount() >= 2); } - @Test - public void conditionsClosedOnException() throws Exception { - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - CompletableFuture exceptionFuture = new CompletableFuture<>(); - exceptionFuture.completeExceptionally(new RuntimeException("Error")); - - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockSynchronizer(exceptionFuture), - () -> new MockSynchronizer(CompletableFuture.completedFuture( - FDv2SourceResult.changeSet(makeChangeSet(false), false) - )) - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, - /* fallbackTimeout= */ Duration.ofSeconds(1), - /* recoveryTimeout= */ Duration.ofSeconds(2), - /* parkInitialDelay= */ Duration.ofMinutes(5), - /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1), - /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Conditions should be closed despite exception - } - - @Test - public void primeSynchronizerHasNoRecoveryCondition() throws Exception { - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - BlockingQueue syncResults = new LinkedBlockingQueue<>(); - syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - // Keep alive - - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(syncResults), - () -> new MockQueuedSynchronizer(new LinkedBlockingQueue<>()) - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, - /* fallbackTimeout= */ Duration.ofSeconds(1), - /* recoveryTimeout= */ Duration.ofSeconds(2), - /* parkInitialDelay= */ Duration.ofMinutes(5), - /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1), - /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Prime synchronizer should not have a recovery condition - // This is tested implicitly by the implementation - } - - @Test - public void nonPrimeSynchronizerHasBothConditions() throws Exception { - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - // First: send INTERRUPTED to trigger fallback - BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); - firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - firstSyncResults.add(makeInterruptedResult()); - - // Second: will have both conditions - BlockingQueue secondSyncResults = new LinkedBlockingQueue<>(); - secondSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(firstSyncResults), - () -> new MockQueuedSynchronizer(secondSyncResults) - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, - /* fallbackTimeout= */ Duration.ofSeconds(1), - /* recoveryTimeout= */ Duration.ofSeconds(2), - /* parkInitialDelay= */ Duration.ofMinutes(5), - /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1), - /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Non-prime synchronizer should have both fallback and recovery - // This is tested implicitly by the implementation - } - - @Test - public void singleSynchronizerHasNoConditions() throws Exception { - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - BlockingQueue syncResults = new LinkedBlockingQueue<>(); - syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(syncResults) - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, - /* fallbackTimeout= */ Duration.ofSeconds(1), - /* recoveryTimeout= */ Duration.ofSeconds(2), - /* parkInitialDelay= */ Duration.ofMinutes(5), - /* parkMaxDelay= */ Duration.ofHours(1), - /* failoverDelay= */ Duration.ofSeconds(1), - /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Single synchronizer should have no conditions - // This is tested implicitly by the implementation - } - @Test public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink();