From f670e391f5cf1a29f7bff98f230ab3fab6b18b41 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Fri, 6 Nov 2026 13:44:20 -0800 Subject: [PATCH 01/12] refactor: Check FDv2 fallback and recovery deadlines in the synchronizer loop --- .../sdk/android/FDv2DataSource.java | 163 +++++++---- .../sdk/android/FDv2DataSourceConditions.java | 158 ----------- .../sdk/android/MonotonicTime.java | 70 +++++ .../android/FDv2DataSourceConditionsTest.java | 263 ------------------ .../sdk/android/FDv2DataSourceTest.java | 102 +++---- .../sdk/android/MonotonicTimeTest.java | 35 +++ 6 files changed, 262 insertions(+), 529 deletions(-) delete mode 100644 launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSourceConditions.java create mode 100644 launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/MonotonicTime.java delete mode 100644 launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceConditionsTest.java create mode 100644 launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/MonotonicTimeTest.java diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java index 9b8d40562..a2026ccab 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java @@ -21,13 +21,14 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -54,10 +55,20 @@ public interface DataSourceFactory { private static final String INITIALIZER_CANCELLED = "Initializer cancelled: {}"; private static final String INITIALIZER_INTERRUPTED = "Initializer interrupted: {}"; + /** + * Default fallback timeout of 2 minutes. + */ + private static final long DEFAULT_FALLBACK_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2); + + /** + * Default recovery timeout of 5 minutes. + */ + private static final long DEFAULT_RECOVERY_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(5); + private final List> cacheInitializers; private final SourceManager sourceManager; - private final long fallbackTimeoutSeconds; - private final long recoveryTimeoutSeconds; + private final long fallbackTimeoutMillis; + private final long recoveryTimeoutMillis; private final ScheduledExecutorService sharedExecutor; private final AtomicBoolean startCalled = new AtomicBoolean(false); @@ -94,8 +105,8 @@ public interface DataSourceFactory { ) { this(evaluationContext, initializers, synchronizers, fdv1FallbackSynchronizer, dataSourceUpdateSink, sharedExecutor, logger, - FDv2DataSourceConditions.DEFAULT_FALLBACK_TIMEOUT_SECONDS, - FDv2DataSourceConditions.DEFAULT_RECOVERY_TIMEOUT_SECONDS); + DEFAULT_FALLBACK_TIMEOUT_MILLIS, + DEFAULT_RECOVERY_TIMEOUT_MILLIS); } /** @@ -110,9 +121,9 @@ public interface DataSourceFactory { * @param sharedExecutor executor used for internal background tasks; must have at least * 2 threads available for this data source to run properly. * @param logger logger - * @param fallbackTimeoutSeconds seconds of INTERRUPTED state before falling back to the + * @param fallbackTimeoutMillis milliseconds of INTERRUPTED state before falling back to the * next synchronizer - * @param recoveryTimeoutSeconds seconds before attempting to recover to the primary + * @param recoveryTimeoutMillis milliseconds before attempting to recover to the primary * synchronizer */ FDv2DataSource( @@ -123,8 +134,8 @@ public interface DataSourceFactory { @NonNull DataSourceUpdateSinkV2 dataSourceUpdateSink, @NonNull ScheduledExecutorService sharedExecutor, @NonNull LDLogger logger, - long fallbackTimeoutSeconds, - long recoveryTimeoutSeconds + long fallbackTimeoutMillis, + long recoveryTimeoutMillis ) { this.evaluationContext = evaluationContext; this.dataSourceUpdateSink = dataSourceUpdateSink; @@ -150,8 +161,8 @@ public interface DataSourceFactory { // note that the source manager only uses the initializers after the cache initializers and not the cache initializers this.sourceManager = new SourceManager(allSynchronizers, generalInitializers); - this.fallbackTimeoutSeconds = fallbackTimeoutSeconds; - this.recoveryTimeoutSeconds = recoveryTimeoutSeconds; + this.fallbackTimeoutMillis = fallbackTimeoutMillis; + this.recoveryTimeoutMillis = recoveryTimeoutMillis; this.sharedExecutor = sharedExecutor; } @@ -468,16 +479,46 @@ private void runGeneralInitializers( } } - private List getConditions(int synchronizerCount, boolean isPrime) { - if (synchronizerCount <= 1) { - return Collections.emptyList(); + /** + * Waits for the future to complete, or until the given time. + * + * @param future the future to wait for + * @param time the time to stop waiting, or null to wait until the future completes + * @return true if the future completed + * @throws InterruptedException if the thread is interrupted while waiting + */ + private static boolean awaitUntil(@NonNull Future future, @Nullable MonotonicTime time) + throws InterruptedException { + try { + if (time == null) { + future.get(); + } else { + future.get(Math.max(0, MonotonicTime.now().nanosUntil(time)), TimeUnit.NANOSECONDS); + } + return true; + } catch (TimeoutException e) { + return false; + } catch (ExecutionException | CancellationException e) { + // The future completed with an error, which the caller gets from the future. + return true; } - List list = new ArrayList<>(); - list.add(new FDv2DataSourceConditions.FallbackCondition(sharedExecutor, fallbackTimeoutSeconds)); - if (!isPrime) { - list.add(new FDv2DataSourceConditions.RecoveryCondition(sharedExecutor, recoveryTimeoutSeconds)); + } + + /** + * Gets the earliest of the given times. + * + * @param times the times, any of which may be null + * @return the earliest time, or null if every time is null + */ + @Nullable + private static MonotonicTime earliest(MonotonicTime... times) { + MonotonicTime earliest = null; + for (MonotonicTime time : times) { + if (time != null && (earliest == null || time.isBefore(earliest))) { + earliest = time; + } } - return list; + return earliest; } private static String detailForThrowable(@Nullable Throwable error) { @@ -520,52 +561,32 @@ private void runSynchronizers( String synchronizerName = synchronizer.name(); logger.info("Synchronizer '{}' is starting.", synchronizerName); resetSynchronizerStatusDedupe(); - int synchronizerCount = sourceManager.getAvailableSynchronizerCount(); - boolean isPrime = sourceManager.isPrimeSynchronizer(); try { boolean running = true; - try (FDv2DataSourceConditions.Conditions conditions = - new FDv2DataSourceConditions.Conditions(getConditions(synchronizerCount, isPrime))) { - while (running) { - Future nextFuture = synchronizer.next(); - // Race the next synchronizer result against any active conditions - // (fallback/recovery timers). Whichever resolves first wins. - Object res = LDFutures.anyOf(conditions.getFuture(), nextFuture).get(); - - if (res instanceof FDv2DataSourceConditions.ConditionType) { - FDv2DataSourceConditions.ConditionType ct = (FDv2DataSourceConditions.ConditionType) res; - switch (ct) { - case FALLBACK: - logger.info( - "Fallback condition met, falling back from synchronizer '{}'.", - synchronizer.name() - ); - break; - case RECOVERY: - sourceManager.resetSourceIndex(); - logger.info( - "Recovery condition met, moving from synchronizer '{}' to primary synchronizer.", - synchronizer.name() - ); - break; - } - running = false; - break; - } - - if (!(res instanceof FDv2SourceResult)) { - logger.error("Unexpected result type from synchronizer: {}", res != null ? res.getClass().getName() : "null"); - continue; - } + // Fallback and recovery apply only when another synchronizer can take over. + boolean canFallBack = sourceManager.getAvailableSynchronizerCount() > 1; + boolean canRecover = canFallBack && !sourceManager.isPrimeSynchronizer(); + MonotonicTime recoveryTime = canRecover + ? MonotonicTime.now().plusMillis(recoveryTimeoutMillis) : null; + MonotonicTime fallbackTime = null; + // The result the loop is waiting for, or null if it has not asked the synchronizer for one. + Future nextFuture = null; + while (running) { + if (nextFuture == null) { + nextFuture = synchronizer.next(); + } - FDv2SourceResult result = (FDv2SourceResult) res; - // Let conditions observe the result before we act on it so - // they can update their internal state (e.g. reset interruption timers). - conditions.inform(result); + // Wait for the next result, or for the next deadline. + if (awaitUntil(nextFuture, earliest(recoveryTime, fallbackTime))) { + FDv2SourceResult result = nextFuture.get(); + // The pending result has arrived. + nextFuture = null; switch (result.getResultType()) { case CHANGE_SET: resetSynchronizerStatusDedupe(); + // The synchronizer is healthy, so it no longer needs to fall back. + fallbackTime = null; ChangeSet> changeSet = result.getChangeSet(); if (changeSet != null) { sink.apply(context, changeSet); @@ -582,6 +603,10 @@ private void runSynchronizers( synchronizer.name(), status.getState() ); + // The fallback timeout runs from the first interruption. + if (canFallBack && fallbackTime == null) { + fallbackTime = MonotonicTime.now().plusMillis(fallbackTimeoutMillis); + } sink.setStatus(DataSourceState.INTERRUPTED, status.getError()); break; case SHUTDOWN: @@ -631,6 +656,28 @@ private void runSynchronizers( running = false; } } + if (!running) { + break; + } + + // Check each deadline against the current time. Recovery takes precedence over fallback. + MonotonicTime now = MonotonicTime.now(); + if (recoveryTime != null && !now.isBefore(recoveryTime)) { + // For recovery, start the search for the next synchronizer at the first one. + sourceManager.resetSourceIndex(); + logger.info( + "Recovery condition met, moving from synchronizer '{}' to primary synchronizer.", + synchronizer.name() + ); + break; + } + if (fallbackTime != null && !now.isBefore(fallbackTime)) { + logger.info( + "Fallback condition met, falling back from synchronizer '{}'.", + synchronizer.name() + ); + break; + } } } catch (ExecutionException e) { Throwable failure = e.getCause() != null ? e.getCause() : e; diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSourceConditions.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSourceConditions.java deleted file mode 100644 index fd9541d24..000000000 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSourceConditions.java +++ /dev/null @@ -1,158 +0,0 @@ -package com.launchdarkly.sdk.android; - -import androidx.annotation.NonNull; - -import com.launchdarkly.sdk.android.subsystems.FDv2SourceResult; -import com.launchdarkly.sdk.fdv2.SourceResultType; -import com.launchdarkly.sdk.fdv2.SourceSignal; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -/** - * Fallback and recovery conditions for switching between FDv2 synchronizers. - * Conditions expose a future that completes when the condition fires. - * Uses {@link LDAwaitFuture} so behavior is equivalent to CompletableFuture on all API levels. - */ -final class FDv2DataSourceConditions { - - static final long DEFAULT_FALLBACK_TIMEOUT_SECONDS = 2 * 60; - static final long DEFAULT_RECOVERY_TIMEOUT_SECONDS = 5 * 60; - - enum ConditionType { FALLBACK, RECOVERY } - - /** Single condition: has a future that completes when it fires, inform(result), and close(). */ - interface Condition { - LDAwaitFuture getFuture(); - void inform(@NonNull FDv2SourceResult result); - void close(); - ConditionType getType(); - } - - /** - * Base for conditions that complete after a timeout. Holds the result future, executor, - * timeout, and optional timer; subclasses define when the timer is started and what completes the future. - */ - static abstract class TimedCondition implements Condition { - protected final LDAwaitFuture resultFuture = new LDAwaitFuture<>(); - protected final ScheduledExecutorService sharedExecutor; - protected final long timeoutSeconds; - /** Future for the timeout task, if any. Null when no timeout is active. */ - protected ScheduledFuture timerFuture; - - TimedCondition(@NonNull ScheduledExecutorService sharedExecutor, long timeoutSeconds) { - this.sharedExecutor = sharedExecutor; - this.timeoutSeconds = timeoutSeconds; - } - - @Override - public LDAwaitFuture getFuture() { - return resultFuture; - } - - @Override - public void close() { - if (timerFuture != null) { - timerFuture.cancel(false); - timerFuture = null; - } - } - } - - /** - * Fallback: on INTERRUPTED start timer; on CHANGE_SET cancel timer. Future completes with FALLBACK when timer fires. - */ - static final class FallbackCondition extends TimedCondition { - - FallbackCondition(@NonNull ScheduledExecutorService executor, long timeoutSeconds) { - super(executor, timeoutSeconds); - } - - @Override - public void inform(@NonNull FDv2SourceResult result) { - if (result.getResultType() == SourceResultType.CHANGE_SET) { - if (timerFuture != null) { - timerFuture.cancel(false); - timerFuture = null; - } - } - if (result.getResultType() == SourceResultType.STATUS - && result.getStatus() != null - && result.getStatus().getState() == SourceSignal.INTERRUPTED) { - if (timerFuture == null) { - timerFuture = sharedExecutor.schedule( - () -> resultFuture.set(ConditionType.FALLBACK), - timeoutSeconds, - TimeUnit.SECONDS); - } - } - } - - @Override - public ConditionType getType() { - return ConditionType.FALLBACK; - } - } - - /** - * Recovery: timer starts when built. Future completes with RECOVERY when timer fires. - */ - static final class RecoveryCondition extends TimedCondition { - - RecoveryCondition(@NonNull ScheduledExecutorService executor, long timeoutSeconds) { - super(executor, timeoutSeconds); - this.timerFuture = executor.schedule( - () -> resultFuture.set(ConditionType.RECOVERY), - timeoutSeconds, - TimeUnit.SECONDS); - } - - @Override - public void inform(@NonNull FDv2SourceResult result) {} - - @Override - public ConditionType getType() { - return ConditionType.RECOVERY; - } - } - - /** - * Wraps a list of conditions; getFuture() completes when the first condition's future completes. - */ - static final class Conditions implements AutoCloseable { - private final List conditions; - private final LDAwaitFuture conditionsFuture; - - Conditions(@NonNull List conditions) { - this.conditions = new ArrayList<>(conditions); - Future[] futures = new Future[conditions.size()]; - for (int i = 0; i < conditions.size(); i++) { - futures[i] = conditions.get(i).getFuture(); - } - this.conditionsFuture = LDFutures.anyOf(futures); - } - - LDAwaitFuture getFuture() { - return conditionsFuture; - } - - void inform(@NonNull FDv2SourceResult result) { - for (Condition c : conditions) { - c.inform(result); - } - } - - @Override - public void close() { - for (Condition c : conditions) { - c.close(); - } - } - } - - private FDv2DataSourceConditions() {} -} diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/MonotonicTime.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/MonotonicTime.java new file mode 100644 index 000000000..9f55280d5 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/MonotonicTime.java @@ -0,0 +1,70 @@ +package com.launchdarkly.sdk.android; + +import androidx.annotation.NonNull; + +import java.util.concurrent.TimeUnit; + +/** + * A point in time on the monotonic clock of {@link System#nanoTime()}. Changes to the system clock do not affect it. + * The value has no meaning as a date, so it is only useful to compare with other {@code MonotonicTime} values. + */ +final class MonotonicTime { + private final long nanos; + + private MonotonicTime(long nanos) { + this.nanos = nanos; + } + + /** + * The current time. + * + * @return the current time + */ + @NonNull + static MonotonicTime now() { + return new MonotonicTime(System.nanoTime()); + } + + /** + * A time with the given value of {@link System#nanoTime()}. Visible for testing. + * + * @param nanos the value of {@link System#nanoTime()} + * @return the time + */ + @NonNull + static MonotonicTime ofNanos(long nanos) { + return new MonotonicTime(nanos); + } + + /** + * The time after the given number of milliseconds. + * + * @param millis the milliseconds to add + * @return the later time + */ + @NonNull + MonotonicTime plusMillis(long millis) { + return new MonotonicTime(nanos + TimeUnit.MILLISECONDS.toNanos(millis)); + } + + /** + * True if this time is before the other time. + * + * @param other the other time + * @return true if this time is before the other time + */ + boolean isBefore(@NonNull MonotonicTime other) { + // The nanosecond count can overflow, so compare the difference instead of the counts. + return nanos - other.nanos < 0; + } + + /** + * The nanoseconds from this time until the other time. It is negative if the other time is before this time. + * + * @param other the other time + * @return the nanoseconds until the other time + */ + long nanosUntil(@NonNull MonotonicTime other) { + return other.nanos - nanos; + } +} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceConditionsTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceConditionsTest.java deleted file mode 100644 index 7acade435..000000000 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceConditionsTest.java +++ /dev/null @@ -1,263 +0,0 @@ -package com.launchdarkly.sdk.android; - -import com.launchdarkly.sdk.android.FDv2DataSourceConditions.ConditionType; -import com.launchdarkly.sdk.android.FDv2DataSourceConditions.Conditions; -import com.launchdarkly.sdk.android.FDv2DataSourceConditions.FallbackCondition; -import com.launchdarkly.sdk.android.FDv2DataSourceConditions.RecoveryCondition; -import com.launchdarkly.sdk.android.subsystems.FDv2SourceResult; -import com.launchdarkly.sdk.fdv2.ChangeSet; -import com.launchdarkly.sdk.fdv2.ChangeSetType; -import com.launchdarkly.sdk.fdv2.Selector; - -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.Timeout; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; - -/** - * Unit tests for {@link FDv2DataSourceConditions}. - */ -public class FDv2DataSourceConditionsTest { - - @Rule - public Timeout globalTimeout = Timeout.seconds(5); - - private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - - @After - public void tearDown() { - executor.shutdownNow(); - } - - // ---- helpers ---- - - private static FDv2SourceResult interrupted() { - return FDv2SourceResult.status(FDv2SourceResult.Status.interrupted(new RuntimeException("interrupted")), false); - } - - private static FDv2SourceResult terminalError() { - return FDv2SourceResult.status(FDv2SourceResult.Status.terminalError(new RuntimeException("terminal")), false); - } - - private static FDv2SourceResult changeSet() { - return FDv2SourceResult.changeSet( - new ChangeSet<>(ChangeSetType.None, Selector.EMPTY, new HashMap<>(), null, true), false); - } - - // ==== FallbackCondition ==== - - @Test - public void fallback_interruptedStartsTimer_timerFiresAsFallback() throws Exception { - FallbackCondition condition = new FallbackCondition(executor, 0); - LDAwaitFuture future = condition.getFuture(); - - condition.inform(interrupted()); - - // Timer fires with delay=0; wait up to 500ms for it. - ConditionType type = future.get(500, TimeUnit.MILLISECONDS); - assertEquals(ConditionType.FALLBACK, type); - } - - @Test - public void fallback_changeSetBeforeTimerFires_cancelsTimer() throws InterruptedException, TimeoutException { - // Use a long timeout so the timer never fires during the test. - FallbackCondition condition = new FallbackCondition(executor, 60); - LDAwaitFuture future = condition.getFuture(); - - condition.inform(interrupted()); // starts timer - condition.inform(changeSet()); // cancels timer - - // Future must NOT complete within 100ms. - try { - future.get(100, TimeUnit.MILLISECONDS); - fail("expected TimeoutException — future should not have completed"); - } catch (ExecutionException e) { - fail("unexpected exception: " + e); - } catch (TimeoutException expected) { - } - } - - @Test - public void fallback_multipleInterrupteds_dontStartMultipleTimers() throws Exception { - // The timer is started only on the first INTERRUPTED; subsequent ones are no-ops. - // We use a very short timeout so the timer fires quickly, then verify the future - // completes exactly once. - FallbackCondition condition = new FallbackCondition(executor, 0); - LDAwaitFuture future = condition.getFuture(); - - condition.inform(interrupted()); - condition.inform(interrupted()); - condition.inform(interrupted()); - - ConditionType type = future.get(500, TimeUnit.MILLISECONDS); - assertEquals(ConditionType.FALLBACK, type); - } - - @Test - public void fallback_close_cancelsInFlightTimer() throws InterruptedException, TimeoutException { - FallbackCondition condition = new FallbackCondition(executor, 60); - LDAwaitFuture future = condition.getFuture(); - - condition.inform(interrupted()); // starts timer - condition.close(); // cancels it - - try { - future.get(100, TimeUnit.MILLISECONDS); - fail("expected TimeoutException — future should not have completed after close"); - } catch (ExecutionException e) { - fail("unexpected exception: " + e); - } catch (TimeoutException expected) { - } - } - - @Test - public void fallback_terminalError_doesNotStartTimer() throws InterruptedException, TimeoutException { - FallbackCondition condition = new FallbackCondition(executor, 0); - LDAwaitFuture future = condition.getFuture(); - - condition.inform(terminalError()); // should NOT start timer - - try { - future.get(100, TimeUnit.MILLISECONDS); - fail("expected TimeoutException — terminal error must not start fallback timer"); - } catch (ExecutionException e) { - fail("unexpected exception: " + e); - } catch (TimeoutException expected) { - } - } - - @Test - public void fallback_informWithNonInterruptedStatus_doesNotStartTimer() throws InterruptedException, TimeoutException { - FallbackCondition condition = new FallbackCondition(executor, 0); - LDAwaitFuture future = condition.getFuture(); - - condition.inform(FDv2SourceResult.status(FDv2SourceResult.Status.goodbye("bye"), false)); - - try { - future.get(100, TimeUnit.MILLISECONDS); - fail("expected TimeoutException — goodbye must not start fallback timer"); - } catch (ExecutionException e) { - fail("unexpected exception: " + e); - } catch (TimeoutException expected) { - } - } - - // ==== RecoveryCondition ==== - - @Test - public void recovery_timerStartsImmediately_andFiresAsRecovery() throws Exception { - RecoveryCondition condition = new RecoveryCondition(executor, 0); - LDAwaitFuture future = condition.getFuture(); - - ConditionType type = future.get(500, TimeUnit.MILLISECONDS); - assertEquals(ConditionType.RECOVERY, type); - } - - @Test - public void recovery_close_cancelsTimer() throws InterruptedException, TimeoutException { - RecoveryCondition condition = new RecoveryCondition(executor, 60); - LDAwaitFuture future = condition.getFuture(); - - condition.close(); - - try { - future.get(100, TimeUnit.MILLISECONDS); - fail("expected TimeoutException — future should not complete after close"); - } catch (ExecutionException e) { - fail("unexpected exception: " + e); - } catch (TimeoutException expected) { - } - } - - @Test - public void recovery_informDoesNothing() throws Exception { - // inform() on RecoveryCondition is a no-op regardless of the result. - RecoveryCondition condition = new RecoveryCondition(executor, 0); - condition.inform(interrupted()); - condition.inform(changeSet()); - condition.inform(terminalError()); - - // The future should still fire (timer was not affected by inform calls). - ConditionType type = condition.getFuture().get(500, TimeUnit.MILLISECONDS); - assertEquals(ConditionType.RECOVERY, type); - } - - // ==== Conditions (wrapper) ==== - - @Test - public void conditions_empty_neverCompletes() throws InterruptedException, TimeoutException { - Conditions conditions = new Conditions(Collections.emptyList()); - Future future = conditions.getFuture(); - - assertFalse(future.isDone()); - try { - future.get(100, TimeUnit.MILLISECONDS); - fail("expected TimeoutException"); - } catch (ExecutionException e) { - fail("unexpected exception: " + e); - } catch (TimeoutException expected) { - } - } - - @Test - public void conditions_firstToFire_completesConditionsFuture() throws Exception { - FallbackCondition fallback = new FallbackCondition(executor, 0); - RecoveryCondition recovery = new RecoveryCondition(executor, 60); // long timeout - - Conditions conditions = new Conditions(Arrays.asList(fallback, recovery)); - - // Trigger the fallback timer. - fallback.inform(interrupted()); - - // The conditions future should complete with FALLBACK. - Object result = conditions.getFuture().get(500, TimeUnit.MILLISECONDS); - assertEquals(ConditionType.FALLBACK, result); - - conditions.close(); - } - - @Test - public void conditions_close_closesAllUnderlyingConditions() throws InterruptedException, TimeoutException { - FallbackCondition fallback = new FallbackCondition(executor, 60); - RecoveryCondition recovery = new RecoveryCondition(executor, 60); - - Conditions conditions = new Conditions(Arrays.asList(fallback, recovery)); - fallback.inform(interrupted()); // starts fallback timer - - conditions.close(); // must cancel both timers - - try { - conditions.getFuture().get(100, TimeUnit.MILLISECONDS); - fail("expected TimeoutException — close() must cancel all timers"); - } catch (ExecutionException e) { - fail("unexpected exception: " + e); - } catch (TimeoutException expected) { - } - } - - @Test - public void conditions_inform_forwardedToAllConditions() throws Exception { - // FallbackCondition should start its timer when Conditions.inform(interrupted) is called. - FallbackCondition fallback = new FallbackCondition(executor, 0); - Conditions conditions = new Conditions(Collections.singletonList(fallback)); - - conditions.inform(interrupted()); - - Object result = conditions.getFuture().get(500, TimeUnit.MILLISECONDS); - assertEquals(ConditionType.FALLBACK, result); - } -} diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java index 33ecfc61a..1efbf9caa 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java @@ -90,8 +90,8 @@ private FDv2DataSource buildDataSource( MockComponents.MockDataSourceUpdateSink sink, List> initializers, List> synchronizers, - long fallbackTimeoutSeconds, - long recoveryTimeoutSeconds) { + long fallbackTimeoutMillis, + long recoveryTimeoutMillis) { return new FDv2DataSource( CONTEXT, initializers, @@ -100,8 +100,8 @@ private FDv2DataSource buildDataSource( sink, executor, logging.logger, - fallbackTimeoutSeconds, - recoveryTimeoutSeconds); + fallbackTimeoutMillis, + recoveryTimeoutMillis); } /** Starts the data source and returns a callback that will receive the start result. */ @@ -539,7 +539,7 @@ public void fallbackAndRecoveryTasksWellBehaved() throws Exception { return new MockQueuedSynchronizer( FDv2SourceResult.changeSet(makeChangeSet(false), false)); }), - 1, 2); + 1000, 2000); AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); @@ -569,7 +569,7 @@ public void canDisposeWhenSynchronizersFallingBack() throws Exception { Collections.singletonList(() -> new MockQueuedSynchronizer( FDv2SourceResult.changeSet(makeChangeSet(false), false), interrupted())), - 1, 2); + 1000, 2000); AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); @@ -687,7 +687,7 @@ public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { return new MockQueuedSynchronizer( FDv2SourceResult.changeSet(makeChangeSet(false), false)); }), - 1, 2); + 1000, 2000); AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); @@ -713,7 +713,7 @@ public void fallbackMovesToNextSynchronizer() throws Exception { interrupted()), () -> { secondCalledQueue.offer(true); return new MockQueuedSynchronizer( FDv2SourceResult.changeSet(makeChangeSet(false), false)); }), - 1, 300); + 1000, 300_000); AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); @@ -724,6 +724,42 @@ public void fallbackMovesToNextSynchronizer() throws Exception { stopDataSource(dataSource); } + @Test + public void changeSetDuringInterruptionRestartsFallbackTimeout() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + MockQueuedSynchronizer primary = new MockQueuedSynchronizer(interrupted()); + BlockingQueue secondaryBuildTimes = new LinkedBlockingQueue<>(); + + // The primary synchronizer reports that it is interrupted. The secondary records when it is built. + // The fallback timeout is 100ms. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Arrays.asList( + () -> primary, + () -> { + secondaryBuildTimes.offer(System.nanoTime()); + return new MockQueuedSynchronizer(); + }), + 100, 300_000); + + // Start the data source. Before the fallback timeout, the primary delivers a change set and is + // interrupted again. + long start = System.nanoTime(); + startDataSource(dataSource); + executor.schedule( + () -> primary.addResult(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 50, + TimeUnit.MILLISECONDS); + executor.schedule(() -> primary.addResult(interrupted()), 90, TimeUnit.MILLISECONDS); + + // The data source falls back a full timeout after the second interruption, not at the original deadline. + Long secondaryBuild = secondaryBuildTimes.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertNotNull(secondaryBuild); + long fallbackMillis = TimeUnit.NANOSECONDS.toMillis(secondaryBuild - start); + assertTrue("fell back after " + fallbackMillis + "ms", fallbackMillis >= 150); + stopDataSource(dataSource); + } + // ============================================================================ // Stop / Close Lifecycle // ============================================================================ @@ -856,7 +892,7 @@ public void stopInterruptsConditionWaiting() throws Exception { Collections.singletonList(() -> new MockQueuedSynchronizer( FDv2SourceResult.changeSet(makeChangeSet(false), false), interrupted())), - 120, 300); + 120_000, 300_000); AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); @@ -1197,26 +1233,9 @@ public LDAwaitFuture run() { } // ============================================================================ - // Condition Lifecycle + // Fallback and Recovery Conditions // ============================================================================ - @Test - public void conditionsClosedAfterSynchronizerLoop() throws Exception { - MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); - - FDv2DataSource dataSource = buildDataSource(sink, - Collections.emptyList(), - Collections.singletonList(() -> new MockQueuedSynchronizer( - FDv2SourceResult.changeSet(makeChangeSet(false), false), - terminalError())), - 1, 2); - - AwaitableCallback startCallback = startDataSource(dataSource); - assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); // changeset arrives first, so start succeeds - - stopDataSource(dataSource); // should complete cleanly if conditions were closed - } - @Test public void conditionsInformedOfAllResults() throws Exception { MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); @@ -1227,7 +1246,7 @@ public void conditionsInformedOfAllResults() throws Exception { FDv2SourceResult.changeSet(makeChangeSet(false), false), interrupted(), FDv2SourceResult.changeSet(makeChangeSet(false), false))), - 10, 20); + 10_000, 20_000); AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); @@ -1237,23 +1256,6 @@ public void conditionsInformedOfAllResults() throws Exception { stopDataSource(dataSource); } - @Test - public void conditionsClosedOnException() throws Exception { - MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); - - FDv2DataSource dataSource = buildDataSource(sink, - Collections.emptyList(), - Arrays.asList( - () -> new MockSynchronizer(new RuntimeException("error")), - () -> new MockSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(false), false))), - 1, 2); - - AwaitableCallback startCallback = startDataSource(dataSource); - assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); - // If conditions were not closed on exception, resources would leak - stopDataSource(dataSource); - } - @Test public void primeSynchronizerHasNoRecoveryCondition() throws Exception { // The prime synchronizer (first available) gets only a FallbackCondition, not a RecoveryCondition. @@ -1269,7 +1271,7 @@ public void primeSynchronizerHasNoRecoveryCondition() throws Exception { Arrays.asList( () -> firstSync, () -> new MockQueuedSynchronizer()), - 1, 2); + 1000, 2000); AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); @@ -1292,7 +1294,7 @@ public void singleSynchronizerHasNoConditions() throws Exception { Collections.singletonList(() -> new MockQueuedSynchronizer( FDv2SourceResult.changeSet(makeChangeSet(false), false), FDv2SourceResult.changeSet(makeChangeSet(false), false))), - 1, 2); + 1000, 2000); AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); @@ -1311,7 +1313,7 @@ public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { Collections.singletonList(() -> new MockQueuedSynchronizer( FDv2SourceResult.changeSet(makeChangeSet(false), false), FDv2SourceResult.changeSet(makeChangeSet(false), false))), - 1, 2); + 1000, 2000); AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); @@ -2026,7 +2028,7 @@ public void orchestrationLogging_fallback_logsInfo() throws Exception { return new MockQueuedSynchronizer( FDv2SourceResult.changeSet(makeChangeSet(false), false)); }), - 1, 300); + 1000, 300_000); AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); assertNotNull(secondCalledQueue.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); @@ -2046,7 +2048,7 @@ public void orchestrationLogging_recovery_logsInfo() throws Exception { interrupted()), () -> new MockQueuedSynchronizer( FDv2SourceResult.changeSet(makeChangeSet(false), false))), - 1, 2); + 1000, 2000); AwaitableCallback startCallback = startDataSource(dataSource); try { assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/MonotonicTimeTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/MonotonicTimeTest.java new file mode 100644 index 000000000..1195e86f0 --- /dev/null +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/MonotonicTimeTest.java @@ -0,0 +1,35 @@ +package com.launchdarkly.sdk.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import java.util.concurrent.TimeUnit; + +public class MonotonicTimeTest { + @Test + public void timeAfterOverflowIsAfterTimeBeforeOverflow() { + MonotonicTime beforeOverflow = MonotonicTime.ofNanos(Long.MAX_VALUE); + + // Move one second past the point where the nanosecond count overflows. + MonotonicTime afterOverflow = beforeOverflow.plusMillis(1000); + + // The later time compares as later. + assertTrue(beforeOverflow.isBefore(afterOverflow)); + assertFalse(afterOverflow.isBefore(beforeOverflow)); + } + + @Test + public void nanosUntilMeasuresDurationAcrossOverflow() { + MonotonicTime beforeOverflow = MonotonicTime.ofNanos(Long.MAX_VALUE); + + // Move one second past the point where the nanosecond count overflows. + MonotonicTime afterOverflow = beforeOverflow.plusMillis(1000); + + // The duration is one second forward, and negative one second back. + assertEquals(TimeUnit.SECONDS.toNanos(1), beforeOverflow.nanosUntil(afterOverflow)); + assertEquals(-TimeUnit.SECONDS.toNanos(1), afterOverflow.nanosUntil(beforeOverflow)); + } +} From bbf90b22744eb6afa052ce139f27284ef426f2ae Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Fri, 6 Nov 2026 14:00:33 -0800 Subject: [PATCH 02/12] feat: Park failed FDv2 synchronizers and retry them after a backoff --- .../sdk/android/FDv2DataSource.java | 194 +++++++- .../sdk/android/SourceManager.java | 68 ++- .../android/SynchronizerFactoryWithState.java | 60 ++- .../sdk/android/FDv2DataSourceTest.java | 418 ++++++++++++++---- 4 files changed, 632 insertions(+), 108 deletions(-) diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java index a2026ccab..47a5e4f65 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java @@ -27,6 +27,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -65,10 +66,28 @@ public interface DataSourceFactory { */ private static final long DEFAULT_RECOVERY_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(5); + /** + * Default base of the park delay that follows an unexpected failure. + */ + private static final long DEFAULT_PARK_INITIAL_DELAY_MILLIS = TimeUnit.MINUTES.toMillis(5); + + /** + * Default maximum of the park delay. + */ + private static final long DEFAULT_PARK_MAX_DELAY_MILLIS = TimeUnit.HOURS.toMillis(1); + + /** + * Default base of the failover delay that follows a park. + */ + private static final long DEFAULT_FAILOVER_DELAY_MILLIS = TimeUnit.SECONDS.toMillis(1); + private final List> cacheInitializers; private final SourceManager sourceManager; private final long fallbackTimeoutMillis; private final long recoveryTimeoutMillis; + private final long parkInitialDelayMillis; + private final long parkMaxDelayMillis; + private final long failoverDelayMillis; private final ScheduledExecutorService sharedExecutor; private final AtomicBoolean startCalled = new AtomicBoolean(false); @@ -90,9 +109,9 @@ public interface DataSourceFactory { // This future is set by either the worker thread terminating or stop() being called. private final LDAwaitFuture shutdownCause = new LDAwaitFuture<>(); /** - * Convenience constructor using default fallback and recovery timeouts. + * Convenience constructor using default timeouts and delays. * See {@link #FDv2DataSource(LDContext, List, List, DataSourceFactory, DataSourceUpdateSinkV2, - * ScheduledExecutorService, LDLogger, long, long)} for parameter documentation. + * ScheduledExecutorService, LDLogger, long, long, long, long, long)} for parameter documentation. */ FDv2DataSource( @NonNull LDContext evaluationContext, @@ -106,7 +125,10 @@ public interface DataSourceFactory { this(evaluationContext, initializers, synchronizers, fdv1FallbackSynchronizer, dataSourceUpdateSink, sharedExecutor, logger, DEFAULT_FALLBACK_TIMEOUT_MILLIS, - DEFAULT_RECOVERY_TIMEOUT_MILLIS); + DEFAULT_RECOVERY_TIMEOUT_MILLIS, + DEFAULT_PARK_INITIAL_DELAY_MILLIS, + DEFAULT_PARK_MAX_DELAY_MILLIS, + DEFAULT_FAILOVER_DELAY_MILLIS); } /** @@ -125,6 +147,9 @@ public interface DataSourceFactory { * next synchronizer * @param recoveryTimeoutMillis milliseconds before attempting to recover to the primary * synchronizer + * @param parkInitialDelayMillis base of the park delay, in milliseconds + * @param parkMaxDelayMillis maximum of the park delay, in milliseconds + * @param failoverDelayMillis base of the failover delay, in milliseconds */ FDv2DataSource( @NonNull LDContext evaluationContext, @@ -135,7 +160,10 @@ public interface DataSourceFactory { @NonNull ScheduledExecutorService sharedExecutor, @NonNull LDLogger logger, long fallbackTimeoutMillis, - long recoveryTimeoutMillis + long recoveryTimeoutMillis, + long parkInitialDelayMillis, + long parkMaxDelayMillis, + long failoverDelayMillis ) { this.evaluationContext = evaluationContext; this.dataSourceUpdateSink = dataSourceUpdateSink; @@ -163,6 +191,9 @@ public interface DataSourceFactory { this.sourceManager = new SourceManager(allSynchronizers, generalInitializers); this.fallbackTimeoutMillis = fallbackTimeoutMillis; this.recoveryTimeoutMillis = recoveryTimeoutMillis; + this.parkInitialDelayMillis = parkInitialDelayMillis; + this.parkMaxDelayMillis = parkMaxDelayMillis; + this.failoverDelayMillis = failoverDelayMillis; this.sharedExecutor = sharedExecutor; } @@ -521,6 +552,118 @@ private static MonotonicTime earliest(MonotonicTime... times) { return earliest; } + /** + * Waits for the given time, or until the data source is stopped. + * + * @param millis the time to wait, in milliseconds + * @return true if the full time passed, or false if the data source was stopped + */ + private boolean waitUnlessStopped(long millis) { + try { + shutdownCause.get(Math.max(0, millis), TimeUnit.MILLISECONDS); + return false; + } catch (TimeoutException e) { + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (ExecutionException e) { + return false; + } + } + + /** + * Subtracts a random jitter of up to half of the given delay. + * + * @param delayMillis the delay before jitter, in milliseconds + * @return the delay after jitter, in milliseconds + */ + private static long jittered(long delayMillis) { + return delayMillis - ThreadLocalRandom.current().nextLong(delayMillis / 2 + 1); + } + + /** + * Computes the park delay for a synchronizer. + * + * @param parkCount the park count of the synchronizer, including the current failure + * @return the park delay, in milliseconds + */ + private long parkDelayMillis(int parkCount) { + double delayMillis = Math.min( + parkInitialDelayMillis * Math.pow(2, parkCount - 1), + parkMaxDelayMillis); + return jittered((long) delayMillis); + } + + /** + * Parks the active synchronizer after an unexpected failure. + * + * @param synchronizerName the name of the active synchronizer, or null if it failed to build + * @param error the error that caused the park + */ + private void parkActiveSynchronizer(@Nullable String synchronizerName, @Nullable Throwable error) { + long parkDelayMillis = parkDelayMillis(sourceManager.incrementCurrentSynchronizerParkCount()); + sourceManager.parkCurrentSynchronizer(MonotonicTime.now().plusMillis(parkDelayMillis)); + logger.warn( + "{} is backing off for {}s after an unexpected error ({}).", + synchronizerName != null ? "Synchronizer '" + synchronizerName + "'" + : "A synchronizer that failed to build", + TimeUnit.MILLISECONDS.toSeconds(parkDelayMillis), + detailForThrowable(error)); + } + + /** + * Gets the next available synchronizer, builds it, and sets it active. While no synchronizer is available, this + * waits for the earliest unpark time. A synchronizer that fails to build is parked, and the next available + * synchronizer is tried after the failover delay. + * + * @param sink the sink for status updates + * @return the active synchronizer, or null if the data source was stopped + */ + @Nullable + private Synchronizer buildNextAvailableSynchronizer(@NonNull DataSourceUpdateSinkV2 sink) { + while (!stopCalled.get()) { + Synchronizer synchronizer; + try { + synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); + } catch (RuntimeException e) { + sink.setStatus(DataSourceState.INTERRUPTED, e); + parkActiveSynchronizer(null, e); + if (!waitUnlessStopped(jittered(failoverDelayMillis))) { + return null; + } + continue; + } + if (synchronizer != null) { + return synchronizer; + } + if (stopCalled.get()) { + return null; + } + + MonotonicTime earliestUnparkTime = sourceManager.getEarliestUnparkTime(); + long waitMillis; + if (earliestUnparkTime == null) { + // This should never happen. + waitMillis = parkInitialDelayMillis; + logger.error("No synchronizer can be started; checking again in {}s.", + TimeUnit.MILLISECONDS.toSeconds(waitMillis)); + } else { + waitMillis = TimeUnit.NANOSECONDS.toMillis(MonotonicTime.now().nanosUntil(earliestUnparkTime)); + logger.warn("All synchronizers are backing off for at least {}s.", + Math.max(0, TimeUnit.MILLISECONDS.toSeconds(waitMillis))); + } + // No synchronizer is running until the wait ends. + sink.setStatus(DataSourceState.INTERRUPTED, null); + if (!waitUnlessStopped(waitMillis)) { + return null; + } + // Start the most-preferred synchronizer that is no longer parked. + sourceManager.resetSourceIndex(); + } + return null; + } + private static String detailForThrowable(@Nullable Throwable error) { if (error == null) { return "unknown error"; @@ -556,11 +699,12 @@ private void runSynchronizers( @NonNull DataSourceUpdateSinkV2 sink ) { try { - Synchronizer synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); + Synchronizer synchronizer = buildNextAvailableSynchronizer(sink); while (synchronizer != null) { String synchronizerName = synchronizer.name(); logger.info("Synchronizer '{}' is starting.", synchronizerName); resetSynchronizerStatusDedupe(); + boolean parked = false; try { boolean running = true; // Fallback and recovery apply only when another synchronizer can take over. @@ -614,21 +758,24 @@ private void runSynchronizers( synchronizer.name(), status.getState() ); - logger.debug("Synchronizer shutdown."); - // Android: advance to the next synchronizer (differs from server, - // which exits the synchronizer phase on SHUTDOWN). + if (stopCalled.get()) { + // Stopping the data source shut down the synchronizer. + logger.debug("Synchronizer shutdown."); + return; + } + // The synchronizer shut down by itself, so park it. + parkActiveSynchronizer(synchronizer.name(), status.getError()); + parked = true; running = false; + sink.setStatus(DataSourceState.INTERRUPTED, status.getError()); break; case TERMINAL_ERROR: maybeLogSynchronizerStatusChange( synchronizer.name(), status.getState() ); - sourceManager.blockCurrentSynchronizer(); - logger.warn( - "Synchronizer '{}' permanently failed and will not be used again until application restart.", - synchronizer.name() - ); + parkActiveSynchronizer(synchronizer.name(), status.getError()); + parked = true; running = false; sink.setStatus(DataSourceState.INTERRUPTED, status.getError()); break; @@ -653,6 +800,8 @@ private void runSynchronizers( && !sourceManager.isCurrentSynchronizerFDv1Fallback()) { logger.info(FDV1_FALLBACK_MESSAGE); sourceManager.fdv1Fallback(); + // The directive takes over, so there is no failover delay after a park. + parked = false; running = false; } } @@ -679,8 +828,8 @@ private void runSynchronizers( break; } } - } catch (ExecutionException e) { - Throwable failure = e.getCause() != null ? e.getCause() : e; + } catch (ExecutionException | RuntimeException e) { + Throwable failure = e instanceof ExecutionException && e.getCause() != null ? e.getCause() : e; sink.setStatus(DataSourceState.INTERRUPTED, failure); if (!stopCalled.get() && !(failure instanceof CancellationException) @@ -691,17 +840,20 @@ private void runSynchronizers( failure.getMessage() != null ? failure.getMessage() : LogValues.exceptionSummary(failure) ); } - } catch (CancellationException e) { - sink.setStatus(DataSourceState.INTERRUPTED, e); + parkActiveSynchronizer(synchronizer.name(), failure); + parked = true; } catch (InterruptedException e) { sink.setStatus(DataSourceState.INTERRUPTED, e); Thread.currentThread().interrupt(); return; } - synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); - } - if (!stopCalled.get()) { - logger.warn("No more synchronizers available."); + // The run of this synchronizer has ended, so stop it. + sourceManager.closeActiveSource(); + + if (parked && !waitUnlessStopped(jittered(failoverDelayMillis))) { + break; + } + synchronizer = buildNextAvailableSynchronizer(sink); } } catch (Exception e) { logger.error("Unexpected error in data source: {}", e.toString()); diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java index bbbd8adb6..cb5090a74 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java @@ -1,6 +1,7 @@ package com.launchdarkly.sdk.android; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import com.launchdarkly.sdk.android.subsystems.Initializer; import com.launchdarkly.sdk.android.subsystems.Synchronizer; @@ -77,8 +78,21 @@ void fdv1Fallback() { } } + /** + * Stops parking every synchronizer whose unpark time has arrived. + */ + void unparkExpiredSynchronizers() { + synchronized (activeSourceLock) { + MonotonicTime now = MonotonicTime.now(); + for (SynchronizerFactoryWithState s : synchronizerFactories) { + s.unparkIfExpired(now); + } + } + } + private SynchronizerFactoryWithState getNextAvailableSynchronizer() { SynchronizerFactoryWithState candidate = null; + unparkExpiredSynchronizers(); int visited = 0; while (visited < synchronizerFactories.size()) { synchronizerIndex++; @@ -86,7 +100,7 @@ private SynchronizerFactoryWithState getNextAvailableSynchronizer() { synchronizerIndex = 0; } SynchronizerFactoryWithState c = synchronizerFactories.get(synchronizerIndex); - if (c.getState() == SynchronizerFactoryWithState.State.Available) { + if (c.isEligible()) { candidate = c; break; } @@ -115,11 +129,12 @@ Synchronizer getNextAvailableSynchronizerAndSetActive() { return null; } tried++; + // Set before the build, so that a build that throws leaves this synchronizer current. + currentSynchronizerFactory = factoryWithState; Synchronizer synchronizer = factoryWithState.build(); if (synchronizer == null) { continue; } - currentSynchronizerFactory = factoryWithState; if (activeSource != null) { safeClose(activeSource); } @@ -151,12 +166,41 @@ private FDv2DataSource.DataSourceFactory getNextInitializer() { return initializers.get(initializerIndex); } - /** Block the current synchronizer so it will not be returned again (e.g. after TERMINAL_ERROR). */ - void blockCurrentSynchronizer() { + /** + * Records an unexpected failure of the current synchronizer. + * + * @return the park count of the current synchronizer, including this failure + */ + int incrementCurrentSynchronizerParkCount() { + synchronized (activeSourceLock) { + return currentSynchronizerFactory.incrementParkCount(); + } + } + + void parkCurrentSynchronizer(@NonNull MonotonicTime unparkTime) { + synchronized (activeSourceLock) { + currentSynchronizerFactory.park(unparkTime); + } + } + + /** + * The earliest unpark time of the available synchronizers. + * + * @return the earliest unpark time, or null if no available synchronizer is parked + */ + @Nullable + MonotonicTime getEarliestUnparkTime() { synchronized (activeSourceLock) { - if (currentSynchronizerFactory != null) { - currentSynchronizerFactory.block(); + MonotonicTime earliest = null; + for (SynchronizerFactoryWithState s : synchronizerFactories) { + MonotonicTime unparkTime = s.getUnparkTime(); + if (s.getState() == SynchronizerFactoryWithState.State.Available + && unparkTime != null + && (earliest == null || unparkTime.isBefore(earliest))) { + earliest = unparkTime; + } } + return earliest; } } @@ -217,6 +261,18 @@ int getAvailableSynchronizerCount() { } } + /** + * Closes the active source, if there is one. + */ + void closeActiveSource() { + synchronized (activeSourceLock) { + if (activeSource != null) { + safeClose(activeSource); + activeSource = null; + } + } + } + @Override public void close() { synchronized (activeSourceLock) { diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SynchronizerFactoryWithState.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SynchronizerFactoryWithState.java index 8afcb6589..25d3248ec 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SynchronizerFactoryWithState.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SynchronizerFactoryWithState.java @@ -1,12 +1,13 @@ package com.launchdarkly.sdk.android; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import com.launchdarkly.sdk.android.subsystems.Synchronizer; /** - * Wraps a synchronizer factory with availability state (available/blocked). - * Used by {@link SourceManager} to skip synchronizers that have been blocked (e.g. after TERMINAL_ERROR). + * Wraps a synchronizer factory with availability state (available/blocked) and park state. + * Used by {@link SourceManager} to skip synchronizers that are blocked or parked. *

* Package-private for internal use by FDv2DataSource. */ @@ -15,7 +16,7 @@ final class SynchronizerFactoryWithState { enum State { /** This synchronizer is available to use. */ Available, - /** This synchronizer is no longer available (e.g. after TERMINAL_ERROR). */ + /** This synchronizer is no longer available to use. */ Blocked } @@ -23,6 +24,16 @@ enum State { private State state = State.Available; private final boolean isFDv1Fallback; + /** + * The number of unexpected failures of the synchronizer. + */ + private int parkCount = 0; + + /** + * The time at which the synchronizer stops being parked, or null if it is not parked. + */ + private MonotonicTime unparkTime = null; + SynchronizerFactoryWithState(@NonNull FDv2DataSource.DataSourceFactory factory) { this(factory, false); } @@ -51,4 +62,47 @@ Synchronizer build() { boolean isFDv1Fallback() { return isFDv1Fallback; } + + /** + * Records an unexpected failure. + * + * @return the park count, including this failure + */ + int incrementParkCount() { + return ++parkCount; + } + + void park(@NonNull MonotonicTime unparkTime) { + this.unparkTime = unparkTime; + } + + /** + * The time at which the synchronizer stops being parked. + * + * @return the unpark time, or null if the synchronizer is not parked + */ + @Nullable + MonotonicTime getUnparkTime() { + return unparkTime; + } + + /** + * Stops parking the synchronizer if its unpark time has arrived. + * + * @param now the current time + */ + void unparkIfExpired(@NonNull MonotonicTime now) { + if (unparkTime != null && !now.isBefore(unparkTime)) { + unparkTime = null; + } + } + + /** + * True if the synchronizer is available and is not parked. + * + * @return true if the synchronizer can be started + */ + boolean isEligible() { + return state == State.Available && unparkTime == null; + } } diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java index 1efbf9caa..144e9b118 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java @@ -4,6 +4,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -72,36 +73,56 @@ public void tearDown() { } } + /** + * Builds a data source with production timeouts and park delays, and a 10ms failover delay. + */ private FDv2DataSource buildDataSource( MockComponents.MockDataSourceUpdateSink sink, List> initializers, List> synchronizers) { - return new FDv2DataSource( - CONTEXT, - initializers, - synchronizers, - null, - sink, - executor, - logging.logger); + return buildDataSource(sink, initializers, synchronizers, 120_000, 300_000); } + /** + * Builds a data source with production park delays, and a 10ms failover delay. + */ private FDv2DataSource buildDataSource( MockComponents.MockDataSourceUpdateSink sink, List> initializers, List> synchronizers, long fallbackTimeoutMillis, long recoveryTimeoutMillis) { + return buildDataSource(sink, initializers, synchronizers, null, + fallbackTimeoutMillis, + recoveryTimeoutMillis, + /* parkInitialDelayMillis= */ TimeUnit.MINUTES.toMillis(5), + /* parkMaxDelayMillis= */ TimeUnit.HOURS.toMillis(1), + /* failoverDelayMillis= */ 10); + } + + private FDv2DataSource buildDataSource( + MockComponents.MockDataSourceUpdateSink sink, + List> initializers, + List> synchronizers, + FDv2DataSource.DataSourceFactory fdv1FallbackSynchronizer, + long fallbackTimeoutMillis, + long recoveryTimeoutMillis, + long parkInitialDelayMillis, + long parkMaxDelayMillis, + long failoverDelayMillis) { return new FDv2DataSource( CONTEXT, initializers, synchronizers, - null, + fdv1FallbackSynchronizer, sink, executor, logging.logger, fallbackTimeoutMillis, - recoveryTimeoutMillis); + recoveryTimeoutMillis, + parkInitialDelayMillis, + parkMaxDelayMillis, + failoverDelayMillis); } /** Starts the data source and returns a callback that will receive the start result. */ @@ -577,8 +598,12 @@ public void canDisposeWhenSynchronizersFallingBack() throws Exception { stopDataSource(dataSource); // should not hang while fallback condition is waiting } + // ============================================================================ + // Source Parking + // ============================================================================ + @Test - public void terminalErrorBlocksSynchronizer() throws Exception { + public void terminalErrorFailsOverToNextSynchronizer() throws Exception { MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); BlockingQueue callOrder = new LinkedBlockingQueue<>(); @@ -607,61 +632,247 @@ public void terminalErrorBlocksSynchronizer() throws Exception { } @Test - public void allThreeSynchronizersFailReportsExhaustion() throws Exception { + public void terminalErrorParksSynchronizerAndRestartsItAfterUnparkTime() throws Exception { MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + // The first instance fails with a terminal error, and the second instance delivers data. FDv2DataSource dataSource = buildDataSource(sink, Collections.emptyList(), - Arrays.asList( - () -> new MockQueuedSynchronizer(terminalError()), - () -> new MockQueuedSynchronizer(terminalError()), - () -> new MockQueuedSynchronizer(terminalError()))); + Collections.singletonList(() -> buildCount.incrementAndGet() == 1 + ? new MockQueuedSynchronizer(terminalError()) + : new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false))), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ 100, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); + // Start the data source with its only synchronizer. AwaitableCallback startCallback = startDataSource(dataSource); - awaitExpectingError(startCallback); + assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); - List statuses = sink.awaitStatuses(4, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertEquals(4, statuses.size()); - assertEquals(DataSourceState.INTERRUPTED, statuses.get(0)); - assertEquals(DataSourceState.INTERRUPTED, statuses.get(1)); - assertEquals(DataSourceState.INTERRUPTED, statuses.get(2)); - assertEquals(DataSourceState.OFF, statuses.get(3)); - assertNotNull(sink.getLastError()); + // The parked synchronizer is built again after its unpark time, and its data initializes the data source. + assertEquals(2, buildCount.get()); + assertEquals(DataSourceState.VALID, sink.getLastState()); + stopDataSource(dataSource); } @Test - public void blockedSynchronizerSkippedInRotation() throws Exception { + public void failedNextParksSynchronizer() throws Exception { MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); - AtomicInteger firstCallCount = new AtomicInteger(0); - AtomicInteger secondCallCount = new AtomicInteger(0); + AtomicInteger buildCount = new AtomicInteger(); + // The first instance fails from next(), and the second instance delivers data. FDv2DataSource dataSource = buildDataSource(sink, Collections.emptyList(), - Arrays.asList( - () -> { firstCallCount.incrementAndGet(); return new MockQueuedSynchronizer(terminalError()); }, - () -> { secondCallCount.incrementAndGet(); return new MockQueuedSynchronizer( - FDv2SourceResult.changeSet(makeChangeSet(false), false)); })); + Collections.singletonList(() -> buildCount.incrementAndGet() == 1 + ? new MockSynchronizer(new RuntimeException("next-boom")) + : new MockSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false))), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ 100, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); + // Start the data source with its only synchronizer. AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); - assertEquals(1, firstCallCount.get()); // called once then blocked - assertTrue(secondCallCount.get() >= 1); + // The failing synchronizer is parked, and is built again after its unpark time. + assertEquals(2, buildCount.get()); + awaitLogContains(logging, "Synchronizer 'MockSynchronizer' is backing off for "); + stopDataSource(dataSource); + } + + @Test + public void exceptionThrownFromNextParksSynchronizer() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Collections.singletonList(() -> { + if (buildCount.incrementAndGet() == 1) { + // The first instance throws from next() instead of returning a future. + return new Synchronizer() { + @Override + public LDAwaitFuture next() { + throw new RuntimeException("next-boom"); + } + + @Override + public void close() { + } + }; + } + // The second instance delivers data. + return new MockSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + }), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ 100, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); + + // Start the data source with its only synchronizer. + AwaitableCallback startCallback = startDataSource(dataSource); + assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); + + // The throwing synchronizer is parked, and is built again after its unpark time. + assertEquals(2, buildCount.get()); + stopDataSource(dataSource); + } + + @Test + public void constructionFailureParksSynchronizer() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Collections.singletonList(() -> { + if (buildCount.incrementAndGet() == 1) { + // The first build fails. + throw new RuntimeException("build-boom"); + } + // The second build delivers data. + return new MockSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + }), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ 100, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); + + // Start the data source with its only synchronizer. + AwaitableCallback startCallback = startDataSource(dataSource); + assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); + + // The synchronizer that failed to build is parked, and is built again after its unpark time. + assertEquals(2, buildCount.get()); + awaitLogContains(logging, "A synchronizer that failed to build is backing off for "); + stopDataSource(dataSource); + } + + // A synchronizer that shuts down without being stopped is parked, like one with any other unexpected failure. + @Test + public void unpromptedShutdownParksSynchronizer() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + + // The first instance shuts down, and the second instance delivers data. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Collections.singletonList(() -> buildCount.incrementAndGet() == 1 + ? new MockQueuedSynchronizer(FDv2SourceResult.status(FDv2SourceResult.Status.shutdown(), false)) + : new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false))), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ 100, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); + + // Start the data source with its only synchronizer. + AwaitableCallback startCallback = startDataSource(dataSource); + assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); + + // The synchronizer is built again after its unpark time, and its data initializes the data source. + assertEquals(2, buildCount.get()); stopDataSource(dataSource); } + // While every synchronizer is parked, the synchronizer that was parked last is stopped. @Test - public void allSynchronizersBlockedReturnsNullAndExits() throws Exception { + public void parkedSynchronizerIsClosedWhileAllSynchronizersAreParked() throws Exception { MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + CountDownLatch closed = new CountDownLatch(1); + // The only synchronizer fails with a terminal error, and records when it is closed. It is parked for much + // longer than the test. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Collections.singletonList(() -> new MockQueuedSynchronizer(terminalError()) { + @Override + public void close() { + closed.countDown(); + super.close(); + } + })); + + // Start the data source. + startDataSource(dataSource); + + // The parked synchronizer is closed while it waits for its unpark time. + assertTrue(closed.await(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + stopDataSource(dataSource); + } + + @Test + public void allSynchronizersParkedRestartsAfterEarliestUnparkTime() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + BlockingQueue buildOrder = new LinkedBlockingQueue<>(); + + // Every instance of every synchronizer fails with a terminal error. FDv2DataSource dataSource = buildDataSource(sink, Collections.emptyList(), Arrays.asList( - () -> new MockQueuedSynchronizer(terminalError()), - () -> new MockQueuedSynchronizer(terminalError()))); + () -> { buildOrder.offer(1); return new MockQueuedSynchronizer(terminalError()); }, + () -> { buildOrder.offer(2); return new MockQueuedSynchronizer(terminalError()); }, + () -> { buildOrder.offer(3); return new MockQueuedSynchronizer(terminalError()); }), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ 100, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); + + // Start the data source, and wait for each synchronizer to be built and then for one more build. + startDataSource(dataSource); + Integer firstBuild = buildOrder.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + Integer secondBuild = buildOrder.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + Integer thirdBuild = buildOrder.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + Integer buildAfterAllParked = buildOrder.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + // The synchronizers fail over in order, and a parked synchronizer starts again after all of them are parked. + assertEquals(Integer.valueOf(1), firstBuild); + assertEquals(Integer.valueOf(2), secondBuild); + assertEquals(Integer.valueOf(3), thirdBuild); + assertNotNull(buildAfterAllParked); + + // Each park reports INTERRUPTED. + List statuses = sink.awaitStatuses(3, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(Arrays.asList( + DataSourceState.INTERRUPTED, + DataSourceState.INTERRUPTED, + DataSourceState.INTERRUPTED), statuses); + stopDataSource(dataSource); + } + + @Test + public void parkedSynchronizerSkippedInRotation() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger firstCallCount = new AtomicInteger(0); + AtomicInteger secondCallCount = new AtomicInteger(0); + + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Arrays.asList( + () -> { firstCallCount.incrementAndGet(); return new MockQueuedSynchronizer(terminalError()); }, + () -> { secondCallCount.incrementAndGet(); return new MockQueuedSynchronizer( + FDv2SourceResult.changeSet(makeChangeSet(false), false)); })); AwaitableCallback startCallback = startDataSource(dataSource); - awaitExpectingError(startCallback); + assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); + + assertEquals(1, firstCallCount.get()); // called once, then parked + assertTrue(secondCallCount.get() >= 1); + stopDataSource(dataSource); } @Test @@ -1468,17 +1679,14 @@ public void statusIncludesErrorInfoOnFailure() throws Exception { Collections.singletonList(() -> new MockQueuedSynchronizer( FDv2SourceResult.status(FDv2SourceResult.Status.terminalError(terminalErr), false)))); - AwaitableCallback startCallback = startDataSource(dataSource); - awaitExpectingError(startCallback); - - DataSourceState first = sink.awaitStatus(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertEquals(DataSourceState.INTERRUPTED, first); - - DataSourceState second = sink.awaitStatus(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertEquals(DataSourceState.OFF, second); + // Start the data source with a synchronizer that fails with a terminal error. + startDataSource(dataSource); + DataSourceState status = sink.awaitStatus(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertEquals(DataSourceState.OFF, sink.getLastState()); - assertNotNull(sink.getLastError()); + // The data source reports INTERRUPTED with the error from the synchronizer. + assertEquals(DataSourceState.INTERRUPTED, status); + assertSame(terminalErr, sink.statusEvents.get(0).error); + stopDataSource(dataSource); } @Test @@ -1502,25 +1710,43 @@ public void statusRemainsValidDuringSynchronizerOperation() throws Exception { } @Test - public void statusTransitionsFromValidToOffWhenAllSynchronizersFail() throws Exception { + public void statusStaysInterruptedWhileOnlySynchronizerIsParked() throws Exception { MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); RuntimeException err = new RuntimeException("server error"); + AtomicInteger buildCount = new AtomicInteger(); + BlockingQueue builds = new LinkedBlockingQueue<>(); + // The first instance sends a changeset and then a terminal error. Later instances send nothing. FDv2DataSource dataSource = buildDataSource(sink, Collections.emptyList(), - Collections.singletonList(() -> new MockQueuedSynchronizer( - FDv2SourceResult.changeSet(makeChangeSet(false), false), - FDv2SourceResult.status(FDv2SourceResult.Status.terminalError(err), false)))); - - AwaitableCallback startCallback = startDataSource(dataSource); - assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); // changeset arrives first, so start succeeds + Collections.singletonList(() -> { + int build = buildCount.incrementAndGet(); + builds.offer(build); + return build == 1 + ? new MockQueuedSynchronizer( + FDv2SourceResult.changeSet(makeChangeSet(false), false), + FDv2SourceResult.status(FDv2SourceResult.Status.terminalError(err), false)) + : new MockQueuedSynchronizer(); + }), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ 100, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); - assertEquals(DataSourceState.VALID, sink.awaitStatus(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); - assertEquals(DataSourceState.INTERRUPTED, sink.awaitStatus(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); - assertEquals(DataSourceState.OFF, sink.awaitStatus(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + // Start the data source, and wait until the parked synchronizer is built again. + startDataSource(dataSource); + Integer firstBuild = builds.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + Integer rebuild = builds.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertEquals(DataSourceState.OFF, sink.getLastState()); - assertNotNull(sink.getLastError()); + // The status goes from VALID to INTERRUPTED, and stays INTERRUPTED through the park. + assertEquals(Integer.valueOf(1), firstBuild); + assertEquals(Integer.valueOf(2), rebuild); + List statuses = sink.awaitStatuses(2, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(Arrays.asList(DataSourceState.VALID, DataSourceState.INTERRUPTED), statuses); + assertEquals(DataSourceState.INTERRUPTED, sink.getLastState()); + stopDataSource(dataSource); } @Test @@ -1745,6 +1971,37 @@ public void fdv1FallbackNotTriggeredWhenAlreadyOnFdv1() throws Exception { stopDataSource(dataSource); } + @Test + public void fdv1FallbackSynchronizerIsParkedAndRestartedDuringDirectedFallback() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger fdv1BuildCount = new AtomicInteger(); + + // The FDv2 synchronizer reports an interruption that carries the FDv1 fallback directive. + // The first instance of the FDv1 fallback synchronizer fails, and the second instance delivers data. + // The FDv1 fallback synchronizer is parked for 50-100ms. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Collections.singletonList(() -> new MockQueuedSynchronizer( + FDv2SourceResult.status(FDv2SourceResult.Status.interrupted(new RuntimeException("500")), true))), + () -> fdv1BuildCount.incrementAndGet() == 1 + ? new MockQueuedSynchronizer(terminalError()) + : new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 120_000, + 300_000, + /* parkInitialDelayMillis= */ 100, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); + + // Start the data source. + startDataSource(dataSource); + + // The FDv1 fallback synchronizer is built again after its unpark time, and its data is applied. + sink.awaitApplyCount(1, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(1, sink.getApplyCount()); + assertEquals(2, fdv1BuildCount.get()); + stopDataSource(dataSource); + } + @Test public void fdv1FallbackNotTriggeredWhenNoFdv1SlotExists() throws Exception { MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); @@ -2081,29 +2338,34 @@ public void orchestrationLogging_fdv1Fallback_logsInfo() throws Exception { } @Test - public void orchestrationLogging_permanentFailure_logsWarn() throws Exception { + public void orchestrationLogging_logsSynchronizerParkedAndAllParked() throws Exception { MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + BlockingQueue builds = new LinkedBlockingQueue<>(); + // Every instance of the only synchronizer fails with a terminal error. Each build is recorded. FDv2DataSource dataSource = buildDataSource(sink, Collections.emptyList(), - Collections.singletonList(() -> new MockQueuedSynchronizer(terminalError()))); - AwaitableCallback startCallback = startDataSource(dataSource); - awaitExpectingError(startCallback); - awaitLogContains(logging, - "Synchronizer 'MockQueuedSynchronizer' permanently failed and will not be used again until application restart."); - } + Collections.singletonList(() -> { + builds.offer(1); + return new MockQueuedSynchronizer(terminalError()); + }), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ 100, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); - @Test - public void orchestrationLogging_allSynchronizersExhausted_logsWarn() throws Exception { - MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); - FDv2DataSource dataSource = buildDataSource(sink, - Collections.emptyList(), - Arrays.asList( - () -> new MockQueuedSynchronizer(terminalError()), - () -> new MockQueuedSynchronizer(terminalError()), - () -> new MockQueuedSynchronizer(terminalError()))); - AwaitableCallback startCallback = startDataSource(dataSource); - awaitExpectingError(startCallback); - awaitLogContains(logging, "No more synchronizers available."); + // Start the data source, and wait until the parked synchronizer is built again. + startDataSource(dataSource); + builds.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertNotNull(builds.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + + // The data source logs the park, and logs that every synchronizer is parked. + awaitLogContains(logging, "Synchronizer 'MockQueuedSynchronizer' is starting."); + awaitLogContains(logging, "Synchronizer 'MockQueuedSynchronizer' reported status: TERMINAL_ERROR."); + awaitLogContains(logging, "Synchronizer 'MockQueuedSynchronizer' is backing off for "); + awaitLogContains(logging, "All synchronizers are backing off for at least "); + stopDataSource(dataSource); } @Test From c8ece8320f8e7e969676f6d1ecb7912ab7d5676e Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Fri, 6 Nov 2026 14:03:43 -0800 Subject: [PATCH 03/12] feat: Fall back or recover only to an FDv2 synchronizer that is not parked --- .../sdk/android/FDv2DataSource.java | 39 +++- .../sdk/android/SourceManager.java | 35 ++++ .../sdk/android/FDv2DataSourceTest.java | 189 ++++++++++++++++++ 3 files changed, 258 insertions(+), 5 deletions(-) diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java index 47a5e4f65..9dc2af2cd 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java @@ -713,6 +713,9 @@ private void runSynchronizers( MonotonicTime recoveryTime = canRecover ? MonotonicTime.now().plusMillis(recoveryTimeoutMillis) : null; MonotonicTime fallbackTime = null; + // True once the loop has seen the deadline pass. + boolean recoveryMet = false; + boolean fallbackMet = false; // The result the loop is waiting for, or null if it has not asked the synchronizer for one. Future nextFuture = null; while (running) { @@ -720,8 +723,12 @@ private void runSynchronizers( nextFuture = synchronizer.next(); } - // Wait for the next result, or for the next deadline. - if (awaitUntil(nextFuture, earliest(recoveryTime, fallbackTime))) { + // Wait for the next result, or for the next deadline that the loop has not seen pass. + MonotonicTime wakeTime = earliest( + recoveryMet ? null : recoveryTime, + fallbackMet ? null : fallbackTime, + sourceManager.getEarliestUnparkTime()); + if (awaitUntil(nextFuture, wakeTime)) { FDv2SourceResult result = nextFuture.get(); // The pending result has arrived. nextFuture = null; @@ -731,6 +738,7 @@ private void runSynchronizers( resetSynchronizerStatusDedupe(); // The synchronizer is healthy, so it no longer needs to fall back. fallbackTime = null; + fallbackMet = false; ChangeSet> changeSet = result.getChangeSet(); if (changeSet != null) { sink.apply(context, changeSet); @@ -809,9 +817,16 @@ private void runSynchronizers( break; } - // Check each deadline against the current time. Recovery takes precedence over fallback. + // Check every deadline against the current time. MonotonicTime now = MonotonicTime.now(); - if (recoveryTime != null && !now.isBefore(recoveryTime)) { + sourceManager.unparkExpiredSynchronizers(); + boolean recoveryJustMet = !recoveryMet && recoveryTime != null && !now.isBefore(recoveryTime); + boolean fallbackJustMet = !fallbackMet && fallbackTime != null && !now.isBefore(fallbackTime); + recoveryMet = recoveryMet || recoveryJustMet; + fallbackMet = fallbackMet || fallbackJustMet; + + // Recovery takes precedence over fallback. + if (recoveryMet && sourceManager.hasMorePreferredAvailableSynchronizer()) { // For recovery, start the search for the next synchronizer at the first one. sourceManager.resetSourceIndex(); logger.info( @@ -820,13 +835,27 @@ private void runSynchronizers( ); break; } - if (fallbackTime != null && !now.isBefore(fallbackTime)) { + if (fallbackMet && sourceManager.hasOtherAvailableSynchronizer()) { logger.info( "Fallback condition met, falling back from synchronizer '{}'.", synchronizer.name() ); break; } + + // Keep running this synchronizer. + if (fallbackJustMet) { + logger.info( + "Fallback condition met, but no other synchronizer is available; continuing with synchronizer '{}'.", + synchronizer.name() + ); + } + if (recoveryJustMet) { + logger.info( + "Recovery condition met, but no more-preferred synchronizer is available; continuing with synchronizer '{}'.", + synchronizer.name() + ); + } } } catch (ExecutionException | RuntimeException e) { Throwable failure = e instanceof ExecutionException && e.getCause() != null ? e.getCause() : e; diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java index cb5090a74..008670214 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java @@ -237,6 +237,41 @@ Initializer getNextInitializerAndSetActive() { } } + /** + * True if a synchronizer other than the current one is available. + * + * @return true if another synchronizer is available + */ + boolean hasOtherAvailableSynchronizer() { + synchronized (activeSourceLock) { + for (SynchronizerFactoryWithState s : synchronizerFactories) { + if (s != currentSynchronizerFactory && s.isEligible()) { + return true; + } + } + return false; + } + } + + /** + * True if a synchronizer earlier in the list than the current one is available. + * + * @return true if a more-preferred synchronizer is available + */ + boolean hasMorePreferredAvailableSynchronizer() { + synchronized (activeSourceLock) { + for (SynchronizerFactoryWithState s : synchronizerFactories) { + if (s == currentSynchronizerFactory) { + return false; + } + if (s.isEligible()) { + return true; + } + } + return false; + } + } + /** True if the current synchronizer is the first available one (prime). */ boolean isPrimeSynchronizer() { synchronized (activeSourceLock) { diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java index 144e9b118..580820b9a 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java @@ -971,6 +971,195 @@ public void changeSetDuringInterruptionRestartsFallbackTimeout() throws Exceptio stopDataSource(dataSource); } + @Test + public void fallbackWithNoOtherAvailableSynchronizerKeepsCurrentSynchronizer() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger secondaryBuildCount = new AtomicInteger(); + MockQueuedSynchronizer firstSecondary = new MockQueuedSynchronizer(interrupted()); + + // The primary synchronizer fails, so it is parked for the rest of the test. + // The first instance of the secondary synchronizer reports that it is interrupted. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Arrays.asList( + () -> new MockQueuedSynchronizer(terminalError()), + () -> secondaryBuildCount.incrementAndGet() == 1 + ? firstSecondary + : new MockQueuedSynchronizer()), + null, + /* fallbackTimeoutMillis= */ 10, + 300_000, + /* parkInitialDelayMillis= */ TimeUnit.MINUTES.toMillis(5), + /* parkMaxDelayMillis= */ TimeUnit.HOURS.toMillis(1), + /* failoverDelayMillis= */ 10); + + // Start the data source. After the fallback condition of the secondary synchronizer is met, send a change + // set to the instance of the secondary synchronizer that was already running. + startDataSource(dataSource); + executor.schedule( + () -> firstSecondary.addResult(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 60, + TimeUnit.MILLISECONDS); + + // The data source applies the change set, so it kept that instance. + sink.awaitApplyCount(1, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(1, sink.getApplyCount()); + assertEquals(1, secondaryBuildCount.get()); + stopDataSource(dataSource); + } + + @Test + public void unparkWhileCurrentSynchronizerIsHealthyKeepsIt() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger primaryBuildCount = new AtomicInteger(); + MockQueuedSynchronizer secondary = new MockQueuedSynchronizer( + FDv2SourceResult.changeSet(makeChangeSet(true), false)); + + // The primary synchronizer fails, so it is parked for 25-50ms. The secondary synchronizer delivers data and + // stays healthy. The fallback and recovery timeouts are too long to matter. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Arrays.asList( + () -> { + primaryBuildCount.incrementAndGet(); + return new MockQueuedSynchronizer(terminalError()); + }, + () -> secondary), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ 50, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); + + // Start the data source, and send another change set after the primary synchronizer unparks. + startDataSource(dataSource); + executor.schedule( + () -> secondary.addResult(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 100, + TimeUnit.MILLISECONDS); + + // The data source applies both change sets, and does not start the primary synchronizer again. + sink.awaitApplyCount(2, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); + assertEquals(1, primaryBuildCount.get()); + stopDataSource(dataSource); + } + + @Test + public void fallbackSwitchesWhenParkedSynchronizerUnparks() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger primaryBuildCount = new AtomicInteger(); + AtomicInteger secondaryBuildCount = new AtomicInteger(); + + // The first instance of the primary synchronizer fails, and the second instance delivers data. + // The secondary synchronizer reports that it is interrupted. + // The fallback condition is met after 10ms, and the primary synchronizer unparks after 50-100ms. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Arrays.asList( + () -> primaryBuildCount.incrementAndGet() == 1 + ? new MockQueuedSynchronizer(terminalError()) + : new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + () -> { + secondaryBuildCount.incrementAndGet(); + return new MockQueuedSynchronizer(interrupted()); + }), + null, + /* fallbackTimeoutMillis= */ 10, + 300_000, + /* parkInitialDelayMillis= */ 100, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); + + // Start the data source. + startDataSource(dataSource); + + // When the primary synchronizer unparks, the data source falls back to it and applies its data. + sink.awaitApplyCount(1, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(2, primaryBuildCount.get()); + assertEquals(1, secondaryBuildCount.get()); + stopDataSource(dataSource); + } + + @Test + public void changeSetAfterFallbackIsMetPreventsSwitchAtUnpark() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger primaryBuildCount = new AtomicInteger(); + MockQueuedSynchronizer secondary = new MockQueuedSynchronizer(interrupted()); + + // The primary synchronizer fails, so it is parked briefly. The secondary synchronizer reports that it is + // interrupted. The fallback condition is met after 10ms, and the primary synchronizer unparks after + // 80-160ms. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Arrays.asList( + () -> { + primaryBuildCount.incrementAndGet(); + return new MockQueuedSynchronizer(terminalError()); + }, + () -> secondary), + null, + /* fallbackTimeoutMillis= */ 10, + 300_000, + /* parkInitialDelayMillis= */ 160, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); + + // Start the data source, send a change set after the fallback is met, and another after the unpark. + startDataSource(dataSource); + executor.schedule( + () -> secondary.addResult(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 50, + TimeUnit.MILLISECONDS); + executor.schedule( + () -> secondary.addResult(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 210, + TimeUnit.MILLISECONDS); + + // The data source applies both change sets, and does not start the primary synchronizer again. + sink.awaitApplyCount(2, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); + assertEquals(1, primaryBuildCount.get()); + stopDataSource(dataSource); + } + + @Test + public void recoveryWithPreferredSynchronizerParkedWaitsForItsUnpark() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger primaryBuildCount = new AtomicInteger(); + AtomicInteger secondaryBuildCount = new AtomicInteger(); + + // The first instance of the primary synchronizer fails, and the second instance delivers data. + // The secondary synchronizer delivers data and stays healthy. + // The recovery condition is met after 50ms, and the primary synchronizer unparks after 100-200ms. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Arrays.asList( + () -> primaryBuildCount.incrementAndGet() == 1 + ? new MockQueuedSynchronizer(terminalError()) + : new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + () -> { + secondaryBuildCount.incrementAndGet(); + return new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + }), + null, + 120_000, + /* recoveryTimeoutMillis= */ 50, + /* parkInitialDelayMillis= */ 200, + /* parkMaxDelayMillis= */ 1000, + /* failoverDelayMillis= */ 10); + + // Start the data source. + startDataSource(dataSource); + + // When the primary synchronizer unparks, the data source recovers to it and applies its data. + sink.awaitApplyCount(2, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(2, primaryBuildCount.get()); + assertEquals(1, secondaryBuildCount.get()); + stopDataSource(dataSource); + } + // ============================================================================ // Stop / Close Lifecycle // ============================================================================ From ef63016788ba11a1408244f8d458d936b8e459a5 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Fri, 6 Nov 2026 14:07:29 -0800 Subject: [PATCH 04/12] feat: End the FDv1 fallback after a TTL and restart the FDv2 synchronizers --- .../sdk/android/FDv2DataSource.java | 101 ++++-- .../sdk/android/SourceManager.java | 58 ++- .../sdk/android/FDv2DataSourceTest.java | 340 ++++++++++++++++-- 3 files changed, 450 insertions(+), 49 deletions(-) diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java index 9dc2af2cd..febd21e92 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java @@ -81,6 +81,11 @@ public interface DataSourceFactory { */ private static final long DEFAULT_FAILOVER_DELAY_MILLIS = TimeUnit.SECONDS.toMillis(1); + /** + * Default duration of a fallback to FDv1, when the directive does not give one. + */ + private static final long DEFAULT_FDV1_FALLBACK_TTL_MILLIS = TimeUnit.HOURS.toMillis(1); + private final List> cacheInitializers; private final SourceManager sourceManager; private final long fallbackTimeoutMillis; @@ -88,6 +93,7 @@ public interface DataSourceFactory { private final long parkInitialDelayMillis; private final long parkMaxDelayMillis; private final long failoverDelayMillis; + private final long defaultFDv1FallbackTtlMillis; private final ScheduledExecutorService sharedExecutor; private final AtomicBoolean startCalled = new AtomicBoolean(false); @@ -111,7 +117,7 @@ public interface DataSourceFactory { /** * Convenience constructor using default timeouts and delays. * See {@link #FDv2DataSource(LDContext, List, List, DataSourceFactory, DataSourceUpdateSinkV2, - * ScheduledExecutorService, LDLogger, long, long, long, long, long)} for parameter documentation. + * ScheduledExecutorService, LDLogger, long, long, long, long, long, long)} for parameter documentation. */ FDv2DataSource( @NonNull LDContext evaluationContext, @@ -128,7 +134,8 @@ public interface DataSourceFactory { DEFAULT_RECOVERY_TIMEOUT_MILLIS, DEFAULT_PARK_INITIAL_DELAY_MILLIS, DEFAULT_PARK_MAX_DELAY_MILLIS, - DEFAULT_FAILOVER_DELAY_MILLIS); + DEFAULT_FAILOVER_DELAY_MILLIS, + DEFAULT_FDV1_FALLBACK_TTL_MILLIS); } /** @@ -150,6 +157,7 @@ public interface DataSourceFactory { * @param parkInitialDelayMillis base of the park delay, in milliseconds * @param parkMaxDelayMillis maximum of the park delay, in milliseconds * @param failoverDelayMillis base of the failover delay, in milliseconds + * @param defaultFDv1FallbackTtlMillis duration of a fallback to FDv1, in milliseconds */ FDv2DataSource( @NonNull LDContext evaluationContext, @@ -163,7 +171,8 @@ public interface DataSourceFactory { long recoveryTimeoutMillis, long parkInitialDelayMillis, long parkMaxDelayMillis, - long failoverDelayMillis + long failoverDelayMillis, + long defaultFDv1FallbackTtlMillis ) { this.evaluationContext = evaluationContext; this.dataSourceUpdateSink = dataSourceUpdateSink; @@ -194,6 +203,7 @@ public interface DataSourceFactory { this.parkInitialDelayMillis = parkInitialDelayMillis; this.parkMaxDelayMillis = parkMaxDelayMillis; this.failoverDelayMillis = failoverDelayMillis; + this.defaultFDv1FallbackTtlMillis = defaultFDv1FallbackTtlMillis; this.sharedExecutor = sharedExecutor; } @@ -243,7 +253,7 @@ public void start(@NonNull Callback resultCallback) { runGeneralInitializers(context, dataSourceUpdateSink); } - if (!sourceManager.hasAvailableSynchronizers()) { + if (!sourceManager.hasFDv2Synchronizers() && !sourceManager.hasAvailableSynchronizers()) { if (!startCompleted.get()) { // try to claim this is the cause of the shutdown, but it might have already been set by an intentional stop(). shutdownCause.set(new LDFailure("All initializers exhausted and there are no available synchronizers.", LDFailure.FailureType.UNKNOWN_ERROR)); @@ -419,7 +429,7 @@ private void runGeneralInitializers( // FDv1 fallback takes priority over all other result processing. // The spec requires honoring the fallback signal from any response, // regardless of whether data was included or what the selector state is. - if (result.isFdv1Fallback() && sourceManager.hasFDv1Fallback()) { + if (result.isFdv1Fallback()) { if (result.getResultType() == SourceResultType.CHANGE_SET) { ChangeSet> changeSet = result.getChangeSet(); if (changeSet != null) { @@ -427,12 +437,14 @@ private void runGeneralInitializers( anyDataReceived = true; } } - logger.info(FDV1_FALLBACK_MESSAGE); - sourceManager.fdv1Fallback(); if (anyDataReceived) { sink.setStatus(DataSourceState.VALID, null); tryCompleteStart(true, null); } + fallBackToFDv1( + sink, + jittered(defaultFDv1FallbackTtlMillis), + result.getStatus() != null ? result.getStatus().getError() : null); return; } @@ -613,9 +625,33 @@ private void parkActiveSynchronizer(@Nullable String synchronizerName, @Nullable } /** - * Gets the next available synchronizer, builds it, and sets it active. While no synchronizer is available, this - * waits for the earliest unpark time. A synchronizer that fails to build is parked, and the next available - * synchronizer is tried after the failover delay. + * Falls back to FDv1 for the given time. If no FDv1 fallback synchronizer is configured, no synchronizer runs + * until the fallback ends. + * + * @param sink the sink for status updates + * @param fallbackTtlMillis how long the fallback lasts, in milliseconds + * @param error the error that came with the directive, or null + */ + private void fallBackToFDv1( + @NonNull DataSourceUpdateSinkV2 sink, + long fallbackTtlMillis, + @Nullable Throwable error + ) { + sourceManager.startFDv1Fallback(MonotonicTime.now().plusMillis(fallbackTtlMillis)); + if (sourceManager.hasFDv1Fallback()) { + logger.info(FDV1_FALLBACK_MESSAGE); + } else { + logger.warn("Fallback to FDv1 was requested, but no FDv1 fallback synchronizer is configured."); + sink.setStatus(DataSourceState.INTERRUPTED, error); + } + logger.info("FDv2 retry scheduled in {}s.", TimeUnit.MILLISECONDS.toSeconds(fallbackTtlMillis)); + } + + /** + * Gets the next available synchronizer, builds it, and sets it active. If the FDv1 fallback has expired, this + * first makes the FDv2 synchronizers available again. While no synchronizer is available, this waits for the + * earliest unpark time or the end of the FDv1 fallback, whichever comes first. A synchronizer that fails to build + * is parked, and the next available synchronizer is tried after the failover delay. * * @param sink the sink for status updates * @return the active synchronizer, or null if the data source was stopped @@ -623,6 +659,9 @@ private void parkActiveSynchronizer(@Nullable String synchronizerName, @Nullable @Nullable private Synchronizer buildNextAvailableSynchronizer(@NonNull DataSourceUpdateSinkV2 sink) { while (!stopCalled.get()) { + if (sourceManager.endFDv1FallbackIfExpired()) { + logger.info("Fallback TTL elapsed, restarting FDv2 data sources."); + } Synchronizer synchronizer; try { synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); @@ -642,14 +681,20 @@ private Synchronizer buildNextAvailableSynchronizer(@NonNull DataSourceUpdateSin } MonotonicTime earliestUnparkTime = sourceManager.getEarliestUnparkTime(); + MonotonicTime fallbackEndTime = sourceManager.getFDv1FallbackEndTime(); long waitMillis; - if (earliestUnparkTime == null) { + if (earliestUnparkTime == null && fallbackEndTime == null) { // This should never happen. waitMillis = parkInitialDelayMillis; logger.error("No synchronizer can be started; checking again in {}s.", TimeUnit.MILLISECONDS.toSeconds(waitMillis)); + } else if (earliestUnparkTime == null) { + waitMillis = TimeUnit.NANOSECONDS.toMillis(MonotonicTime.now().nanosUntil(fallbackEndTime)); + logger.warn("No synchronizer is available until the FDv1 fallback ends in {}s.", + Math.max(0, TimeUnit.MILLISECONDS.toSeconds(waitMillis))); } else { - waitMillis = TimeUnit.NANOSECONDS.toMillis(MonotonicTime.now().nanosUntil(earliestUnparkTime)); + MonotonicTime earliest = earliest(earliestUnparkTime, fallbackEndTime); + waitMillis = TimeUnit.NANOSECONDS.toMillis(MonotonicTime.now().nanosUntil(earliest)); logger.warn("All synchronizers are backing off for at least {}s.", Math.max(0, TimeUnit.MILLISECONDS.toSeconds(waitMillis))); } @@ -727,7 +772,8 @@ private void runSynchronizers( MonotonicTime wakeTime = earliest( recoveryMet ? null : recoveryTime, fallbackMet ? null : fallbackTime, - sourceManager.getEarliestUnparkTime()); + sourceManager.getEarliestUnparkTime(), + sourceManager.getFDv1FallbackEndTime()); if (awaitUntil(nextFuture, wakeTime)) { FDv2SourceResult result = nextFuture.get(); // The pending result has arrived. @@ -803,14 +849,23 @@ private void runSynchronizers( // whether the synchronizer is still running — a terminal error // response can still carry the fallback header, and the server's // instruction to use FDv1 should take precedence. - if (result.isFdv1Fallback() - && sourceManager.hasFDv1Fallback() - && !sourceManager.isCurrentSynchronizerFDv1Fallback()) { - logger.info(FDV1_FALLBACK_MESSAGE); - sourceManager.fdv1Fallback(); - // The directive takes over, so there is no failover delay after a park. - parked = false; - running = false; + if (result.isFdv1Fallback()) { + if (sourceManager.isCurrentSynchronizerFDv1Fallback()) { + // A directive during the FDv1 fallback replaces its TTL, and FDv1 keeps running. + long fallbackTtlMillis = jittered(defaultFDv1FallbackTtlMillis); + sourceManager.startFDv1Fallback(MonotonicTime.now().plusMillis(fallbackTtlMillis)); + logger.info("FDv2 retry scheduled in {}s.", + TimeUnit.MILLISECONDS.toSeconds(fallbackTtlMillis)); + } else { + // A directive to fall back to FDv1 ends the run of an FDv2 synchronizer. + fallBackToFDv1( + sink, + jittered(defaultFDv1FallbackTtlMillis), + result.getStatus() != null ? result.getStatus().getError() : null); + // The directive takes over, so there is no failover delay after a park. + parked = false; + running = false; + } } } if (!running) { @@ -820,6 +875,10 @@ private void runSynchronizers( // Check every deadline against the current time. MonotonicTime now = MonotonicTime.now(); sourceManager.unparkExpiredSynchronizers(); + if (sourceManager.endFDv1FallbackIfExpired()) { + logger.info("Fallback TTL elapsed, restarting FDv2 data sources."); + break; + } boolean recoveryJustMet = !recoveryMet && recoveryTime != null && !now.isBefore(recoveryTime); boolean fallbackJustMet = !fallbackMet && fallbackTime != null && !now.isBefore(fallbackTime); recoveryMet = recoveryMet || recoveryJustMet; diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java index 008670214..4e4fcb8ec 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java @@ -32,6 +32,12 @@ final class SourceManager implements Closeable { private SynchronizerFactoryWithState currentSynchronizerFactory; + /** + * The time at which the fallback to FDv1 ends, or null if the SDK is not falling back to FDv1. Accessed only + * under activeSourceLock. + */ + private MonotonicTime fdv1FallbackEndTime; + SourceManager( @NonNull List synchronizerFactories, @NonNull List> initializers @@ -60,13 +66,25 @@ boolean hasFDv1Fallback() { return false; } + boolean hasFDv2Synchronizers() { + for (SynchronizerFactoryWithState s : synchronizerFactories) { + if (!s.isFDv1Fallback()) { + return true; + } + } + return false; + } + /** * Block all non-FDv1 synchronizers, unblock the FDv1 fallback, and reset the * synchronizer index so the next {@link #getNextAvailableSynchronizerAndSetActive()} - * picks the now-unblocked FDv1 slot. + * picks the now-unblocked FDv1 slot. The fallback lasts until the given time. + * + * @param endTime the time at which the fallback to FDv1 ends */ - void fdv1Fallback() { + void startFDv1Fallback(@NonNull MonotonicTime endTime) { synchronized (activeSourceLock) { + fdv1FallbackEndTime = endTime; for (SynchronizerFactoryWithState s : synchronizerFactories) { if (s.isFDv1Fallback()) { s.unblock(); @@ -78,6 +96,42 @@ void fdv1Fallback() { } } + /** + * The time at which the fallback to FDv1 ends. + * + * @return the end time, or null if the SDK is not falling back to FDv1 + */ + @Nullable + MonotonicTime getFDv1FallbackEndTime() { + synchronized (activeSourceLock) { + return fdv1FallbackEndTime; + } + } + + /** + * Ends the fallback to FDv1 if its end time has arrived. This unblocks the FDv2 synchronizers, blocks the FDv1 + * fallback, and starts the search for the next synchronizer from the most-preferred one. + * + * @return true if the fallback ended + */ + boolean endFDv1FallbackIfExpired() { + synchronized (activeSourceLock) { + if (fdv1FallbackEndTime == null || MonotonicTime.now().isBefore(fdv1FallbackEndTime)) { + return false; + } + fdv1FallbackEndTime = null; + for (SynchronizerFactoryWithState s : synchronizerFactories) { + if (s.isFDv1Fallback()) { + s.block(); + } else { + s.unblock(); + } + } + synchronizerIndex = -1; + return true; + } + } + /** * Stops parking every synchronizer whose unpark time has arrived. */ diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java index 580820b9a..dc694683e 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java @@ -74,7 +74,7 @@ public void tearDown() { } /** - * Builds a data source with production timeouts and park delays, and a 10ms failover delay. + * Builds a data source with production timeouts, park delays, and fallback TTL, and a 10ms failover delay. */ private FDv2DataSource buildDataSource( MockComponents.MockDataSourceUpdateSink sink, @@ -84,7 +84,7 @@ private FDv2DataSource buildDataSource( } /** - * Builds a data source with production park delays, and a 10ms failover delay. + * Builds a data source with production park delays and fallback TTL, and a 10ms failover delay. */ private FDv2DataSource buildDataSource( MockComponents.MockDataSourceUpdateSink sink, @@ -97,7 +97,8 @@ private FDv2DataSource buildDataSource( recoveryTimeoutMillis, /* parkInitialDelayMillis= */ TimeUnit.MINUTES.toMillis(5), /* parkMaxDelayMillis= */ TimeUnit.HOURS.toMillis(1), - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); } private FDv2DataSource buildDataSource( @@ -109,7 +110,8 @@ private FDv2DataSource buildDataSource( long recoveryTimeoutMillis, long parkInitialDelayMillis, long parkMaxDelayMillis, - long failoverDelayMillis) { + long failoverDelayMillis, + long fdv1FallbackTtlMillis) { return new FDv2DataSource( CONTEXT, initializers, @@ -122,7 +124,8 @@ private FDv2DataSource buildDataSource( recoveryTimeoutMillis, parkInitialDelayMillis, parkMaxDelayMillis, - failoverDelayMillis); + failoverDelayMillis, + fdv1FallbackTtlMillis); } /** Starts the data source and returns a callback that will receive the start result. */ @@ -647,7 +650,8 @@ public void terminalErrorParksSynchronizerAndRestartsItAfterUnparkTime() throws 300_000, /* parkInitialDelayMillis= */ 100, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source with its only synchronizer. AwaitableCallback startCallback = startDataSource(dataSource); @@ -675,7 +679,8 @@ public void failedNextParksSynchronizer() throws Exception { 300_000, /* parkInitialDelayMillis= */ 100, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source with its only synchronizer. AwaitableCallback startCallback = startDataSource(dataSource); @@ -716,7 +721,8 @@ public void close() { 300_000, /* parkInitialDelayMillis= */ 100, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source with its only synchronizer. AwaitableCallback startCallback = startDataSource(dataSource); @@ -747,7 +753,8 @@ public void constructionFailureParksSynchronizer() throws Exception { 300_000, /* parkInitialDelayMillis= */ 100, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source with its only synchronizer. AwaitableCallback startCallback = startDataSource(dataSource); @@ -776,7 +783,8 @@ public void unpromptedShutdownParksSynchronizer() throws Exception { 300_000, /* parkInitialDelayMillis= */ 100, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source with its only synchronizer. AwaitableCallback startCallback = startDataSource(dataSource); @@ -830,7 +838,8 @@ public void allSynchronizersParkedRestartsAfterEarliestUnparkTime() throws Excep 300_000, /* parkInitialDelayMillis= */ 100, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source, and wait for each synchronizer to be built and then for one more build. startDataSource(dataSource); @@ -991,7 +1000,8 @@ public void fallbackWithNoOtherAvailableSynchronizerKeepsCurrentSynchronizer() t 300_000, /* parkInitialDelayMillis= */ TimeUnit.MINUTES.toMillis(5), /* parkMaxDelayMillis= */ TimeUnit.HOURS.toMillis(1), - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source. After the fallback condition of the secondary synchronizer is met, send a change // set to the instance of the secondary synchronizer that was already running. @@ -1030,7 +1040,8 @@ public void unparkWhileCurrentSynchronizerIsHealthyKeepsIt() throws Exception { 300_000, /* parkInitialDelayMillis= */ 50, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source, and send another change set after the primary synchronizer unparks. startDataSource(dataSource); @@ -1070,7 +1081,8 @@ public void fallbackSwitchesWhenParkedSynchronizerUnparks() throws Exception { 300_000, /* parkInitialDelayMillis= */ 100, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source. startDataSource(dataSource); @@ -1104,7 +1116,8 @@ public void changeSetAfterFallbackIsMetPreventsSwitchAtUnpark() throws Exception 300_000, /* parkInitialDelayMillis= */ 160, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source, send a change set after the fallback is met, and another after the unpark. startDataSource(dataSource); @@ -1148,7 +1161,8 @@ public void recoveryWithPreferredSynchronizerParkedWaitsForItsUnpark() throws Ex /* recoveryTimeoutMillis= */ 50, /* parkInitialDelayMillis= */ 200, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source. startDataSource(dataSource); @@ -1922,7 +1936,8 @@ public void statusStaysInterruptedWhileOnlySynchronizerIsParked() throws Excepti 300_000, /* parkInitialDelayMillis= */ 100, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source, and wait until the parked synchronizer is built again. startDataSource(dataSource); @@ -2179,7 +2194,8 @@ public void fdv1FallbackSynchronizerIsParkedAndRestartedDuringDirectedFallback() 300_000, /* parkInitialDelayMillis= */ 100, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source. startDataSource(dataSource); @@ -2191,22 +2207,293 @@ public void fdv1FallbackSynchronizerIsParkedAndRestartedDuringDirectedFallback() stopDataSource(dataSource); } + // When a synchronizer signals FDv1 fallback but no FDv1 fallback synchronizer is configured, the data source + // reports INTERRUPTED, runs no synchronizer until the fallback TTL ends, and then restarts the FDv2 synchronizers. @Test - public void fdv1FallbackNotTriggeredWhenNoFdv1SlotExists() throws Exception { + public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredRecoversAfterTTL() throws Exception { MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger firstSyncBuildCount = new AtomicInteger(); + AtomicBoolean secondSyncCalled = new AtomicBoolean(false); - MockQueuedSynchronizer fdv2Sync = new MockQueuedSynchronizer( - FDv2SourceResult.changeSet(makeChangeSet(true), true)); - + // The first instance sends a payload, then one with the directive. The second sends a payload. + // The second synchronizer must never run. + // No FDv1 fallback synchronizer is configured, and the fallback TTL ends after 50-100ms. FDv2DataSource dataSource = buildDataSource(sink, - Collections.>emptyList(), - Collections.>singletonList(() -> fdv2Sync)); + Collections.emptyList(), + Arrays.asList( + () -> firstSyncBuildCount.incrementAndGet() == 1 + ? new MockQueuedSynchronizer( + FDv2SourceResult.changeSet(makeChangeSet(false), false), + FDv2SourceResult.changeSet(makeChangeSet(false), true)) + : new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + () -> { + secondSyncCalled.set(true); + return new MockQueuedSynchronizer(); + }), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ TimeUnit.MINUTES.toMillis(5), + /* parkMaxDelayMillis= */ TimeUnit.HOURS.toMillis(1), + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ 100); + // Start the data source. AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); - assertEquals(DataSourceState.VALID, sink.awaitStatus(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + // Both payloads report VALID. The directive and the wait for the TTL report INTERRUPTED. The first + // synchronizer restarts after the TTL and reports VALID. + List statuses = sink.awaitStatuses(5, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(Arrays.asList( + DataSourceState.VALID, + DataSourceState.VALID, + DataSourceState.INTERRUPTED, + DataSourceState.INTERRUPTED, + DataSourceState.VALID), statuses); + assertEquals(2, firstSyncBuildCount.get()); + assertFalse(secondSyncCalled.get()); + stopDataSource(dataSource); + } + + // While no FDv1 fallback synchronizer is configured, the FDv2 synchronizer that sent the directive is stopped. + @Test + public void directiveWithoutFDv1ClosesFDv2SynchronizerDuringFallback() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + CountDownLatch closed = new CountDownLatch(1); + // The synchronizer sends a payload with the directive, and records when it is closed. + // No FDv1 fallback synchronizer is configured, and the fallback lasts much longer than the test. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Collections.singletonList(() -> new MockQueuedSynchronizer( + FDv2SourceResult.changeSet(makeChangeSet(true), true)) { + @Override + public void close() { + closed.countDown(); + super.close(); + } + })); + + // Start the data source. + startDataSource(dataSource); + + // The synchronizer is closed during the fallback. + assertTrue(closed.await(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + stopDataSource(dataSource); + } + + // When the fallback TTL ends, the data source stops the FDv1 synchronizer and restarts the FDv2 synchronizers. + @Test + public void fdv1FallbackEndsAfterTTL() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger fdv2BuildCount = new AtomicInteger(); + AtomicBoolean fdv1Closed = new AtomicBoolean(false); + + // The first FDv2 instance sends a payload with the directive. The second sends a payload. + // The FDv1 fallback synchronizer sends a payload, and records when it is closed. + // The fallback TTL ends after 50-100ms. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Collections.singletonList(() -> new MockQueuedSynchronizer( + FDv2SourceResult.changeSet(makeChangeSet(true), fdv2BuildCount.incrementAndGet() == 1))), + () -> new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false)) { + @Override + public void close() { + fdv1Closed.set(true); + super.close(); + } + }, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ TimeUnit.MINUTES.toMillis(5), + /* parkMaxDelayMillis= */ TimeUnit.HOURS.toMillis(1), + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ 100); + + // Start the data source. + startDataSource(dataSource); + + // After the TTL, the data source has stopped the FDv1 fallback synchronizer and applied FDv2 data again. + sink.awaitApplyCount(3, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(2, fdv2BuildCount.get()); + assertTrue(fdv1Closed.get()); + stopDataSource(dataSource); + } + + // When the fallback TTL ends before the FDv1 synchronizer's unpark time, the FDv2 synchronizers restart at the TTL. + @Test + public void fdv1FallbackEndsWhileFDv1SynchronizerIsParked() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger fdv2BuildCount = new AtomicInteger(); + AtomicInteger fdv1BuildCount = new AtomicInteger(); + + // The first FDv2 instance reports an interruption with the directive. The second sends a payload with the + // directive. The FDv1 fallback synchronizer fails, so the data source parks it. The FDv1 fallback + // synchronizer is parked for minutes, and the fallback TTL ends after 100-200ms. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Collections.singletonList(() -> fdv2BuildCount.incrementAndGet() == 1 + ? new MockQueuedSynchronizer(FDv2SourceResult.status( + FDv2SourceResult.Status.interrupted(new RuntimeException("500")), true)) + : new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), true))), + () -> { + fdv1BuildCount.incrementAndGet(); + return new MockQueuedSynchronizer(terminalError()); + }, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ TimeUnit.MINUTES.toMillis(5), + /* parkMaxDelayMillis= */ TimeUnit.HOURS.toMillis(1), + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ 200); + + // Start the data source. + startDataSource(dataSource); + + // The FDv2 synchronizer is built again after the TTL, and its data is applied. + sink.awaitApplyCount(1, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(1, sink.getApplyCount()); + assertEquals(2, fdv2BuildCount.get()); + + // The interruption, the FDv1 failure, and the wait for the TTL report INTERRUPTED. The FDv2 payload reports + // VALID. Its directive falls back to the FDv1 synchronizer, which is still parked, so nothing runs until + // the TTL, and the wait reports INTERRUPTED. + List statuses = sink.awaitStatuses(5, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(Arrays.asList( + DataSourceState.INTERRUPTED, + DataSourceState.INTERRUPTED, + DataSourceState.INTERRUPTED, + DataSourceState.VALID, + DataSourceState.INTERRUPTED), statuses); + assertEquals(1, fdv1BuildCount.get()); + stopDataSource(dataSource); + } + + // A directive during the FDv1 fallback replaces the TTL, and the FDv1 synchronizer keeps running. + @Test + public void directiveDuringFDv1FallbackExtendsTTL() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger fdv2BuildCount = new AtomicInteger(); + AtomicInteger fdv1BuildCount = new AtomicInteger(); + BlockingQueue fdv2RestartTimes = new LinkedBlockingQueue<>(); + MockQueuedSynchronizer fdv1Sync = new MockQueuedSynchronizer(); + + // The first FDv2 instance sends a payload with the directive. The second records when it is built. + // The FDv1 fallback synchronizer sends the results that the test gives it. + // The fallback TTL ends after 100-200ms. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Collections.singletonList(() -> { + if (fdv2BuildCount.incrementAndGet() == 1) { + return new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), true)); + } + fdv2RestartTimes.offer(System.nanoTime()); + return new MockQueuedSynchronizer(); + }), + () -> { + fdv1BuildCount.incrementAndGet(); + return fdv1Sync; + }, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ TimeUnit.MINUTES.toMillis(5), + /* parkMaxDelayMillis= */ TimeUnit.HOURS.toMillis(1), + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ 200); + + // Start the data source. FDv1 sends a directive every 50ms until 300ms, which is longer than the TTL. + long start = System.nanoTime(); + startDataSource(dataSource); + for (int delay = 50; delay <= 300; delay += 50) { + executor.schedule( + () -> fdv1Sync.addResult(FDv2SourceResult.changeSet(makeChangeSet(false), true)), + delay, + TimeUnit.MILLISECONDS); + } + + // FDv2 restarts at least the shortest TTL after the last directive, and FDv1 never restarts. + Long fdv2Restart = fdv2RestartTimes.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertNotNull(fdv2Restart); + long restartMillis = TimeUnit.NANOSECONDS.toMillis(fdv2Restart - start); + assertTrue("restarted FDv2 after " + restartMillis + "ms", restartMillis >= 400); + assertEquals(1, fdv1BuildCount.get()); + stopDataSource(dataSource); + } + + // When the FDv2 synchronizers send the directive again after the fallback TTL ends, the data source falls back + // again. + @Test + public void directiveDuringRecoveryFallsBackToFDv1Again() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger fdv2BuildCount = new AtomicInteger(); + AtomicInteger fdv1BuildCount = new AtomicInteger(); + + // Every instance of the FDv2 synchronizer sends a payload with the directive. + // Every instance of the FDv1 fallback synchronizer sends a payload. + // The fallback TTL ends after 50-100ms. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Collections.singletonList(() -> { + fdv2BuildCount.incrementAndGet(); + return new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), true)); + }), + () -> { + fdv1BuildCount.incrementAndGet(); + return new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + }, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ TimeUnit.MINUTES.toMillis(5), + /* parkMaxDelayMillis= */ TimeUnit.HOURS.toMillis(1), + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ 100); + + // Start the data source. + startDataSource(dataSource); + + // After the TTL, the FDv2 synchronizer sends the directive again, and the data source starts FDv1 again. + sink.awaitApplyCount(4, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(2, fdv2BuildCount.get()); + assertEquals(2, fdv1BuildCount.get()); + stopDataSource(dataSource); + } + + // When an initializer signals FDv1 fallback but no FDv1 synchronizer is configured, the data source reports + // INTERRUPTED, runs no synchronizer until the fallback TTL ends, and then starts the FDv2 synchronizers. + @Test + public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredRecoversAfterTTL() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger syncBuildCount = new AtomicInteger(); + + // The initializer fails with the directive. The synchronizer delivers data. + // No FDv1 fallback synchronizer is configured, and the fallback TTL ends after 50-100ms. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.singletonList(() -> new MockInitializer(FDv2SourceResult.status( + FDv2SourceResult.Status.terminalError(new RuntimeException("fallback requested")), true))), + Collections.singletonList(() -> { + syncBuildCount.incrementAndGet(); + return new MockQueuedSynchronizer(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + }), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ TimeUnit.MINUTES.toMillis(5), + /* parkMaxDelayMillis= */ TimeUnit.HOURS.toMillis(1), + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ 100); + + // Start the data source, and wait for it to be initialized by the synchronizer. + AwaitableCallback startCallback = startDataSource(dataSource); + assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); + + // The directive and the wait for the TTL report INTERRUPTED, and the synchronizer starts after the TTL. + List statuses = sink.awaitStatuses(3, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(Arrays.asList( + DataSourceState.INTERRUPTED, + DataSourceState.INTERRUPTED, + DataSourceState.VALID), statuses); + assertEquals(1, syncBuildCount.get()); stopDataSource(dataSource); } @@ -2542,7 +2829,8 @@ public void orchestrationLogging_logsSynchronizerParkedAndAllParked() throws Exc 300_000, /* parkInitialDelayMillis= */ 100, /* parkMaxDelayMillis= */ 1000, - /* failoverDelayMillis= */ 10); + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source, and wait until the parked synchronizer is built again. startDataSource(dataSource); From bb4c1b539d145c2fdc491bd0bf3cbc6dfca264b9 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Fri, 6 Nov 2026 14:08:35 -0800 Subject: [PATCH 05/12] feat: Reset an FDv2 synchronizer's backoff when it meets its reset threshold --- .../sdk/android/FDv2DataSource.java | 4 ++ .../sdk/android/SourceManager.java | 6 +++ .../android/SynchronizerFactoryWithState.java | 6 ++- .../android/subsystems/FDv2SourceResult.java | 30 +++++++++++-- .../sdk/android/FDv2DataSourceTest.java | 42 +++++++++++++++++++ 5 files changed, 84 insertions(+), 4 deletions(-) diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java index febd21e92..099f8002e 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java @@ -779,6 +779,10 @@ private void runSynchronizers( // The pending result has arrived. nextFuture = null; + if (result.isResetThresholdMet()) { + sourceManager.resetCurrentSynchronizerParkCount(); + } + switch (result.getResultType()) { case CHANGE_SET: resetSynchronizerStatusDedupe(); diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java index 4e4fcb8ec..87d935da0 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SourceManager.java @@ -231,6 +231,12 @@ int incrementCurrentSynchronizerParkCount() { } } + void resetCurrentSynchronizerParkCount() { + synchronized (activeSourceLock) { + currentSynchronizerFactory.resetParkCount(); + } + } + void parkCurrentSynchronizer(@NonNull MonotonicTime unparkTime) { synchronized (activeSourceLock) { currentSynchronizerFactory.park(unparkTime); diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SynchronizerFactoryWithState.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SynchronizerFactoryWithState.java index 25d3248ec..23e1e10f7 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SynchronizerFactoryWithState.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/SynchronizerFactoryWithState.java @@ -25,7 +25,7 @@ enum State { private final boolean isFDv1Fallback; /** - * The number of unexpected failures of the synchronizer. + * The number of unexpected failures since the synchronizer last reported that it met its reset threshold. */ private int parkCount = 0; @@ -72,6 +72,10 @@ int incrementParkCount() { return ++parkCount; } + void resetParkCount() { + parkCount = 0; + } + void park(@NonNull MonotonicTime unparkTime) { this.unparkTime = unparkTime; } diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/FDv2SourceResult.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/FDv2SourceResult.java index 0403c1734..4d0d2d1ee 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/FDv2SourceResult.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/FDv2SourceResult.java @@ -75,25 +75,30 @@ public String getReason() { @Nullable private final Status status; private final boolean fdv1Fallback; + private final boolean resetThresholdMet; private FDv2SourceResult(@NonNull SourceResultType resultType, @Nullable ChangeSet> changeSet, @Nullable Status status, - boolean fdv1Fallback) { + boolean fdv1Fallback, + boolean resetThresholdMet) { this.resultType = resultType; this.changeSet = changeSet; this.status = status; this.fdv1Fallback = fdv1Fallback; + this.resetThresholdMet = resetThresholdMet; } @NonNull public static FDv2SourceResult changeSet(@NonNull ChangeSet> changeSet, boolean fdv1Fallback) { - return new FDv2SourceResult(SourceResultType.CHANGE_SET, changeSet, null, fdv1Fallback); + return new FDv2SourceResult(SourceResultType.CHANGE_SET, changeSet, null, fdv1Fallback, + /* resetThresholdMet= */ false); } @NonNull public static FDv2SourceResult status(@NonNull Status status, boolean fdv1Fallback) { - return new FDv2SourceResult(SourceResultType.STATUS, null, status, fdv1Fallback); + return new FDv2SourceResult(SourceResultType.STATUS, null, status, fdv1Fallback, + /* resetThresholdMet= */ false); } @NonNull @@ -118,4 +123,23 @@ public Status getStatus() { public boolean isFdv1Fallback() { return fdv1Fallback; } + + /** + * True if the source met its reset threshold at the time of this result. + * + * @return true if the source met its reset threshold + */ + public boolean isResetThresholdMet() { + return resetThresholdMet; + } + + /** + * Creates a copy of this result that reports that the source met its reset threshold. + * + * @return a new FDv2SourceResult with {@link #isResetThresholdMet()} set to true + */ + @NonNull + public FDv2SourceResult withResetThresholdMet() { + return new FDv2SourceResult(resultType, changeSet, status, fdv1Fallback, /* resetThresholdMet= */ true); + } } diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java index dc694683e..e811d1c07 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java @@ -884,6 +884,48 @@ public void parkedSynchronizerSkippedInRotation() throws Exception { stopDataSource(dataSource); } + @Test + public void resetThresholdMetReturnsParkDelayToBase() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + BlockingQueue buildTimes = new LinkedBlockingQueue<>(); + + FDv2DataSource dataSource = buildDataSource(sink, + Collections.emptyList(), + Collections.singletonList(() -> { + buildTimes.offer(System.nanoTime()); + if (buildCount.incrementAndGet() == 3) { + // The third instance reports that it met its reset threshold before it fails. + return new MockQueuedSynchronizer( + FDv2SourceResult.changeSet(makeChangeSet(true), false).withResetThresholdMet(), + terminalError()); + } + // Every instance fails with a terminal error. + return new MockQueuedSynchronizer(terminalError()); + }), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ 100, + /* parkMaxDelayMillis= */ 10_000, + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); + + // Start the data source, and record when the synchronizer is built for the third and fourth times. + startDataSource(dataSource); + buildTimes.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + buildTimes.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + Long thirdBuild = buildTimes.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + Long fourthBuild = buildTimes.poll(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + // After the reset, the park is at most the base delay. Without the reset, it would be at least twice the base. + assertNotNull(thirdBuild); + assertNotNull(fourthBuild); + long parkMillis = TimeUnit.NANOSECONDS.toMillis(fourthBuild - thirdBuild); + assertTrue("park lasted " + parkMillis + "ms", parkMillis < 200); + stopDataSource(dataSource); + } + @Test public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); From 49a00f9f7236c9f782d337123e3550d7746fabe0 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Fri, 6 Nov 2026 14:09:39 -0800 Subject: [PATCH 06/12] feat: Report the reset threshold from the FDv2 polling synchronizer --- .../sdk/android/FDv2PollingSynchronizer.java | 22 ++++++++ .../android/FDv2PollingSynchronizerTest.java | 55 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2PollingSynchronizer.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2PollingSynchronizer.java index 2b0e43708..6904eac3f 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2PollingSynchronizer.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2PollingSynchronizer.java @@ -26,6 +26,11 @@ * that subsequent {@link #next()} calls also return the terminal result immediately. */ final class FDv2PollingSynchronizer extends FDv2PollingBase implements Synchronizer { + /** + * The number of consecutive successful polls that meets the reset threshold. + */ + private static final int RESET_THRESHOLD_SUCCESSFUL_POLLS = 2; + private final SelectorSource selectorSource; private final LDAsyncQueue resultQueue = new LDAsyncQueue<>(); @@ -34,6 +39,12 @@ final class FDv2PollingSynchronizer extends FDv2PollingBase implements Synchroni private volatile ScheduledFuture scheduledTask; private final Object taskLock = new Object(); + /** + * Only {@link #pollAndEnqueue()} reads or writes this. {@code scheduleWithFixedDelay} never starts a run of + * {@code pollAndEnqueue} before the previous run has finished. + */ + private int consecutiveSuccessfulPolls = 0; + /** * @param requestor the FDv2 requestor used to perform each poll * @param selectorSource source of the current selector, sent as the {@code basis} param @@ -70,8 +81,18 @@ private void pollAndEnqueue() { try { FDv2SourceResult result = doPoll(selectorSource.getSelector(), false); + if (result.getResultType() == SourceResultType.CHANGE_SET) { + consecutiveSuccessfulPolls++; + if (consecutiveSuccessfulPolls == RESET_THRESHOLD_SUCCESSFUL_POLLS) { + result = result.withResetThresholdMet(); + } + } + if (result.getResultType() == SourceResultType.STATUS) { FDv2SourceResult.Status status = result.getStatus(); + if (status != null && status.getState() == SourceSignal.INTERRUPTED) { + consecutiveSuccessfulPolls = 0; + } if (status != null && status.getState() == SourceSignal.TERMINAL_ERROR) { synchronized (taskLock) { if (scheduledTask != null) { @@ -93,6 +114,7 @@ private void pollAndEnqueue() { // cancels the task on any unchecked exception, ending all future polls with no // error signal. Log and enqueue an INTERRUPTED result so the consumer is notified LDUtil.logExceptionAtErrorLevel(logger, e, "Unexpected exception in polling synchronizer task"); + consecutiveSuccessfulPolls = 0; resultQueue.put(FDv2SourceResult.status(FDv2SourceResult.Status.interrupted(e), false)); } } diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2PollingSynchronizerTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2PollingSynchronizerTest.java index ffa8cfd20..304ecbf04 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2PollingSynchronizerTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2PollingSynchronizerTest.java @@ -21,7 +21,9 @@ import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; /** * Unit tests for {@link FDv2PollingSynchronizer}. @@ -109,6 +111,59 @@ public void consecutivePolls_deliveredInOrder() throws Exception { } } + // ---- reset threshold ---- + + @Test + public void resetThresholdMetIsReportedOnSecondConsecutiveSuccessfulPoll() throws Exception { + MockFDv2Requestor requestor = new MockFDv2Requestor(); + for (int i = 0; i < 3; i++) { + requestor.queueResponse( + FDv2Requestor.FDv2PayloadResponse.success(parseEvents(XFER_FULL_JSON), 200, false)); + } + FDv2PollingSynchronizer synchronizer = buildSynchronizer(requestor, 0, 10); + + try { + // Read the results of three consecutive successful polls. + FDv2SourceResult first = synchronizer.next().get(1, TimeUnit.SECONDS); + FDv2SourceResult second = synchronizer.next().get(1, TimeUnit.SECONDS); + FDv2SourceResult third = synchronizer.next().get(1, TimeUnit.SECONDS); + + // Only the second successful poll reports that the reset threshold was met. + assertFalse(first.isResetThresholdMet()); + assertTrue(second.isResetThresholdMet()); + assertFalse(third.isResetThresholdMet()); + } finally { + synchronizer.close(); + } + } + + @Test + public void failedPollRestartsResetThresholdCount() throws Exception { + MockFDv2Requestor requestor = new MockFDv2Requestor(); + requestor.queueResponse(FDv2Requestor.FDv2PayloadResponse.success(parseEvents(XFER_FULL_JSON), 200, false)); + requestor.queueError(new IOException("network error")); + requestor.queueResponse(FDv2Requestor.FDv2PayloadResponse.success(parseEvents(XFER_FULL_JSON), 200, false)); + requestor.queueResponse(FDv2Requestor.FDv2PayloadResponse.success(parseEvents(XFER_FULL_JSON), 200, false)); + FDv2PollingSynchronizer synchronizer = buildSynchronizer(requestor, 0, 10); + + try { + // Read the results of a successful poll, a failed poll, and two more successful polls. + FDv2SourceResult successBeforeFailure = synchronizer.next().get(1, TimeUnit.SECONDS); + FDv2SourceResult failure = synchronizer.next().get(1, TimeUnit.SECONDS); + FDv2SourceResult firstSuccessAfterFailure = synchronizer.next().get(1, TimeUnit.SECONDS); + FDv2SourceResult secondSuccessAfterFailure = synchronizer.next().get(1, TimeUnit.SECONDS); + + // The failure restarts the count, so the threshold is met on the second success after it. + assertFalse(successBeforeFailure.isResetThresholdMet()); + assertEquals(SourceSignal.INTERRUPTED, failure.getStatus().getState()); + assertFalse(failure.isResetThresholdMet()); + assertFalse(firstSuccessAfterFailure.isResetThresholdMet()); + assertTrue(secondSuccessAfterFailure.isResetThresholdMet()); + } finally { + synchronizer.close(); + } + } + // ---- close() ---- @Test From 61700726e1716956c6c2899c1daef9ed83388b06 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Fri, 6 Nov 2026 14:12:54 -0800 Subject: [PATCH 07/12] feat: Report the reset threshold from the FDv2 streaming synchronizer --- .../sdk/android/DataSystemComponents.java | 3 +- .../android/FDv2StreamingSynchronizer.java | 75 +++++++++++++ .../FDv2StreamingSynchronizerTest.java | 102 ++++++++++++++++-- 3 files changed, 173 insertions(+), 7 deletions(-) diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DataSystemComponents.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DataSystemComponents.java index 7e6c5ec1d..b0f57abb0 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DataSystemComponents.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/DataSystemComponents.java @@ -141,8 +141,9 @@ public Synchronizer build(DataSourceBuildInputs inputs) { StandardEndpoints.FDV2_STREAMING_REQUEST_BASE_PATH, requestor, initialReconnectDelayMillis, + FDv2StreamingSynchronizer.RESET_THRESHOLD_MILLIS, inputs.isEvaluationReasons(), inputs.getHttp().shouldUsePost(), - httpProps, inputs.getSharedExecutor(), + httpProps, inputs.getSharedExecutor(), inputs.getSharedExecutor(), inputs.getBaseLogger(), null); } } diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizer.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizer.java index c44e0f71d..b3db3bd3c 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizer.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizer.java @@ -22,6 +22,7 @@ import com.launchdarkly.sdk.android.subsystems.FDv2SourceResult; import com.launchdarkly.sdk.android.subsystems.Synchronizer; import com.launchdarkly.sdk.fdv2.ChangeSet; +import com.launchdarkly.sdk.fdv2.ChangeSetType; import com.launchdarkly.sdk.fdv2.Selector; import com.launchdarkly.sdk.internal.GsonHelpers; import com.launchdarkly.sdk.internal.events.DiagnosticStore; @@ -33,9 +34,12 @@ import com.launchdarkly.sdk.json.JsonSerialization; import java.io.IOException; import java.net.URI; +import java.util.Collections; import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -58,6 +62,11 @@ final class FDv2StreamingSynchronizer implements Synchronizer { private static final long READ_TIMEOUT_MS = 300_000; // 5 minutes private static final long MAX_RECONNECT_TIME_MS = 300_000; // 5 minutes + /** + * The time a stream must stay healthy, from its first valid result, to meet the reset threshold. + */ + static final long RESET_THRESHOLD_MILLIS = TimeUnit.SECONDS.toMillis(60); + private final HttpProperties httpProperties; private final URI streamBaseUri; private final String streamRequestPath; @@ -68,10 +77,12 @@ final class FDv2StreamingSynchronizer implements Synchronizer { private final FDv2Requestor requestor; private final boolean evaluationReasons; private final int initialReconnectDelayMillis; + private final long resetThresholdMillis; @Nullable private final DiagnosticStore diagnosticStore; private final LDLogger logger; private final Executor executor; + private final ScheduledExecutorService scheduler; private final LDAsyncQueue resultQueue = new LDAsyncQueue<>(); private final LDAwaitFuture shutdownFuture = new LDAwaitFuture<>(); @@ -84,6 +95,14 @@ final class FDv2StreamingSynchronizer implements Synchronizer { private volatile EventSource eventSource; private volatile long streamStarted = 0; + private final Object resetThresholdLock = new Object(); + + /** + * The timer that reports that the current stream met its reset threshold, or null if the current stream has + * not produced a valid result. Guarded by {@link #resetThresholdLock}. + */ + private ScheduledFuture resetThresholdFuture; + /** * @param evaluationContext the context to evaluate flags for * @param selectorSource source of the current selector, sent as the request basis @@ -91,11 +110,14 @@ final class FDv2StreamingSynchronizer implements Synchronizer { * @param streamRequestPath path appended to the base URI for the stream request * @param requestor optional requestor for handling ping events via poll; may be null * @param initialReconnectDelayMillis delay before reconnecting after an error, in milliseconds + * @param resetThresholdMillis time a stream must stay healthy to meet the reset threshold, in + * milliseconds * @param evaluationReasons true to request evaluation reasons in the stream * @param usePost true to send the context in the request body via HTTP POST * @param httpProperties HTTP configuration for the stream request * @param executor executor used to run the streaming loop on a background * thread; should use background-priority threads + * @param scheduler scheduler for the reset threshold timer * @param logger logger * @param diagnosticStore optional store for stream diagnostics; may be null */ @@ -106,10 +128,12 @@ final class FDv2StreamingSynchronizer implements Synchronizer { @NonNull String streamRequestPath, @Nullable FDv2Requestor requestor, int initialReconnectDelayMillis, + long resetThresholdMillis, boolean evaluationReasons, boolean usePost, @NonNull HttpProperties httpProperties, @NonNull Executor executor, + @NonNull ScheduledExecutorService scheduler, @NonNull LDLogger logger, @Nullable DiagnosticStore diagnosticStore ) { @@ -119,10 +143,12 @@ final class FDv2StreamingSynchronizer implements Synchronizer { this.streamRequestPath = streamRequestPath; this.requestor = requestor; this.initialReconnectDelayMillis = initialReconnectDelayMillis; + this.resetThresholdMillis = resetThresholdMillis; this.evaluationReasons = evaluationReasons; this.usePost = usePost; this.httpProperties = httpProperties; this.executor = executor; + this.scheduler = scheduler; this.logger = logger; this.diagnosticStore = diagnosticStore; } @@ -159,6 +185,7 @@ public void close() { } catch (IOException ignored) { } } + cancelResetThresholdTimer(); shutdownFuture.set(FDv2SourceResult.status(FDv2SourceResult.Status.shutdown(), false)); } @@ -230,6 +257,7 @@ private void startStream() { } } LDUtil.logExceptionAtErrorLevel(logger, e, "Stream thread ended with unexpected exception"); + cancelResetThresholdTimer(); recordStreamInit(true); resultQueue.put(FDv2SourceResult.status( FDv2SourceResult.Status.interrupted( @@ -260,6 +288,50 @@ private void recordStreamInit(boolean failed) { } } + /** + * Starts the reset threshold timer if the current stream does not have one yet. Call this for each + * valid result, so that the first valid result on a stream starts the timer. + */ + private void startResetThresholdTimerIfNeeded() { + synchronized (resetThresholdLock) { + if (resetThresholdFuture != null) { + return; + } + resetThresholdFuture = scheduler.schedule( + this::reportResetThresholdMet, + resetThresholdMillis, + TimeUnit.MILLISECONDS); + } + } + + /** + * Stops the reset threshold timer of the current stream. Call this when the stream ends. + */ + private void cancelResetThresholdTimer() { + synchronized (resetThresholdLock) { + if (resetThresholdFuture != null) { + resetThresholdFuture.cancel(false); + resetThresholdFuture = null; + } + } + } + + private void reportResetThresholdMet() { + synchronized (resetThresholdLock) { + // The timer can start to run before a cancel. Do not report anything for a stream that ended. + if (resetThresholdFuture == null) { + return; + } + ChangeSet> noChanges = new ChangeSet<>( + ChangeSetType.None, + Selector.EMPTY, + Collections.emptyMap(), + /* environmentId= */ null, + /* shouldPersist= */ true); + resultQueue.put(FDv2SourceResult.changeSet(noChanges, /* fdv1Fallback= */ false).withResetThresholdMet()); + } + } + private static boolean isFdv1Fallback(@Nullable ResponseHeaders headers) { if (headers == null) { return false; @@ -319,6 +391,7 @@ void handleMessage(MessageEvent event) { recordStreamInit(false); streamStarted = 0; resultQueue.put(FDv2SourceResult.changeSet(changeSet, fdv1Fallback)); + startResetThresholdTimerIfNeeded(); } catch (Exception e) { LDUtil.logExceptionAtErrorLevel(logger, e, "Failed to translate changeset"); FDv2SourceResult result = FDv2SourceResult.status(FDv2SourceResult.Status.interrupted( @@ -393,6 +466,7 @@ private void handleError(FaultEvent event) { return; } + cancelResetThresholdTimer(); recordStreamInit(true); protocolHandler.reset(); @@ -449,6 +523,7 @@ private void handleError(FaultEvent event) { * @param failed true if the restart is due to an error (for diagnostic recording) */ private void restartStream(boolean failed) { + cancelResetThresholdTimer(); recordStreamInit(failed); streamStarted = System.currentTimeMillis(); diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizerTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizerTest.java index 17937c138..9785a79b0 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizerTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizerTest.java @@ -3,6 +3,7 @@ import com.launchdarkly.logging.LDLogger; import com.launchdarkly.sdk.LDContext; import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.fdv2.ChangeSetType; import com.launchdarkly.sdk.fdv2.Selector; import com.launchdarkly.sdk.fdv2.SourceResultType; import com.launchdarkly.sdk.fdv2.SourceSignal; @@ -24,6 +25,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -39,10 +41,12 @@ public class FDv2StreamingSynchronizerTest { public Timeout globalTimeout = Timeout.seconds(10); private final ExecutorService executor = Executors.newCachedThreadPool(); + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); @After public void tearDown() { executor.shutdownNow(); + scheduler.shutdownNow(); } private static final LDContext CONTEXT = LDContext.create("test-context"); @@ -82,10 +86,12 @@ private FDv2StreamingSynchronizer makeSynchronizer( STREAM_PATH, null, 100, + FDv2StreamingSynchronizer.RESET_THRESHOLD_MILLIS, false, false, httpProperties(), executor, + scheduler, LOGGER, diagnosticStore); } @@ -95,8 +101,18 @@ private FDv2StreamingSynchronizer makeSynchronizer( boolean evaluationReasons, boolean usePost) { return new FDv2StreamingSynchronizer( CONTEXT, EMPTY_SELECTOR_SOURCE, streamBaseUri, STREAM_PATH, - null, 100, evaluationReasons, usePost, - httpProperties(), executor, LOGGER, null); + null, 100, FDv2StreamingSynchronizer.RESET_THRESHOLD_MILLIS, evaluationReasons, usePost, + httpProperties(), executor, scheduler, LOGGER, null); + } + + private FDv2StreamingSynchronizer makeSynchronizer( + URI streamBaseUri, + int initialReconnectDelayMillis, + long resetThresholdMillis) { + return new FDv2StreamingSynchronizer( + CONTEXT, EMPTY_SELECTOR_SOURCE, streamBaseUri, STREAM_PATH, + null, initialReconnectDelayMillis, resetThresholdMillis, false, false, + httpProperties(), executor, scheduler, LOGGER, null); } private static DiagnosticStore basicDiagnosticStore() { @@ -183,6 +199,80 @@ public void shutdownAfterEventReceived() throws Exception { } } + // ---- reset threshold ---- + + @Test + public void resetThresholdMetIsReportedAfterStreamStaysHealthy() throws Exception { + String serverIntent = makeEvent("server-intent", "{\"payloads\":[{\"id\":\"payload-1\",\"target\":100,\"intentCode\":\"xfer-full\",\"reason\":\"payload-missing\"}]}"); + String payloadTransferred = makeEvent("payload-transferred", "{\"state\":\"(p:payload-1:100)\",\"version\":100}"); + + try (HttpServer server = HttpServer.start(Handlers.all( + Handlers.SSE.start(), + Handlers.SSE.event(serverIntent), + Handlers.SSE.event(payloadTransferred), + Handlers.SSE.leaveOpen()))) { + + FDv2StreamingSynchronizer sync = makeSynchronizer(server.getUri(), 100, 100); + + try { + // Read the first valid result, then the result that follows while the stream stays open. + FDv2SourceResult changeSet = nextChangeSet(sync, "change set"); + FDv2SourceResult reset = sync.next().get(5, TimeUnit.SECONDS); + + // After the stream stays healthy for the threshold, it reports that the reset threshold was met, + // with no changes. + assertFalse(changeSet.isResetThresholdMet()); + assertEquals(SourceResultType.CHANGE_SET, reset.getResultType()); + assertEquals(ChangeSetType.None, reset.getChangeSet().getType()); + assertTrue(reset.isResetThresholdMet()); + } finally { + sync.close(); + } + } + } + + @Test + public void streamFailureCancelsPendingResetThresholdReport() throws Exception { + String serverIntent = makeEvent("server-intent", "{\"payloads\":[{\"id\":\"payload-1\",\"target\":100,\"intentCode\":\"xfer-full\",\"reason\":\"payload-missing\"}]}"); + String payloadTransferred = makeEvent("payload-transferred", "{\"state\":\"(p:payload-1:100)\",\"version\":100}"); + + try (HttpServer server = HttpServer.start(Handlers.sequential( + Handlers.all( + Handlers.SSE.start(), + Handlers.SSE.event(serverIntent), + Handlers.SSE.event(payloadTransferred)), + Handlers.all( + Handlers.SSE.start(), + Handlers.SSE.event(serverIntent), + Handlers.SSE.event(payloadTransferred), + Handlers.SSE.leaveOpen())))) { + + // The reconnect delay is longer than the reset threshold, even with jitter, so a reset from the + // first stream would arrive before the second stream connects. + FDv2StreamingSynchronizer sync = makeSynchronizer(server.getUri(), 1000, 250); + + try { + // Read the results of a stream that ends after one change set, then of a second stream that stays + // open. + FDv2SourceResult firstStreamChangeSet = sync.next().get(5, TimeUnit.SECONDS); + FDv2SourceResult failure = sync.next().get(5, TimeUnit.SECONDS); + FDv2SourceResult secondStreamChangeSet = sync.next().get(5, TimeUnit.SECONDS); + FDv2SourceResult reset = sync.next().get(5, TimeUnit.SECONDS); + + // The failure cancels the first stream's timer, so the only report follows the second stream's + // change set. + assertEquals(SourceResultType.CHANGE_SET, firstStreamChangeSet.getResultType()); + assertFalse(firstStreamChangeSet.isResetThresholdMet()); + assertEquals(SourceSignal.INTERRUPTED, failure.getStatus().getState()); + assertEquals(SourceResultType.CHANGE_SET, secondStreamChangeSet.getResultType()); + assertFalse(secondStreamChangeSet.isResetThresholdMet()); + assertTrue(reset.isResetThresholdMet()); + } finally { + sync.close(); + } + } + } + // ---- SSE events ---- @Test @@ -1131,8 +1221,8 @@ public void pingEventWithRequestorTriggersPoll() throws Exception { FDv2StreamingSynchronizer sync = new FDv2StreamingSynchronizer( CONTEXT, EMPTY_SELECTOR_SOURCE, streamServer.getUri(), STREAM_PATH, - requestor, 100, false, false, - httpProperties(), executor, LOGGER, null); + requestor, 100, FDv2StreamingSynchronizer.RESET_THRESHOLD_MILLIS, false, false, + httpProperties(), executor, scheduler, LOGGER, null); try { Future resultFuture = sync.next(); @@ -1171,8 +1261,8 @@ public void close() { FDv2StreamingSynchronizer sync = new FDv2StreamingSynchronizer( CONTEXT, EMPTY_SELECTOR_SOURCE, URI.create("http://localhost:9999/"), STREAM_PATH, - trackingRequestor, 100, false, false, - httpProperties(), executor, LOGGER, null); + trackingRequestor, 100, FDv2StreamingSynchronizer.RESET_THRESHOLD_MILLIS, false, false, + httpProperties(), executor, scheduler, LOGGER, null); assertFalse(requestorClosed.get()); sync.close(); From e1d8894e44fb43562426e510e9310d806836df6b Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Fri, 6 Nov 2026 14:13:33 -0800 Subject: [PATCH 08/12] feat: Report the reset threshold from the FDv1 fallback synchronizer --- .../sdk/android/FDv1PollingSynchronizer.java | 22 ++++++++ .../android/FDv1PollingSynchronizerTest.java | 50 +++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv1PollingSynchronizer.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv1PollingSynchronizer.java index cbdf7454a..21024de53 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv1PollingSynchronizer.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv1PollingSynchronizer.java @@ -31,6 +31,11 @@ * objects so it can be used as a drop-in synchronizer within the FDv2 data source pipeline. */ final class FDv1PollingSynchronizer implements Synchronizer { + /** + * The number of consecutive successful polls that meets the reset threshold, the same as for an FDv2 polling + * synchronizer. + */ + private static final int RESET_THRESHOLD_SUCCESSFUL_POLLS = 2; private final LDContext evaluationContext; private final FeatureFetcher fetcher; @@ -42,6 +47,12 @@ final class FDv1PollingSynchronizer implements Synchronizer { private volatile ScheduledFuture scheduledTask; private final Object taskLock = new Object(); + /** + * Only {@link #pollAndEnqueue()} reads or writes this. {@code scheduleWithFixedDelay} never starts a run of + * {@code pollAndEnqueue} before the previous run has finished. + */ + private int consecutiveSuccessfulPolls = 0; + /** * @param evaluationContext the context to evaluate flags for * @param fetcher the HTTP transport for FDv1 polling requests @@ -74,8 +85,18 @@ private void pollAndEnqueue() { try { FDv2SourceResult result = doPoll(); + if (result.getResultType() == SourceResultType.CHANGE_SET) { + consecutiveSuccessfulPolls++; + if (consecutiveSuccessfulPolls == RESET_THRESHOLD_SUCCESSFUL_POLLS) { + result = result.withResetThresholdMet(); + } + } + if (result.getResultType() == SourceResultType.STATUS) { FDv2SourceResult.Status status = result.getStatus(); + if (status != null && status.getState() == SourceSignal.INTERRUPTED) { + consecutiveSuccessfulPolls = 0; + } if (status != null && status.getState() == SourceSignal.TERMINAL_ERROR) { synchronized (taskLock) { if (scheduledTask != null) { @@ -91,6 +112,7 @@ private void pollAndEnqueue() { resultQueue.put(result); } catch (RuntimeException e) { LDUtil.logExceptionAtErrorLevel(logger, e, "Unexpected exception in FDv1 polling synchronizer task"); + consecutiveSuccessfulPolls = 0; resultQueue.put(FDv2SourceResult.status(FDv2SourceResult.Status.interrupted(e), false)); } } diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv1PollingSynchronizerTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv1PollingSynchronizerTest.java index 4c08f7691..5a018c37e 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv1PollingSynchronizerTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv1PollingSynchronizerTest.java @@ -118,6 +118,56 @@ public void successfulPollReturnsChangeSet() throws Exception { } } + @Test + public void resetThresholdMetIsReportedOnSecondConsecutiveSuccessfulPoll() throws Exception { + MockFetcher fetcher = new MockFetcher(); + for (int i = 0; i < 3; i++) { + fetcher.queueSuccess(VALID_FDV1_JSON); + } + + FDv1PollingSynchronizer sync = makeSynchronizer(fetcher, 0, 10); + try { + // Read the results of three consecutive successful polls. + FDv2SourceResult first = sync.next().get(5, TimeUnit.SECONDS); + FDv2SourceResult second = sync.next().get(5, TimeUnit.SECONDS); + FDv2SourceResult third = sync.next().get(5, TimeUnit.SECONDS); + + // Only the second successful poll reports that the reset threshold was met. + assertFalse(first.isResetThresholdMet()); + assertTrue(second.isResetThresholdMet()); + assertFalse(third.isResetThresholdMet()); + } finally { + sync.close(); + } + } + + @Test + public void failedPollRestartsResetThresholdCount() throws Exception { + MockFetcher fetcher = new MockFetcher(); + fetcher.queueSuccess(VALID_FDV1_JSON); + fetcher.queueError(new IOException("network error")); + fetcher.queueSuccess(VALID_FDV1_JSON); + fetcher.queueSuccess(VALID_FDV1_JSON); + + FDv1PollingSynchronizer sync = makeSynchronizer(fetcher, 0, 10); + try { + // Read the results of a successful poll, a failed poll, and two more successful polls. + FDv2SourceResult successBeforeFailure = sync.next().get(5, TimeUnit.SECONDS); + FDv2SourceResult failure = sync.next().get(5, TimeUnit.SECONDS); + FDv2SourceResult firstSuccessAfterFailure = sync.next().get(5, TimeUnit.SECONDS); + FDv2SourceResult secondSuccessAfterFailure = sync.next().get(5, TimeUnit.SECONDS); + + // The failure restarts the count, so the threshold is met on the second success after it. + assertFalse(successBeforeFailure.isResetThresholdMet()); + assertEquals(SourceSignal.INTERRUPTED, failure.getStatus().getState()); + assertFalse(failure.isResetThresholdMet()); + assertFalse(firstSuccessAfterFailure.isResetThresholdMet()); + assertTrue(secondSuccessAfterFailure.isResetThresholdMet()); + } finally { + sync.close(); + } + } + @Test public void nonRecoverableHttpErrorReturnsTerminalError() throws Exception { MockFetcher fetcher = new MockFetcher(); From e34527d29ca712ec57d36c6f56af54381ec9c78c Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Fri, 6 Nov 2026 14:14:37 -0800 Subject: [PATCH 09/12] fix: Treat an exception on the FDv2 stream thread as terminal --- .../android/FDv2StreamingSynchronizer.java | 5 ++-- .../FDv2StreamingSynchronizerTest.java | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizer.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizer.java index b3db3bd3c..5418aeb0a 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizer.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizer.java @@ -259,8 +259,9 @@ private void startStream() { LDUtil.logExceptionAtErrorLevel(logger, e, "Stream thread ended with unexpected exception"); cancelResetThresholdTimer(); recordStreamInit(true); - resultQueue.put(FDv2SourceResult.status( - FDv2SourceResult.Status.interrupted( + // The stream thread has ended and does not restart, so the failure is terminal. + shutdownFuture.set(FDv2SourceResult.status( + FDv2SourceResult.Status.terminalError( new LDFailure("Stream thread ended unexpectedly", e, LDFailure.FailureType.UNKNOWN_ERROR)), false)); diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizerTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizerTest.java index 9785a79b0..e9bc90004 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizerTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2StreamingSynchronizerTest.java @@ -273,6 +273,30 @@ public void streamFailureCancelsPendingResetThresholdReport() throws Exception { } } + // ---- unexpected exceptions ---- + + @Test + public void unexpectedExceptionOnStreamThreadIsTerminal() throws Exception { + try (HttpServer server = HttpServer.start(Handlers.SSE.start())) { + // The selector source throws when the stream connects, which ends the stream thread. + SelectorSource selectorSource = () -> { + throw new IllegalStateException("unexpected"); + }; + FDv2StreamingSynchronizer sync = makeSynchronizer(server.getUri(), selectorSource); + + try { + // Start the stream. + FDv2SourceResult result = sync.next().get(5, TimeUnit.SECONDS); + + // The stream thread cannot recover, so the failure is terminal for this synchronizer. + assertEquals(SourceResultType.STATUS, result.getResultType()); + assertEquals(SourceSignal.TERMINAL_ERROR, result.getStatus().getState()); + } finally { + sync.close(); + } + } + } + // ---- SSE events ---- @Test From 4bf11a3a93169b7e6c38bf296d5dcb7e6341820d Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Fri, 6 Nov 2026 14:27:01 -0800 Subject: [PATCH 10/12] test: Queue only new FDv2 data source states and errors in the mock sink --- .../sdk/android/FDv2DataSourceTest.java | 48 +++++++++---------- .../sdk/android/MockComponents.java | 6 ++- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java index e811d1c07..a9aacafdd 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java @@ -1922,7 +1922,14 @@ public void statusIncludesErrorInfoOnFailure() throws Exception { FDv2DataSource dataSource = buildDataSource(sink, Collections.emptyList(), Collections.singletonList(() -> new MockQueuedSynchronizer( - FDv2SourceResult.status(FDv2SourceResult.Status.terminalError(terminalErr), false)))); + FDv2SourceResult.status(FDv2SourceResult.Status.terminalError(terminalErr), false))), + null, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ TimeUnit.MINUTES.toMillis(5), + /* parkMaxDelayMillis= */ TimeUnit.HOURS.toMillis(1), + /* failoverDelayMillis= */ 1000, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); // Start the data source with a synchronizer that fails with a terminal error. startDataSource(dataSource); @@ -1930,7 +1937,7 @@ public void statusIncludesErrorInfoOnFailure() throws Exception { // The data source reports INTERRUPTED with the error from the synchronizer. assertEquals(DataSourceState.INTERRUPTED, status); - assertSame(terminalErr, sink.statusEvents.get(0).error); + assertSame(terminalErr, sink.getLastError()); stopDataSource(dataSource); } @@ -2140,10 +2147,10 @@ public void fdv1FallbackSwitchesToFdv1Synchronizer() throws Exception { AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); - assertEquals(DataSourceState.VALID, sink.awaitStatus(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); - - DataSourceState secondValid = sink.awaitStatus(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertEquals(DataSourceState.VALID, secondValid); + // The FDv2 payload and then the FDv1 payload are applied. + sink.awaitApplyCount(2, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); + assertEquals(DataSourceState.VALID, sink.getLastState()); stopDataSource(dataSource); } @@ -2209,8 +2216,9 @@ public void fdv1FallbackNotTriggeredWhenAlreadyOnFdv1() throws Exception { AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); - assertEquals(DataSourceState.VALID, sink.awaitStatus(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); - assertEquals(DataSourceState.VALID, sink.awaitStatus(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + // The FDv2 payload and then the FDv1 payload are applied. + sink.awaitApplyCount(2, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); assertEquals(1, fdv1BuildCount.get()); @@ -2284,14 +2292,11 @@ public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredRecoversAfterTTL() th AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); - // Both payloads report VALID. The directive and the wait for the TTL report INTERRUPTED. The first - // synchronizer restarts after the TTL and reports VALID. - List statuses = sink.awaitStatuses(5, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + // The data source reports INTERRUPTED, and restarts the first synchronizer after the TTL. + List statuses = sink.awaitStatuses(3, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); assertEquals(Arrays.asList( - DataSourceState.VALID, DataSourceState.VALID, DataSourceState.INTERRUPTED, - DataSourceState.INTERRUPTED, DataSourceState.VALID), statuses); assertEquals(2, firstSyncBuildCount.get()); assertFalse(secondSyncCalled.get()); @@ -2398,12 +2403,9 @@ public void fdv1FallbackEndsWhileFDv1SynchronizerIsParked() throws Exception { assertEquals(1, sink.getApplyCount()); assertEquals(2, fdv2BuildCount.get()); - // The interruption, the FDv1 failure, and the wait for the TTL report INTERRUPTED. The FDv2 payload reports - // VALID. Its directive falls back to the FDv1 synchronizer, which is still parked, so nothing runs until - // the TTL, and the wait reports INTERRUPTED. - List statuses = sink.awaitStatuses(5, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + // Each directive falls back to the FDv1 synchronizer, which is still parked, so nothing runs until the TTL. + List statuses = sink.awaitStatuses(4, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); assertEquals(Arrays.asList( - DataSourceState.INTERRUPTED, DataSourceState.INTERRUPTED, DataSourceState.INTERRUPTED, DataSourceState.VALID, @@ -2529,12 +2531,10 @@ public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredRecoversAfterTTL() thr AwaitableCallback startCallback = startDataSource(dataSource); assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); - // The directive and the wait for the TTL report INTERRUPTED, and the synchronizer starts after the TTL. - List statuses = sink.awaitStatuses(3, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertEquals(Arrays.asList( - DataSourceState.INTERRUPTED, - DataSourceState.INTERRUPTED, - DataSourceState.VALID), statuses); + // The data source never reported OFF, and started the synchronizer after the TTL. + List statuses = sink.awaitStatuses(2, AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertFalse(statuses.contains(DataSourceState.OFF)); + assertEquals(DataSourceState.VALID, sink.getLastState()); assertEquals(1, syncBuildCount.get()); stopDataSource(dataSource); } diff --git a/shared-test-code/src/main/java/com/launchdarkly/sdk/android/MockComponents.java b/shared-test-code/src/main/java/com/launchdarkly/sdk/android/MockComponents.java index c22be1ad0..4067467d7 100644 --- a/shared-test-code/src/main/java/com/launchdarkly/sdk/android/MockComponents.java +++ b/shared-test-code/src/main/java/com/launchdarkly/sdk/android/MockComponents.java @@ -134,8 +134,12 @@ public void setStatus(@NonNull ConnectionInformation.ConnectionMode connectionMo @Override public void setStatus(@NonNull DataSourceState state, @Nullable Throwable failure) { + // Queue only a new state or a new error. statusEvents records every status, including repeats. + DataSourceState oldState = getLastState(); statusEvents.add(new StatusEvent(state, failure)); - statusUpdates.offer(state); + if (state != oldState || failure != null) { + statusUpdates.offer(state); + } } public volatile boolean shutDownCalled = false; From d6270ea24d4b1b9db28d05f927bdb48f40fdf9e4 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Fri, 6 Nov 2026 14:44:22 -0800 Subject: [PATCH 11/12] fix: Signal FDv2 readiness on an initializer's FDv1 directive only when its payload has a selector --- .../sdk/android/FDv2DataSource.java | 10 ++-- .../sdk/android/FDv2DataSourceTest.java | 47 +++++++++++++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java index 099f8002e..7fb22b903 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/FDv2DataSource.java @@ -434,13 +434,13 @@ private void runGeneralInitializers( ChangeSet> changeSet = result.getChangeSet(); if (changeSet != null) { sink.apply(context, changeSet); - anyDataReceived = true; + // Only a payload with a selector that comes with the directive makes the data source ready. + if (!changeSet.getSelector().isEmpty()) { + sink.setStatus(DataSourceState.VALID, null); + tryCompleteStart(true, null); + } } } - if (anyDataReceived) { - sink.setStatus(DataSourceState.VALID, null); - tryCompleteStart(true, null); - } fallBackToFDv1( sink, jittered(defaultFDv1FallbackTtlMillis), diff --git a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java index a9aacafdd..a64af1c94 100644 --- a/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java +++ b/launchdarkly-android-client-sdk/src/test/java/com/launchdarkly/sdk/android/FDv2DataSourceTest.java @@ -41,6 +41,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -2125,6 +2126,52 @@ public void fdv1FallbackDuringInitializationWithNonEmptySelectorSwitchesToFdv1() stopDataSource(dataSource); } + // A payload without a selector that comes with the directive is applied, but it does not make the data source + // ready. The data from the FDv1 fallback synchronizer does. + @Test + public void fdv1FallbackDuringInitializationWithoutSelectorWaitsForFdv1Data() throws Exception { + MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); + CountDownLatch fdv1Built = new CountDownLatch(1); + MockQueuedSynchronizer fdv1Sync = new MockQueuedSynchronizer(); + + // The initializer returns a payload without a selector, with the directive. The FDv1 fallback synchronizer + // sends the results that the test gives it. + FDv2DataSource dataSource = buildDataSource(sink, + Collections.singletonList(() -> new MockInitializer( + FDv2SourceResult.changeSet(makeChangeSet(false), true))), + Collections.singletonList(() -> new MockQueuedSynchronizer()), + () -> { + fdv1Built.countDown(); + return fdv1Sync; + }, + 120_000, + 300_000, + /* parkInitialDelayMillis= */ TimeUnit.MINUTES.toMillis(5), + /* parkMaxDelayMillis= */ TimeUnit.HOURS.toMillis(1), + /* failoverDelayMillis= */ 10, + /* fdv1FallbackTtlMillis= */ TimeUnit.HOURS.toMillis(1)); + + // Start the data source, and wait until it starts the FDv1 fallback synchronizer. + AwaitableCallback startCallback = startDataSource(dataSource); + assertTrue(fdv1Built.await(AWAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + + // The payload is applied, but the data source is not ready and has not reported VALID. + assertEquals(1, sink.getApplyCount()); + try { + startCallback.await(0); + fail("Expected the data source not to be ready"); + } catch (TimeoutException expected) { + // The data source is not ready. + } + assertFalse(sink.statusEvents.stream().anyMatch(event -> event.state == DataSourceState.VALID)); + + // The data from the FDv1 fallback synchronizer makes the data source ready. + fdv1Sync.addResult(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + assertTrue(startCallback.await(AWAIT_TIMEOUT_SECONDS * 1000)); + assertEquals(DataSourceState.VALID, sink.getLastState()); + stopDataSource(dataSource); + } + @Test public void fdv1FallbackSwitchesToFdv1Synchronizer() throws Exception { MockComponents.MockDataSourceUpdateSink sink = new MockComponents.MockDataSourceUpdateSink(); From 1c5f1342fce95c458501f7b05b7cffd03d609c46 Mon Sep 17 00:00:00 2001 From: Bee Klimt Date: Fri, 6 Nov 2026 14:57:19 -0800 Subject: [PATCH 12/12] docs: Mark FDv2SourceResult as not stable and in early access --- .../launchdarkly/sdk/android/subsystems/FDv2SourceResult.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/FDv2SourceResult.java b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/FDv2SourceResult.java index 4d0d2d1ee..263ab47c7 100644 --- a/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/FDv2SourceResult.java +++ b/launchdarkly-android-client-sdk/src/main/java/com/launchdarkly/sdk/android/subsystems/FDv2SourceResult.java @@ -13,6 +13,9 @@ /** * Result type for FDv2 initializers and synchronizers. An initializer produces a single result; * a synchronizer produces a stream of results. + *

+ * This class is not stable, and not subject to any backwards compatibility guarantees or semantic versioning. + * It is in early access. If you want access to this feature please join the EAP. https://launchdarkly.com/docs/sdk/features/data-saving-mode * * @see Initializer * @see Synchronizer