diff --git a/contract-tests/async_client_entity.py b/contract-tests/async_client_entity.py index 2569a156..d45f9072 100644 --- a/contract-tests/async_client_entity.py +++ b/contract-tests/async_client_entity.py @@ -26,6 +26,7 @@ ) from ldclient.impl.util import Result from ldclient.integrations import Redis +from ldclient.integrations.overrides import FileOverrideSourceBuilder from ldclient.interfaces import DataStoreMode from ldclient.migrations import ( AsyncMigratorBuilder, @@ -56,8 +57,10 @@ async def start(self): } datasystem_config = config_params.get('dataSystem') + if config_params.get('overrides') is not None and datasystem_config is None: + raise ValueError("flag overrides require the data system to be configured") if datasystem_config is not None: - opts["datasystem_config"] = _build_async_data_system(datasystem_config, opts) + opts["datasystem_config"] = _build_async_data_system(datasystem_config, opts, config_params.get('overrides')) elif config_params.get("streaming") is not None: streaming = config_params["streaming"] if streaming.get("baseUri") is not None: @@ -297,13 +300,13 @@ def _set_optional_value(params_in: dict, name_in: str, func: Callable[[Any], Any func(params_in[name_in]) -def _build_async_data_system(datasystem_config: dict, opts: dict) -> AsyncDataSystemConfig: +def _build_async_data_system(datasystem_config: dict, opts: dict, overrides_config: Optional[dict] = None) -> AsyncDataSystemConfig: """Build an AsyncDataSystemConfig from the harness's dataSystem config. Wires the FDv2 initializers, the ordered synchronizer chain, the FDv1 - fallback synchronizer, the payload filter, and an optional async - persistent store. The async client injects its shared aiohttp session - into these builders when it starts. + fallback synchronizer, the payload filter, an optional async persistent + store, and the optional file-based override source. The async client + injects its shared aiohttp session into these builders when it starts. """ initializers: Optional[list] = None init_configs = datasystem_config.get('initializers') @@ -356,6 +359,9 @@ def _build_async_data_system(datasystem_config: dict, opts: dict) -> AsyncDataSy "fdv1_fallback_synchronizer": fdv1_fallback_synchronizer, } + if overrides_config is not None: + ds_kwargs["override_source"] = _make_override_source(overrides_config) + store_config = datasystem_config.get("store") if store_config is not None: persistent_store_config = store_config.get("persistentDataStore") @@ -370,6 +376,18 @@ def _build_async_data_system(datasystem_config: dict, opts: dict) -> AsyncDataSy return AsyncDataSystemConfig(**ds_kwargs) +def _make_override_source(params: dict) -> FileOverrideSourceBuilder: + """ + Builds the file-based override source from the harness's overrides configuration. The + files are written by the harness on a file system shared with this service. + """ + builder = FileOverrideSourceBuilder(params["filePaths"]) + _set_optional_value(params, "duplicateKeysHandling", builder.duplicate_keys_handling) + _set_optional_value(params, "changeDetection", builder.change_detection) + _set_optional_time(params, "pollIntervalMs", builder.poll_interval) + return builder + + def _create_async_persistent_store(persistent_store_config: dict): """Create an async persistent feature store from the harness config. diff --git a/contract-tests/async_service.py b/contract-tests/async_service.py index b3c04d8a..6a11abc0 100644 --- a/contract-tests/async_service.py +++ b/contract-tests/async_service.py @@ -69,6 +69,7 @@ async def handle_status(request: aiohttp.web.Request) -> aiohttp.web.Response: 'migrations', 'persistent-data-store-redis', 'fdv1-fallback', + 'flag-overrides', ] } return aiohttp.web.Response( diff --git a/contract-tests/client_entity.py b/contract-tests/client_entity.py index f93b5d85..c02fc792 100644 --- a/contract-tests/client_entity.py +++ b/contract-tests/client_entity.py @@ -28,6 +28,7 @@ from ldclient.feature_store import CacheConfig from ldclient.impl.datasourcev2.polling import PollingDataSourceBuilder from ldclient.integrations import Consul, DynamoDB, Redis +from ldclient.integrations.overrides import FileOverrideSourceBuilder from ldclient.interfaces import DataStoreMode @@ -44,6 +45,8 @@ def __init__(self, tag, config): } datasystem_config = config.get('dataSystem') + if config.get('overrides') is not None and datasystem_config is None: + raise ValueError("flag overrides require the data system to be configured") if datasystem_config is not None: datasystem = custom() @@ -107,6 +110,10 @@ def __init__(self, tag, config): datasystem.data_store(store, store_mode) + overrides_config = config.get('overrides') + if overrides_config is not None: + datasystem.overrides(_make_override_source(overrides_config)) + opts["datasystem_config"] = datasystem.build() elif config.get("streaming") is not None: @@ -325,6 +332,18 @@ def _set_optional_value(params_in: dict, name_in: str, func: Callable[[Any], Any func(params_in[name_in]) +def _make_override_source(params: dict) -> FileOverrideSourceBuilder: + """ + Builds the file-based override source from the harness's overrides configuration. The + files are written by the harness on a file system shared with this service. + """ + builder = FileOverrideSourceBuilder(params["filePaths"]) + _set_optional_value(params, "duplicateKeysHandling", builder.duplicate_keys_handling) + _set_optional_value(params, "changeDetection", builder.change_detection) + _set_optional_time(params, "pollIntervalMs", builder.poll_interval) + return builder + + def _create_persistent_store(persistent_store_config: dict): """ Creates a persistent store instance based on the configuration. diff --git a/contract-tests/service.py b/contract-tests/service.py index a8e93674..2583748e 100644 --- a/contract-tests/service.py +++ b/contract-tests/service.py @@ -86,6 +86,7 @@ def status(): 'flag-change-listeners', 'flag-value-change-listeners', 'fdv1-fallback', + 'flag-overrides', ] } return json.dumps(body), 200, {'Content-type': 'application/json'} diff --git a/docs/api-integrations.rst b/docs/api-integrations.rst index 8d8146ff..594d7ed8 100644 --- a/docs/api-integrations.rst +++ b/docs/api-integrations.rst @@ -8,3 +8,13 @@ ldclient.integrations module :members: :special-members: __init__ :show-inheritance: + +ldclient.integrations.overrides module +-------------------------------------- + +The entry point for this feature is :class:`ldclient.integrations.overrides.FileOverrideSourceBuilder`. + +.. automodule:: ldclient.integrations.overrides + :members: + :special-members: __init__ + :show-inheritance: diff --git a/ldclient/async_client.py b/ldclient/async_client.py index b741aa17..11c1461a 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -4,7 +4,7 @@ import asyncio import traceback -from typing import Any, Callable, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple from uuid import uuid4 from ldclient.async_config import AsyncConfig @@ -28,7 +28,7 @@ ) from ldclient.impl.client_common import secure_mode_hash as _secure_mode_hash from ldclient.impl.datasystem import AsyncDataSystem, DataAvailability -from ldclient.impl.evaluator_common import error_reason +from ldclient.impl.evaluator_common import error_reason, is_override_definition from ldclient.impl.events.async_event_processor import ( DefaultAsyncEventProcessor ) @@ -95,6 +95,7 @@ def __init__(self, config: AsyncConfig): # these flags, so plain booleans are safe. self._eval_cached_data_warned = False self._all_flags_cached_data_warned = False + self._all_flags_overrides_only_warned = False # Build the object graph here (loop-free). start() supplies the loop-bound # resources: the HTTP session (created lazily), the data source, the @@ -485,7 +486,12 @@ async def evaluate(): tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker}) - detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE')) + # The type mismatch replaces the reason. The evaluation read the same definitions, + # so the new reason keeps the override-affected marking. + reason = error_reason('WRONG_TYPE') + if detail.reason.get('overrideAffected') is True: + reason['overrideAffected'] = True + detail = EvaluationDetail(default_stage.value, None, reason) tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker}) @@ -498,6 +504,13 @@ async def _evaluate_internal(self, key: str, context: Context, default: Any, eve if self._config.offline: return EvaluationDetail(default, None, error_reason('CLIENT_NOT_READY')), None + def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: + log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key) + reason = error_reason('CLIENT_NOT_READY') + self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) + return EvaluationDetail(default, None, reason), None + + flag: Any = None availability = await self._data_system.data_availability() if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -505,23 +518,28 @@ async def _evaluate_internal(self, key: str, context: Context, default: Any, eve self._eval_cached_data_warned = True log.warning("Feature Flag evaluation attempted before client has initialized - using last known values from feature store for feature key: " + key + ". This message is logged once.") else: - log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key) - reason = error_reason('CLIENT_NOT_READY') - self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) - return EvaluationDetail(default, None, reason), None + # No data from LaunchDarkly is available. A flag that the override layer holds + # is served anyway. A flag the layer does not hold gets the not-ready default, + # as it does without an override source. + layer = self._data_system.override_layer + if layer is not None: + flag = layer.get(FEATURES, key) + if flag is None: + return not_ready() if not context.valid: log.warning("Context was invalid for flag evaluation (%s); returning default value" % context.error) return EvaluationDetail(default, None, error_reason('USER_NOT_SPECIFIED')), None - try: - flag = await self._data_system.store.get(FEATURES, key) - except Exception as e: - log.error("Unexpected error while retrieving feature flag \"%s\": %s" % (key, repr(e))) - log.debug(traceback.format_exc()) - reason = error_reason('EXCEPTION') - self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) - return EvaluationDetail(default, None, reason), None + if flag is None: + try: + flag = await self._data_system.store.get(FEATURES, key) + except Exception as e: + log.error("Unexpected error while retrieving feature flag \"%s\": %s" % (key, repr(e))) + log.debug(traceback.format_exc()) + reason = error_reason('EXCEPTION') + self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) + return EvaluationDetail(default, None, reason), None if not flag: reason = error_reason('FLAG_NOT_FOUND') self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) @@ -534,13 +552,19 @@ async def _evaluate_internal(self, key: str, context: Context, default: Any, eve detail = result.detail if detail.is_default_value(): detail = EvaluationDetail(default, None, detail.reason) - self._send_event(event_factory.new_eval_event(flag, context, detail, default)) + self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected)) return detail, flag except Exception as e: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = error_reason('EXCEPTION') - self._send_event(event_factory.new_default_event(flag, context, default, reason)) + # The evaluation read the flag definition before it failed. A definition from + # the override layer marks the result, so the failure is reported in the + # summary counters only, like any other override-affected evaluation. + override_affected = is_override_definition(flag) + if override_affected: + reason['overrideAffected'] = True + self._send_event(event_factory.new_default_event(flag, context, default, reason, override_affected)) return EvaluationDetail(default, None, reason), flag async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: @@ -572,6 +596,7 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState log.warning("all_flags_state() called, but client is in offline mode. Returning empty state") return FeatureFlagsState(False) + override_flags: Optional[Dict[str, Any]] = None availability = await self._data_system.data_availability() if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -579,25 +604,49 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState self._all_flags_cached_data_warned = True log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.") else: - log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") - return FeatureFlagsState(False) + # No data from LaunchDarkly is available. When the override layer holds flags, + # the state contains only those, read from the layer itself: a store that is not + # initialized can still hold LaunchDarkly data, which the state must not include. + # Without override flags the state is unavailable, as it is without an override + # source. + layer = self._data_system.override_layer + override_flags = layer.all(FEATURES) if layer is not None else {} + if len(override_flags) == 0: + log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") + return FeatureFlagsState(False) if not context.valid: log.warning("Context was invalid for all_flags_state (%s); returning default value" % context.error) return FeatureFlagsState(False) - state = FeatureFlagsState(True) + valid = True client_only = kwargs.get('client_side_only', False) with_reasons = kwargs.get('with_reasons', False) details_only_if_tracked = kwargs.get('details_only_for_tracked_flags', False) - try: - flags_map = await self._data_system.store.all(FEATURES) - if flags_map is None: - raise ValueError("feature store error") - except Exception as e: - log.error("Unable to read flags for all_flag_state: %s" % repr(e)) - return FeatureFlagsState(False) - + if override_flags is not None: + if not self._all_flags_overrides_only_warned: + self._all_flags_overrides_only_warned = True + log.warning("all_flags_state() called before client has finished initializing! Returning only flags from the override layer. This message is logged once.") + flags_map = override_flags + else: + try: + flags_map = await self._data_system.store.all(FEATURES) + if flags_map is None: + raise ValueError("feature store error") + except Exception as e: + log.error("Unable to read flags for all_flag_state: %s" % repr(e)) + # Overrides only add to the state. Without them the state is invalid and empty + # here, so it holds the override layer's flags and stays invalid. A client-side + # SDK bootstrapped from the state uses its flags whatever $valid says (the + # JavaScript client SDKs 3.x and 4.x only warn), so the overridden values still + # reach it. + layer = self._data_system.override_layer + flags_map = layer.all(FEATURES) if layer is not None else {} + if len(flags_map) == 0: + return FeatureFlagsState(False) + valid = False + + state = FeatureFlagsState(valid) for key, flag in flags_map.items(): if client_only and not flag.get('clientSide', False): continue @@ -605,13 +654,29 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState result = await self._evaluator.evaluate(flag, context, self._event_factory_default) detail = result.detail prerequisites = result.prerequisites + override_affected = result.override_affected except Exception as e: log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e))) log.debug(traceback.format_exc()) - reason = {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + reason = error_reason('EXCEPTION') + # The evaluation read the flag definition before it failed, so a definition from + # the override layer marks the result. + override_affected = is_override_definition(flag) + if override_affected: + reason['overrideAffected'] = True detail = EvaluationDetail(None, None, reason) prerequisites = [] requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) + track_events = flag.get('trackEvents', False) or requires_experiment_data + track_reason = requires_experiment_data + debug_events_until_date = flag.get('debugEventsUntilDate', None) + if override_affected: + # A consumer of this state sends individual events according to these fields. + # An override-affected evaluation produces no individual events, so the state + # turns them off for this flag. The flag, its value, and its reason stay. + track_events = False + track_reason = False + debug_events_until_date = None flag_state = { 'key': flag['key'], 'value': detail.value, @@ -619,9 +684,9 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState 'reason': detail.reason, 'version': flag['version'], 'prerequisites': prerequisites, - 'trackEvents': flag.get('trackEvents', False) or requires_experiment_data, - 'trackReason': requires_experiment_data, - 'debugEventsUntilDate': flag.get('debugEventsUntilDate', None), + 'trackEvents': track_events, + 'trackReason': track_reason, + 'debugEventsUntilDate': debug_events_until_date, } state.add_flag(flag_state, with_reasons, details_only_if_tracked) diff --git a/ldclient/async_config.py b/ldclient/async_config.py index b46b362e..c64d0689 100644 --- a/ldclient/async_config.py +++ b/ldclient/async_config.py @@ -21,6 +21,7 @@ DataSourceBuilder, DataSourceBuilderConfig, HTTPConfig, + OverrideSourceBuilder, PrivateAttributesConfig ) from ldclient.hook import AsyncHook @@ -127,6 +128,16 @@ class AsyncDataSystemConfig: fdv1_fallback_synchronizer: Optional[DataSourceBuilder[AsyncSynchronizer]] = None """An optional fallback synchronizer that will read from FDv1""" + override_source: Optional[OverrideSourceBuilder] = None + """ + An optional override source. Its flag and segment definitions take precedence over + LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject + to change. + + When the client is offline, the source is not built or started: no overrides are served, and + a configuration error in the source is not reported. The SDK logs a warning instead. + """ + class AsyncConfig(DataSourceBuilderConfig, PrivateAttributesConfig): """Advanced configuration options for the async SDK client. diff --git a/ldclient/client.py b/ldclient/client.py index b4460bc0..e670f04e 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -28,6 +28,7 @@ from ldclient.impl.datasystem import DataAvailability, DataSystem from ldclient.impl.datasystem.fdv2 import FDv2 from ldclient.impl.evaluator import Evaluator, error_reason +from ldclient.impl.evaluator_common import is_override_definition from ldclient.impl.events.diagnostics import ( _DiagnosticAccumulator, create_diagnostic_id @@ -89,6 +90,7 @@ def __init__(self, config: Config, start_wait: float = 5): self._cached_data_warning_lock = threading.Lock() self._eval_cached_data_warned = False self._all_flags_cached_data_warned = False + self._all_flags_overrides_only_warned = False self._owner_pid = os.getpid() self._fork_warned_pids: Dict[int, object] = {} @@ -211,7 +213,13 @@ def __start_up(self, start_wait: float): self.__register_plugins(environment_metadata) update_processor_ready = threading.Event() - self._data_system.start(update_processor_ready) + try: + self._data_system.start(update_processor_ready) + except BaseException: + # A data system that fails to start fails the constructor. The components that + # are already running stop here, so the failure leaves no threads behind. + self._close_components() + raise if not self._config.offline and not self._config.use_ldd: if start_wait > 60: @@ -281,6 +289,15 @@ def close(self): self._data_system.stop() self.__big_segment_store_manager.stop() + def _close_components(self): + """ + Stops the SDK components after a failed start-up. The event processor is last because + it may still be sending events the other components generated. + """ + self._data_system.stop() + self.__big_segment_store_manager.stop() + self._event_processor.stop() + # These magic methods allow a client object to be automatically cleaned up by the "with" scope operator def __enter__(self): return self @@ -454,7 +471,12 @@ def evaluate(): tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker}) - detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE')) + # The type mismatch replaces the reason. The evaluation read the same definitions, + # so the new reason keeps the override-affected marking. + reason = error_reason('WRONG_TYPE') + if detail.reason.get('overrideAffected') is True: + reason['overrideAffected'] = True + detail = EvaluationDetail(default_stage.value, None, reason) tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker}) @@ -467,6 +489,13 @@ def _evaluate_internal(self, key: str, context: Context, default: Any, event_fac if self._config.offline: return EvaluationDetail(default, None, error_reason('CLIENT_NOT_READY')), None + def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: + log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key) + reason = error_reason('CLIENT_NOT_READY') + self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) + return EvaluationDetail(default, None, reason), None + + flag: Any = None availability = self._data_system.data_availability if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -476,23 +505,28 @@ def _evaluate_internal(self, key: str, context: Context, default: Any, event_fac self._eval_cached_data_warned = True log.warning("Feature Flag evaluation attempted before client has initialized - using last known values from feature store for feature key: " + key + ". This message is logged once.") else: - log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key) - reason = error_reason('CLIENT_NOT_READY') - self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) - return EvaluationDetail(default, None, reason), None + # No data from LaunchDarkly is available. A flag that the override layer holds + # is served anyway. A flag the layer does not hold gets the not-ready default, + # as it does without an override source. + layer = self._data_system.override_layer + if layer is not None: + flag = layer.get(FEATURES, key) + if flag is None: + return not_ready() if not context.valid: log.warning("Context was invalid for flag evaluation (%s); returning default value" % context.error) return EvaluationDetail(default, None, error_reason('USER_NOT_SPECIFIED')), None - try: - flag = self._data_system.store.get(FEATURES, key) - except Exception as e: - log.error("Unexpected error while retrieving feature flag \"%s\": %s" % (key, repr(e))) - log.debug(traceback.format_exc()) - reason = error_reason('EXCEPTION') - self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) - return EvaluationDetail(default, None, reason), None + if flag is None: + try: + flag = self._data_system.store.get(FEATURES, key) + except Exception as e: + log.error("Unexpected error while retrieving feature flag \"%s\": %s" % (key, repr(e))) + log.debug(traceback.format_exc()) + reason = error_reason('EXCEPTION') + self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) + return EvaluationDetail(default, None, reason), None if not flag: reason = error_reason('FLAG_NOT_FOUND') self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) @@ -505,13 +539,19 @@ def _evaluate_internal(self, key: str, context: Context, default: Any, event_fac detail = result.detail if detail.is_default_value(): detail = EvaluationDetail(default, None, detail.reason) - self._send_event(event_factory.new_eval_event(flag, context, detail, default)) + self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected)) return detail, flag except Exception as e: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = error_reason('EXCEPTION') - self._send_event(event_factory.new_default_event(flag, context, default, reason)) + # The evaluation read the flag definition before it failed. A definition from + # the override layer marks the result, so the failure is reported in the + # summary counters only, like any other override-affected evaluation. + override_affected = is_override_definition(flag) + if override_affected: + reason['overrideAffected'] = True + self._send_event(event_factory.new_default_event(flag, context, default, reason, override_affected)) return EvaluationDetail(default, None, reason), flag def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: @@ -545,6 +585,7 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: self._check_forked() + override_flags: Optional[Dict[str, Any]] = None availability = self._data_system.data_availability if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -554,25 +595,51 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: self._all_flags_cached_data_warned = True log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.") else: - log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") - return FeatureFlagsState(False) + # No data from LaunchDarkly is available. When the override layer holds flags, + # the state contains only those, read from the layer itself: a store that is not + # initialized can still hold LaunchDarkly data, which the state must not include. + # Without override flags the state is unavailable, as it is without an override + # source. + layer = self._data_system.override_layer + override_flags = layer.all(FEATURES) if layer is not None else {} + if len(override_flags) == 0: + log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") + return FeatureFlagsState(False) if not context.valid: log.warning("Context was invalid for all_flags_state (%s); returning default value" % context.error) return FeatureFlagsState(False) - state = FeatureFlagsState(True) + valid = True client_only = kwargs.get('client_side_only', False) with_reasons = kwargs.get('with_reasons', False) details_only_if_tracked = kwargs.get('details_only_for_tracked_flags', False) - try: - flags_map = self._data_system.store.all(FEATURES, lambda x: x) - if flags_map is None: - raise ValueError("feature store error") - except Exception as e: - log.error("Unable to read flags for all_flag_state: %s" % repr(e)) - return FeatureFlagsState(False) - + if override_flags is not None: + if not self._all_flags_overrides_only_warned: + with self._cached_data_warning_lock: + if not self._all_flags_overrides_only_warned: + self._all_flags_overrides_only_warned = True + log.warning("all_flags_state() called before client has finished initializing! Returning only flags from the override layer. This message is logged once.") + flags_map = override_flags + else: + try: + flags_map = self._data_system.store.all(FEATURES, lambda x: x) + if flags_map is None: + raise ValueError("feature store error") + except Exception as e: + log.error("Unable to read flags for all_flag_state: %s" % repr(e)) + # Overrides only add to the state. Without them the state is invalid and empty + # here, so it holds the override layer's flags and stays invalid. A client-side + # SDK bootstrapped from the state uses its flags whatever $valid says (the + # JavaScript client SDKs 3.x and 4.x only warn), so the overridden values still + # reach it. + layer = self._data_system.override_layer + flags_map = layer.all(FEATURES) if layer is not None else {} + if len(flags_map) == 0: + return FeatureFlagsState(False) + valid = False + + state = FeatureFlagsState(valid) for key, flag in flags_map.items(): if client_only and not flag.get('clientSide', False): continue @@ -580,14 +647,30 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: result = self._evaluator.evaluate(flag, context, self._event_factory_default) detail = result.detail prerequisites = result.prerequisites + override_affected = result.override_affected except Exception as e: log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e))) log.debug(traceback.format_exc()) - reason = {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + reason = error_reason('EXCEPTION') + # The evaluation read the flag definition before it failed, so a definition from + # the override layer marks the result. + override_affected = is_override_definition(flag) + if override_affected: + reason['overrideAffected'] = True detail = EvaluationDetail(None, None, reason) prerequisites = [] requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) + track_events = flag.get('trackEvents', False) or requires_experiment_data + track_reason = requires_experiment_data + debug_events_until_date = flag.get('debugEventsUntilDate', None) + if override_affected: + # A consumer of this state sends individual events according to these fields. + # An override-affected evaluation produces no individual events, so the state + # turns them off for this flag. The flag, its value, and its reason stay. + track_events = False + track_reason = False + debug_events_until_date = None flag_state = { 'key': flag['key'], 'value': detail.value, @@ -595,9 +678,9 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: 'reason': detail.reason, 'version': flag['version'], 'prerequisites': prerequisites, - 'trackEvents': flag.get('trackEvents', False) or requires_experiment_data, - 'trackReason': requires_experiment_data, - 'debugEventsUntilDate': flag.get('debugEventsUntilDate', None), + 'trackEvents': track_events, + 'trackReason': track_reason, + 'debugEventsUntilDate': debug_events_until_date, } state.add_flag(flag_state, with_reasons, details_only_if_tracked) diff --git a/ldclient/config.py b/ldclient/config.py index d7ac76a2..4fb2b572 100644 --- a/ldclient/config.py +++ b/ldclient/config.py @@ -24,6 +24,7 @@ EventProcessor, FeatureStore, Initializer, + OverrideSource, Synchronizer, UpdateProcessor ) @@ -255,6 +256,23 @@ def build(self, config: DataSourceBuilderConfig) -> T_co: raise NotImplementedError +class OverrideSourceBuilder(Protocol): # pylint: disable=too-few-public-methods + """ + Protocol for building an override source. Flag overrides are currently experimental and + subject to change. + """ + + def build(self, config: DataSourceBuilderConfig) -> OverrideSource: + """ + Builds the override source. Invalid configuration raises an exception, which the client + constructor propagates. + + :param config: the SDK configuration + :return: the built override source + """ + raise NotImplementedError + + @dataclass(frozen=True) class DataSystemConfig: """Configuration for LaunchDarkly's data acquisition strategy.""" @@ -278,6 +296,16 @@ class DataSystemConfig: fdv1_fallback_synchronizer: Optional[DataSourceBuilder[Synchronizer]] = None """An optional fallback synchronizer that will read from FDv1""" + override_source: Optional[OverrideSourceBuilder] = None + """ + An optional override source. Its flag and segment definitions take precedence over + LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject + to change. + + When the client is offline, the source is not built or started: no overrides are served, and + a configuration error in the source is not reported. The SDK logs a warning instead. + """ + class Config(DataSourceBuilderConfig, PrivateAttributesConfig): """Advanced configuration options for the SDK client. @@ -715,4 +743,4 @@ def _validate(self): log.warning("Missing or blank SDK key") -__all__ = ['Config', 'BigSegmentsConfig', 'DataSourceBuilder', 'DataSystemConfig', 'HTTPConfig'] +__all__ = ['Config', 'BigSegmentsConfig', 'DataSourceBuilder', 'DataSystemConfig', 'HTTPConfig', 'OverrideSourceBuilder'] diff --git a/ldclient/datasystem.py b/ldclient/datasystem.py index d3a61a54..0e36ec34 100644 --- a/ldclient/datasystem.py +++ b/ldclient/datasystem.py @@ -4,7 +4,11 @@ from typing import List, Optional -from ldclient.config import DataSourceBuilder, DataSystemConfig +from ldclient.config import ( + DataSourceBuilder, + DataSystemConfig, + OverrideSourceBuilder +) from ldclient.impl.datasourcev2.polling import ( FallbackToFDv1PollingDataSourceBuilder, PollingDataSourceBuilder @@ -32,6 +36,7 @@ def __init__(self) -> None: self._fdv1_fallback_synchronizer: Optional[DataSourceBuilder[Synchronizer]] = None self._store_mode: DataStoreMode = DataStoreMode.READ_ONLY self._data_store: Optional[FeatureStore] = None + self._override_source: Optional[OverrideSourceBuilder] = None def initializers(self, initializers: Optional[List[DataSourceBuilder[Initializer]]]) -> "ConfigBuilder": """ @@ -80,6 +85,38 @@ def data_store(self, data_store: FeatureStore, store_mode: DataStoreMode) -> "Co self._store_mode = store_mode return self + def overrides(self, source: OverrideSourceBuilder) -> "ConfigBuilder": + """ + Configures the SDK with an override source. Flag overrides are currently experimental + and subject to change. + + The source supplies flag and segment definitions that take precedence over data + received from LaunchDarkly on a per-key basis. Overrides let an operator force one or + more flags to a known state on a running client, whether or not the client can reach + LaunchDarkly. Flags not present in the override data are unaffected. + + The override source is not a data source. It has no effect on the client's + initialization status or data source status. Configuring it changes nothing until the + source actually supplies an override. At most one override source is accepted. + + When the client is offline, the source is not built or started: no overrides are served, + and a configuration error in the source is not reported. The SDK logs a warning instead. + + Example, with the file-based source and the SDK's default data sources: + :: + + from ldclient import datasystem + from ldclient.integrations.overrides import FileOverrideSourceBuilder + + source = FileOverrideSourceBuilder(['/etc/launchdarkly/overrides.json']) + config = Config(sdk_key, datasystem_config=datasystem.default().overrides(source).build()) + + :param source: a builder for the override source, such as + :class:`ldclient.integrations.overrides.FileOverrideSourceBuilder` + """ + self._override_source = source + return self + def build(self) -> DataSystemConfig: """ Builds the data system configuration. @@ -90,6 +127,7 @@ def build(self) -> DataSystemConfig: fdv1_fallback_synchronizer=self._fdv1_fallback_synchronizer, data_store_mode=self._store_mode, data_store=self._data_store, + override_source=self._override_source, ) diff --git a/ldclient/evaluation.py b/ldclient/evaluation.py index 18aa9e29..a65b9c83 100644 --- a/ldclient/evaluation.py +++ b/ldclient/evaluation.py @@ -57,6 +57,12 @@ def reason(self) -> dict: * ``inExperiment``: This is true if the evaluation resulted in an experiment rollout and served one of the variations in the experiment. Otherwise it is false. + * ``overrideAffected``: This is true if a flag override affected the evaluation, directly + or indirectly. It is present only when true. An evaluation is affected when the flag + itself, a prerequisite flag at any depth, or a segment consulted during rule matching + came from the SDK's override store. Flag overrides are currently experimental and + subject to change. + * ``bigSegmentsStatus``: describes the validity of Big Segment information, if and only if the flag evaluation required querying at least one Big Segment; otherwise it returns None. Allowable values are defined in :class:`BigSegmentsStatus`. For more information, read the diff --git a/ldclient/impl/async_evaluator.py b/ldclient/impl/async_evaluator.py index ead01e08..31f3466c 100644 --- a/ldclient/impl/async_evaluator.py +++ b/ldclient/impl/async_evaluator.py @@ -15,7 +15,8 @@ _match_clause_by_kind, _match_single_context_value, _maybe_negate, - check_targets + check_targets, + mark_override_affected ) from ldclient.impl.events.types import EventFactory from ldclient.impl.model import * @@ -56,15 +57,23 @@ def __init__( async def evaluate(self, flag: FeatureFlag, context: Context, event_factory: EventFactory) -> EvalResult: state = EvalResult() state.original_flag_key = flag.key + # Reading the flag's own definition is the first read of the evaluation, so the marking + # starts from the flag's override marker. + state.override_affected = flag.is_override try: state.detail = await self._evaluate(flag, context, state, event_factory) except EvaluationException as e: if self.__logger is not None: self.__logger.error('Could not evaluate flag "%s": %s' % (flag.key, e.message)) state.detail = EvaluationDetail(None, None, {'kind': 'ERROR', 'errorKind': e.error_kind}) + # An evaluation that fails is still marked when it read an override definition. + if state.override_affected: + mark_override_affected(state.detail) return state if state.big_segments_status is not None: state.detail.reason['bigSegmentsStatus'] = state.big_segments_status + if state.override_affected: + mark_override_affected(state.detail) return state async def _evaluate(self, flag: FeatureFlag, context: Context, state: EvalResult, event_factory: EventFactory) -> EvaluationDetail: @@ -120,14 +129,26 @@ async def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: log.warning("Missing prereq flag: " + prereq_key) failed_prereq = prereq else: - state.depth += 1 - prereq_res = await self._evaluate(prereq_flag, context, state, event_factory) - state.depth -= 1 + # The prerequisite is an evaluation in its own right. Its marking starts from + # its own definition and covers only the definitions its subtree reads. The + # parent's marking is restored and merged afterward, so the marking propagates + # upward only. A failure inside the subtree still merges what was read. + parent_override_affected = state.override_affected + state.override_affected = prereq_flag.is_override + try: + state.depth += 1 + prereq_res = await self._evaluate(prereq_flag, context, state, event_factory) + state.depth -= 1 + finally: + prereq_override_affected = state.override_affected + state.override_affected = parent_override_affected or prereq_override_affected + if prereq_override_affected: + mark_override_affected(prereq_res) # Note that if the prerequisite flag is off, we don't consider it a match no matter what its # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: failed_prereq = prereq - event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag) + event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected) state.add_event(event) if failed_prereq: return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key} @@ -146,7 +167,14 @@ async def _clause_matches_context(self, clause: Clause, context: Context, state: if clause.op == 'segmentMatch': for seg_key in clause.values: segment = await self.__get_segment(seg_key) - if segment is not None and await self._segment_matches_context(segment, context, state): + if segment is None: + continue + # The segment definition was read at this point, so an override segment marks the + # evaluation whether or not it matches. A segment that does not match still shapes + # the outcome, for example through a negated clause. + if segment.is_override: + state.override_affected = True + if await self._segment_matches_context(segment, context, state): return _maybe_negate(clause, True) return _maybe_negate(clause, False) diff --git a/ldclient/impl/datasystem/__init__.py b/ldclient/impl/datasystem/__init__.py index 26be520d..6799351f 100644 --- a/ldclient/impl/datasystem/__init__.py +++ b/ldclient/impl/datasystem/__init__.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from ldclient.impl.aio.concurrency import AsyncEvent + from ldclient.impl.overrides.layer import OverrideLayer from ldclient.impl.listeners import Listeners from ldclient.interfaces import ( @@ -158,6 +159,17 @@ def environment_id(self) -> Optional[str]: """ raise NotImplementedError + @property + @abstractmethod + def override_layer(self) -> Optional["OverrideLayer"]: + """ + Returns the override layer when the data system was built with an override source, + and None otherwise. The store served by :attr:`store` overlays this layer. The client + consults the layer directly before it has LaunchDarkly data, so that a flag the layer + holds is served and a flag it does not hold gets the not-ready handling. + """ + raise NotImplementedError + class AsyncDataSystem(Protocol): """ @@ -232,6 +244,15 @@ def store(self) -> AsyncReadOnlyStore: """ raise NotImplementedError + @property + @abstractmethod + def override_layer(self) -> Optional["OverrideLayer"]: + """ + Returns the override layer when the data system was built with an override source, + and None otherwise. See :attr:`DataSystem.override_layer`. + """ + raise NotImplementedError + class DiagnosticAccumulator(Protocol): def record_stream_init(self, timestamp, duration, failed): diff --git a/ldclient/impl/datasystem/async_fdv1.py b/ldclient/impl/datasystem/async_fdv1.py index 96fbc660..a7bd1455 100644 --- a/ldclient/impl/datasystem/async_fdv1.py +++ b/ldclient/impl/datasystem/async_fdv1.py @@ -124,6 +124,11 @@ async def stop(self): def store(self) -> AsyncReadOnlyStore: return self._store_view + @property + def override_layer(self) -> None: + # Overrides are an option of the FDv2 data system only. + return None + def set_diagnostic_accumulator(self, diagnostic_accumulator: DiagnosticAccumulator): """ Sets the diagnostic accumulator for streaming initialization metrics. diff --git a/ldclient/impl/datasystem/async_fdv2.py b/ldclient/impl/datasystem/async_fdv2.py index 612fe4cb..a0db7c3d 100644 --- a/ldclient/impl/datasystem/async_fdv2.py +++ b/ldclient/impl/datasystem/async_fdv2.py @@ -38,6 +38,11 @@ ) from ldclient.impl.datasystem.store import _decode from ldclient.impl.listeners import Listeners +from ldclient.impl.overrides import ( + AsyncOverrideStoreView, + OverrideLayer, + OverrideSinkImpl +) from ldclient.impl.util import _LD_FD_FALLBACK_HEADER, _Fail, log from ldclient.interfaces import ( AsyncFeatureStore, @@ -47,7 +52,9 @@ DataSourceErrorKind, DataSourceState, DataStoreMode, - DataStoreStatus + DataStoreStatus, + FlagChange, + OverrideSource ) from ldclient.versioned_data_kind import VersionedDataKind @@ -270,6 +277,21 @@ def __init__( self._store_view = _AsyncReadOnlyStoreView(self._store) + # Set only when an override source is configured. See the sync FDv2 for the roles of + # the layer, the overlay, and the source. Building the source here makes an invalid + # source configuration a construction error. + self._override_source: Optional[OverrideSource] = None + self._override_layer: Optional[OverrideLayer] = None + self._override_sink: Optional[OverrideSinkImpl] = None + self._overlay: Optional[AsyncOverrideStoreView] = None + if data_system_config.override_source is not None and not self._disabled: + self._override_source = data_system_config.override_source.build(config) + self._override_layer = OverrideLayer() + self._overlay = AsyncOverrideStoreView(self._store_view, self._override_layer) + self._store.set_external_dependencies(self._override_layer) + elif data_system_config.override_source is not None: + log.warning("Offline mode is enabled; the override source is not started and no overrides are served") + # Concurrency self._stop_event = AsyncEvent() self._lock = AsyncLock() @@ -314,11 +336,50 @@ def start(self, set_on_ready: AsyncEvent): self._stop_event.clear() + if self._override_source is not None and self._override_layer is not None: + # The source runs on its own threads. Flag change notifications are delivered on + # the event loop, like every other change notification of the async client. The + # merged view for the change computation reads the in-memory store, which is the + # only store that can be read synchronously. While a persistent store is the + # active store, the dependency fan-out is limited to the in-memory data, and the + # directly changed keys are still notified. + loop = asyncio.get_running_loop() + listeners = self._flag_change_listeners + + def notify(key: str) -> None: + try: + loop.call_soon_threadsafe(listeners.notify, FlagChange(key)) + except RuntimeError: + # A reload can finish after the client closed. The loop is closed then, + # and the notification has nobody left to reach. + log.debug("Dropped the flag change notification for %s because the event loop is closed", key) + + self._override_sink = OverrideSinkImpl(self._override_layer, self._store._memory_store, notify, listeners.has_listeners) + try: + self._override_source.start(self._override_sink) + except Exception as e: + # The SDK's own file-based source does not raise from start; it logs a failed + # load and retries it. A custom source could, and a client in an incident is + # better served by running without its overrides than by not starting. + log.error("Unable to start the override source; the client runs without overrides: %s", e) + self._close_override_source("Unable to close the override source after it failed to start") + # The source may have supplied a snapshot before it failed. Clearing the layer + # removes it, so the client runs without overrides as logged, and a late + # snapshot from the closed source is ignored. + self._override_sink.close_and_clear() + # Start the main coordination loop self._runner.spawn("AsyncFDv2-main", lambda: self._run_main_loop(set_on_ready)) async def stop(self): """Stop the AsyncFDv2 data system and all the work it is coordinating.""" + if self._override_source is not None: + # The sink stops accepting snapshots first, so a snapshot that the source supplies + # while it shuts down cannot update the layer or notify listeners. + if self._override_sink is not None: + self._override_sink.close() + self._close_override_source("Unable to close the override source") + self._stop_event.set() async with self._lock: @@ -334,6 +395,18 @@ async def stop(self): # Close the store await self._store.close() + def _close_override_source(self, failure_message: str) -> None: + # A custom source can raise from close. That is logged and not raised, so the rest of + # the client still stops, or still starts after a failed start. + source = self._override_source + self._override_source = None + if source is None: + return + try: + source.close() + except Exception as e: + log.error("%s: %s", failure_message, e) + async def _run_main_loop(self, set_on_ready: AsyncEvent): """Main coordination loop that manages initializers and synchronizers.""" try: @@ -628,9 +701,18 @@ async def reader(): @property def store(self) -> AsyncReadOnlyStore: - """Get the underlying store for flag evaluation.""" + """ + Get the store for flag evaluation. When an override source is configured, this is the + overlay that serves override entries in preference to LaunchDarkly data. + """ + if self._overlay is not None: + return self._overlay return self._store_view + @property + def override_layer(self) -> Optional[OverrideLayer]: + return self._override_layer + async def data_availability(self) -> DataAvailability: # type: ignore[override] """Reports what form of data is currently available, awaiting the store's readiness so a persistent store populated by another process is recognized diff --git a/ldclient/impl/datasystem/fdv1.py b/ldclient/impl/datasystem/fdv1.py index 09ca6939..9d0008a1 100644 --- a/ldclient/impl/datasystem/fdv1.py +++ b/ldclient/impl/datasystem/fdv1.py @@ -233,6 +233,11 @@ def stop(self): def store(self) -> ReadOnlyStore: return self._store_view + @property + def override_layer(self) -> None: + # Overrides are an option of the FDv2 data system only. + return None + @property def environment_id(self) -> Optional[str]: return self._data_source_update_sink.environment_id diff --git a/ldclient/impl/datasystem/fdv2.py b/ldclient/impl/datasystem/fdv2.py index 75f8d28c..2e034001 100644 --- a/ldclient/impl/datasystem/fdv2.py +++ b/ldclient/impl/datasystem/fdv2.py @@ -16,6 +16,11 @@ ) from ldclient.impl.datasystem.store import Store, _decode from ldclient.impl.listeners import Listeners +from ldclient.impl.overrides import ( + OverrideLayer, + OverrideSinkImpl, + OverrideStoreView +) from ldclient.impl.repeating_task import RepeatingTask from ldclient.impl.rwlock import ReadWriteLock from ldclient.impl.util import _LD_FD_FALLBACK_HEADER, _Fail, log @@ -26,6 +31,8 @@ DataStoreMode, DataStoreStatus, FeatureStore, + FlagChange, + OverrideSource, ReadOnlyStore, Synchronizer ) @@ -257,6 +264,27 @@ def __init__( self._store_view = _ReadOnlyStoreView(self._store) + # The following are set only when an override source is configured. The layer holds the + # override entries. The overlay serves them in preference to the store's data at the + # store read boundary. The source populates the layer at runtime. The override system + # has no effect on initialization status or data availability. Building the source + # here makes an invalid source configuration a construction error, like any other + # invalid component configuration. + self._override_source: Optional[OverrideSource] = None + self._override_layer: Optional[OverrideLayer] = None + self._override_sink: Optional[OverrideSinkImpl] = None + self._overlay: Optional[OverrideStoreView] = None + if data_system_config.override_source is not None and not self._disabled: + self._override_source = data_system_config.override_source.build(config) + self._override_layer = OverrideLayer() + self._overlay = OverrideStoreView(self._store_view, self._override_layer) + # A LaunchDarkly change to a flag or segment that an override entry depends on + # must notify that entry's flag as well. The store reaches those entries through + # the layer's dependency graph. + self._store.set_external_dependencies(self._override_layer) + elif data_system_config.override_source is not None: + log.warning("Offline mode is enabled; the override source is not started and no overrides are served") + def _create_store(self, flag_change_listeners: Listeners, change_set_listeners: Listeners) -> Store: return Store(flag_change_listeners, change_set_listeners) @@ -288,6 +316,28 @@ def start(self, set_on_ready: Event): self._stop_event.clear() + if self._override_source is not None and self._override_layer is not None: + # The source starts before the main thread, so a source that loads synchronously + # has its overrides in place before the client begins evaluating. + self._override_sink = OverrideSinkImpl( + self._override_layer, + self._store_view, + lambda key: self._flag_change_listeners.notify(FlagChange(key)), + self._flag_change_listeners.has_listeners, + ) + try: + self._override_source.start(self._override_sink) + except Exception as e: + # The SDK's own file-based source does not raise from start; it logs a failed + # load and retries it. A custom source could, and a client in an incident is + # better served by running without its overrides than by not starting. + log.error("Unable to start the override source; the client runs without overrides: %s", e) + self._close_override_source("Unable to close the override source after it failed to start") + # The source may have supplied a snapshot before it failed. Clearing the layer + # removes it, so the client runs without overrides as logged, and a late + # snapshot from the closed source is ignored. + self._override_sink.close_and_clear() + # Start the main coordination thread main_thread = Thread( target=self._run_main_loop, @@ -300,6 +350,13 @@ def start(self, set_on_ready: Event): def stop(self): """Stop the FDv2 data system and all associated threads.""" + if self._override_source is not None: + # The sink stops accepting snapshots first, so a snapshot that the source supplies + # while it shuts down cannot update the layer or notify listeners. + if self._override_sink is not None: + self._override_sink.close() + self._close_override_source("Unable to close the override source") + self._stop_event.set() with self._lock.write(): @@ -625,6 +682,18 @@ def _record_environment_id(self, environment_id: Optional[str]): with self._lock.write(): self._environment_id = environment_id + def _close_override_source(self, failure_message: str) -> None: + # A custom source can raise from close. That is logged and not raised, so the rest of + # the client still stops, or still starts after a failed start. + source = self._override_source + self._override_source = None + if source is None: + return + try: + source.close() + except Exception as e: + log.error("%s: %s", failure_message, e) + @property def environment_id(self) -> Optional[str]: """Get the environment ID reported by LaunchDarkly, if known.""" @@ -633,9 +702,18 @@ def environment_id(self) -> Optional[str]: @property def store(self) -> ReadOnlyStore: - """Get the underlying store for flag evaluation.""" + """ + Get the store for flag evaluation. When an override source is configured, this is the + overlay that serves override entries in preference to LaunchDarkly data. + """ + if self._overlay is not None: + return self._overlay return self._store_view + @property + def override_layer(self) -> Optional[OverrideLayer]: + return self._override_layer + __all__ = [ 'ConditionDirective', diff --git a/ldclient/impl/datasystem/store.py b/ldclient/impl/datasystem/store.py index b6532605..3904312b 100644 --- a/ldclient/impl/datasystem/store.py +++ b/ldclient/impl/datasystem/store.py @@ -10,7 +10,11 @@ from collections import defaultdict from typing import Any, Callable, Dict, List, Optional, Set -from ldclient.impl.dependency_tracker import DependencyTracker, KindAndKey +from ldclient.impl.dependency_tracker import ( + DependencyTracker, + ExternalDependencies, + KindAndKey +) from ldclient.impl.listeners import Listeners from ldclient.impl.model.entity import ModelEntity from ldclient.impl.rwlock import ReadWriteLock @@ -182,6 +186,10 @@ def __init__( # Used to track dependencies between items in the store self._dependency_tracker = DependencyTracker() + # A second dependency graph, such as the override layer's, whose items may depend on + # items in this store. Set once at construction of the data system, before any data. + self._external_dependencies: Optional[ExternalDependencies] = None + # Listeners for events self._flag_change_listeners = flag_change_listeners self._change_set_listeners = change_set_listeners @@ -195,6 +203,40 @@ def __init__( # Thread synchronization self._lock = threading.RLock() + def set_external_dependencies(self, dependencies: Optional[ExternalDependencies]) -> None: + """ + Registers a second dependency graph whose items may depend on items in this store. A + change to an item in this store then also notifies the external items that depend on + it, directly or through any chain of dependencies in either graph. + """ + with self._lock: + self._external_dependencies = dependencies + + def _add_affected_items(self, items_out: Set[KindAndKey], initial_modified_item: KindAndKey) -> None: + """ + Adds the item and everything that depends on it to the set, following this store's + dependency graph and the external one. Every item reached through either graph is + expanded through both, so a dependency chain may cross between them any number of times. + """ + external = self._external_dependencies + if external is None: + self._dependency_tracker.add_affected_items(items_out, initial_modified_item) + return + + pending: List[KindAndKey] = [initial_modified_item] + while pending: + current = pending.pop() + # Each graph gets its own working set: a tracker does not expand an item that is + # already present, so sharing one set would stop the second graph at the start. + via_store: Set[KindAndKey] = set() + self._dependency_tracker.add_affected_items(via_store, current) + via_external: Set[KindAndKey] = set() + external.add_affected_items(via_external, current) + for reached in via_store | via_external: + if reached not in items_out: + items_out.add(reached) + pending.append(reached) + def selector(self) -> Selector: """Returns the current selector.""" with self._lock: @@ -282,7 +324,7 @@ def _apply_delta( kind, key, item ) if has_listeners: - self._dependency_tracker.add_affected_items( + self._add_affected_items( affected_items, KindAndKey(kind=kind, key=key) ) @@ -353,11 +395,11 @@ def _compute_changed_items_for_full_data_set( # If either is missing or versions differ, it's a change if old_item is None or new_item is None: - self._dependency_tracker.add_affected_items( + self._add_affected_items( affected_items, KindAndKey(kind=kind, key=key) ) elif old_item.get("version", 0) != new_item.get("version", 0): - self._dependency_tracker.add_affected_items( + self._add_affected_items( affected_items, KindAndKey(kind=kind, key=key) ) diff --git a/ldclient/impl/dependency_tracker.py b/ldclient/impl/dependency_tracker.py index 23d6b0d5..10601862 100644 --- a/ldclient/impl/dependency_tracker.py +++ b/ldclient/impl/dependency_tracker.py @@ -1,6 +1,7 @@ -from typing import Dict, List, NamedTuple, Optional, Set, Union +from typing import Dict, List, NamedTuple, Optional, Protocol, Set, Union from ldclient.impl.model.clause import Clause +from ldclient.impl.model.entity import ModelEntity from ldclient.impl.model.feature_flag import FeatureFlag from ldclient.impl.model.segment import Segment from ldclient.versioned_data_kind import FEATURES, SEGMENTS, VersionedDataKind @@ -11,6 +12,20 @@ class KindAndKey(NamedTuple): key: str +class ExternalDependencies(Protocol): + """ + A second dependency graph whose items may depend on items held elsewhere, such as the + override layer, whose entries may depend on LaunchDarkly data. A store that is handed one + extends its change notifications to the external items that depend on a changed item. + """ + + def add_affected_items(self, items_out: Set[KindAndKey], initial_modified_item: KindAndKey) -> None: + """ + Adds the given item and every item in this graph that directly or indirectly depends on + it to the set. Items already in the set are not expanded again. + """ + + class DependencyTracker: """ The DependencyTracker is responsible for tracking both up and downstream @@ -23,7 +38,7 @@ def __init__(self): self.__children: Dict[KindAndKey, Set[KindAndKey]] = {} self.__parents: Dict[KindAndKey, Set[KindAndKey]] = {} - def update_dependencies_from(self, from_kind: VersionedDataKind, from_key: str, from_item: Optional[Union[dict, FeatureFlag, Segment]]): + def update_dependencies_from(self, from_kind: VersionedDataKind, from_key: str, from_item: Optional[Union[dict, ModelEntity]]): """ Updates the dependency graph when an item has changed. @@ -83,7 +98,7 @@ def reset(self): self.__parents.clear() @staticmethod - def compute_dependencies_from(from_kind: VersionedDataKind, from_item: Optional[Union[dict, FeatureFlag, Segment]]) -> Set[KindAndKey]: + def compute_dependencies_from(from_kind: VersionedDataKind, from_item: Optional[Union[dict, ModelEntity]]) -> Set[KindAndKey]: """ @param from_kind [String] @param from_item [LaunchDarkly::Impl::Model::FeatureFlag, LaunchDarkly::Impl::Model::Segment] diff --git a/ldclient/impl/evaluator.py b/ldclient/impl/evaluator.py index 77a811d4..bbce2260 100644 --- a/ldclient/impl/evaluator.py +++ b/ldclient/impl/evaluator.py @@ -17,7 +17,8 @@ _maybe_negate, _variation_index_for_context, check_targets, - error_reason + error_reason, + mark_override_affected ) from ldclient.impl.events.types import EventFactory from ldclient.impl.model import * @@ -57,15 +58,23 @@ def __init__( def evaluate(self, flag: FeatureFlag, context: Context, event_factory: EventFactory) -> EvalResult: state = EvalResult() state.original_flag_key = flag.key + # Reading the flag's own definition is the first read of the evaluation, so the marking + # starts from the flag's override marker. + state.override_affected = flag.is_override try: state.detail = self._evaluate(flag, context, state, event_factory) except EvaluationException as e: if self.__logger is not None: self.__logger.error('Could not evaluate flag "%s": %s' % (flag.key, e.message)) state.detail = EvaluationDetail(None, None, {'kind': 'ERROR', 'errorKind': e.error_kind}) + # An evaluation that fails is still marked when it read an override definition. + if state.override_affected: + mark_override_affected(state.detail) return state if state.big_segments_status is not None: state.detail.reason['bigSegmentsStatus'] = state.big_segments_status + if state.override_affected: + mark_override_affected(state.detail) return state def _evaluate(self, flag: FeatureFlag, context: Context, state: EvalResult, event_factory: EventFactory) -> EvaluationDetail: @@ -118,14 +127,26 @@ def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: EvalR log.warning("Missing prereq flag: " + prereq_key) failed_prereq = prereq else: - state.depth += 1 - prereq_res = self._evaluate(prereq_flag, context, state, event_factory) - state.depth -= 1 + # The prerequisite is an evaluation in its own right. Its marking starts from + # its own definition and covers only the definitions its subtree reads. The + # parent's marking is restored and merged afterward, so the marking propagates + # upward only. A failure inside the subtree still merges what was read. + parent_override_affected = state.override_affected + state.override_affected = prereq_flag.is_override + try: + state.depth += 1 + prereq_res = self._evaluate(prereq_flag, context, state, event_factory) + state.depth -= 1 + finally: + prereq_override_affected = state.override_affected + state.override_affected = parent_override_affected or prereq_override_affected + if prereq_override_affected: + mark_override_affected(prereq_res) # Note that if the prerequisite flag is off, we don't consider it a match no matter what its # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: failed_prereq = prereq - event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag) + event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected) state.add_event(event) if failed_prereq: return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key} @@ -144,7 +165,14 @@ def _clause_matches_context(self, clause: Clause, context: Context, state: EvalR if clause.op == 'segmentMatch': for seg_key in clause.values: segment = self.__get_segment(seg_key) - if segment is not None and self._segment_matches_context(segment, context, state): + if segment is None: + continue + # The segment definition was read at this point, so an override segment marks the + # evaluation whether or not it matches. A segment that does not match still shapes + # the outcome, for example through a negated clause. + if segment.is_override: + state.override_affected = True + if self._segment_matches_context(segment, context, state): return _maybe_negate(clause, True) return _maybe_negate(clause, False) diff --git a/ldclient/impl/evaluator_common.py b/ldclient/impl/evaluator_common.py index 04dbebfd..7385f4e8 100644 --- a/ldclient/impl/evaluator_common.py +++ b/ldclient/impl/evaluator_common.py @@ -25,7 +25,7 @@ # prerequisite evaluations, and the cached state of any Big Segments query that we may have # ended up having to do for the context. class EvalResult: - __slots__ = ['detail', 'events', 'big_segments_status', 'big_segments_membership', 'original_flag_key', 'prereq_stack', 'segment_stack', 'depth', 'prerequisites'] + __slots__ = ['detail', 'events', 'big_segments_status', 'big_segments_membership', 'original_flag_key', 'prereq_stack', 'segment_stack', 'depth', 'prerequisites', 'override_affected'] def __init__(self): self.detail = None @@ -37,6 +37,11 @@ def __init__(self): self.segment_stack = None # type: Optional[List[str]] self.depth = 0 self.prerequisites = [] # type: List[str] + # True when the evaluation in progress has read a flag or segment definition that carries + # the override marker. While a prerequisite is evaluated, this holds the marking of the + # prerequisite's own subtree. The evaluator merges it into the parent's marking afterward, + # so the marking propagates upward only. + self.override_affected = False def record_prerequisite(self, key: str): if self.depth == 0: @@ -255,3 +260,22 @@ def check_targets(flag: FeatureFlag, context: Context) -> Optional[EvaluationDet def error_reason(error_kind: str) -> dict: return {'kind': 'ERROR', 'errorKind': error_kind} + + +def is_override_definition(item: Any) -> bool: + """ + Reports whether a flag or segment read from the store came from the override layer. The + data systems decode the dicts a store holds into model objects before the client reads + them, but an error path reads the marker this way so that an unexpected item cannot make + it fail too. + """ + return getattr(item, 'is_override', False) is True + + +def mark_override_affected(detail: EvaluationDetail) -> None: + """ + Sets the override-affected indicator on the evaluation reason. The indicator is present + only when true, so a reason that was not affected by an override is left untouched and + serializes exactly as before. + """ + detail.reason['overrideAffected'] = True diff --git a/ldclient/impl/events/event_processor_common.py b/ldclient/impl/events/event_processor_common.py index 95ea278f..75f634b1 100644 --- a/ldclient/impl/events/event_processor_common.py +++ b/ldclient/impl/events/event_processor_common.py @@ -181,7 +181,7 @@ def make_summary_event(self, summary: EventSummary): } counters = [] for ckey, cval in flag_data.counters.items(): - variation, version = ckey + variation, version, override_affected = ckey counter = {'count': cval.count, 'value': cval.value} if variation is not None: counter['variation'] = variation @@ -189,6 +189,9 @@ def make_summary_event(self, summary: EventSummary): counter['unknown'] = True else: counter['version'] = version + # The marker is present only when true, like the unknown marker. + if override_affected: + counter['overrideAffected'] = True counters.append(counter) flag_data_out['counters'] = counters flags_out[key] = flag_data_out @@ -266,9 +269,11 @@ def _process_event(self, event: EventInput): context = event.context if not event.exclude_from_summaries: self._outbox.add_to_summary(event) - if event.track_events: + # An override-affected evaluation appears only in the summary counters. It produces + # no individual feature event and no debug event, even when the flag requests them. + if event.track_events and not event.override_affected: full_event = event - if self._should_debug_event(event): + if not event.override_affected and self._should_debug_event(event): debug_event = DebugEvent(event) elif isinstance(event, EventInputIdentify): if self._omit_anonymous_contexts: diff --git a/ldclient/impl/events/event_summarizer.py b/ldclient/impl/events/event_summarizer.py index acf3a88f..ca2b5b5e 100644 --- a/ldclient/impl/events/event_summarizer.py +++ b/ldclient/impl/events/event_summarizer.py @@ -22,10 +22,16 @@ def __repr__(self) -> str: # used only in test debugging return "EventSummaryCounter(%d, %s)" % (self.count, self.value) +# A counter is keyed by variation index, flag version, and whether the evaluations it counts +# were override-affected. The marker is part of the key, so override-affected and ordinary +# evaluations of the same flag, variation, and version accumulate into separate counters. +CounterKey = Tuple[Optional[int], Optional[int], bool] + + class EventSummaryFlag: __slots__ = ['context_kinds', 'default', 'counters'] - def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[Tuple[Optional[int], Optional[int]], EventSummaryCounter]): + def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[CounterKey, EventSummaryCounter]): self.context_kinds = context_kinds self.counters = counters self.default = default @@ -71,7 +77,7 @@ def summarize_event(self, event: EventInputEvaluation): if c is not None: flag_data.context_kinds.add(c.kind) - counter_key = (event.variation, None if event.flag is None else event.flag.version) + counter_key = (event.variation, None if event.flag is None else event.flag.version, event.override_affected) counter = flag_data.counters.get(counter_key) if counter is None: counter = EventSummaryCounter(1, event.value) diff --git a/ldclient/impl/events/types.py b/ldclient/impl/events/types.py index d0883c47..6f754a23 100644 --- a/ldclient/impl/events/types.py +++ b/ldclient/impl/events/types.py @@ -52,6 +52,7 @@ class EventInputEvaluation(EventInput): "track_events", "sampling_ratio", "exclude_from_summaries", + "override_affected", ] def __init__( @@ -66,6 +67,7 @@ def __init__( default_value: Any, prereq_of: Optional[FeatureFlag] = None, track_events: bool = False, + override_affected: bool = False, ): super().__init__(timestamp, context, 1 if flag is None else flag.sampling_ratio) self.key = key @@ -79,6 +81,11 @@ def __init__( self.exclude_from_summaries = ( False if flag is None else flag.exclude_from_summaries ) + # True when a flag override affected this evaluation, directly or through a + # prerequisite or segment. The event processor keys on this scalar alone, never on the + # reason: such an evaluation produces no individual feature or debug event and is + # counted in a summary counter that carries the override-affected marker. + self.override_affected = override_affected def to_debugging_dict(self) -> dict: return { @@ -94,6 +101,7 @@ def to_debugging_dict(self) -> dict: "track_events": self.track_events, "exclude_from_summaries": self.exclude_from_summaries, "sampling_ratio": self.sampling_ratio, + "override_affected": self.override_affected, } @@ -155,6 +163,7 @@ def new_eval_event( detail: EvaluationDetail, default_value: Any, prereq_of_flag: Optional[FeatureFlag] = None, + override_affected: bool = False, ) -> EventInputEvaluation: add_experiment_data = self.is_experiment(flag, detail.reason) return EventInputEvaluation( @@ -168,6 +177,7 @@ def new_eval_event( default_value, prereq_of_flag, flag.track_events or add_experiment_data, + override_affected, ) def new_default_event( @@ -176,6 +186,7 @@ def new_default_event( context: Context, default_value: Any, reason: Optional[dict], + override_affected: bool = False, ) -> EventInputEvaluation: return EventInputEvaluation( self._timestamp_fn(), @@ -188,6 +199,7 @@ def new_default_event( default_value, None, flag.track_events, + override_affected, ) def new_unknown_flag_event( diff --git a/ldclient/impl/integrations/files/filedata.py b/ldclient/impl/integrations/files/filedata.py index a4f71d6a..f8caf95d 100644 --- a/ldclient/impl/integrations/files/filedata.py +++ b/ldclient/impl/integrations/files/filedata.py @@ -6,9 +6,11 @@ A data file is a JSON or YAML document with optional ``flags``, ``flagValues``, and ``segments`` members. ``flags`` and ``segments`` hold full definitions keyed by key. ``flagValues`` maps a flag key to a single value. It expands into a full flag definition -that returns that value for every context. +that returns that value for every context. The file name extension decides the format: a +file whose name ends in ``.yaml`` or ``.yml`` is YAML, and any other file is JSON. """ +import codecs import hashlib import json import os @@ -54,6 +56,9 @@ # because it does not exist yet. _WATCH_RETRY_INTERVAL = 1.0 +# The file name extensions of YAML files, compared without regard to case. Any other file is JSON. +YAML_EXTENSIONS = frozenset([".yaml", ".yml"]) + # The watchdog event types that can change a file's content or presence. Opening or reading a # file also produces events, and a reload reads the files, so those must not count as changes. _CHANGE_EVENT_TYPES = frozenset(["created", "modified", "moved", "deleted", "closed"]) @@ -148,33 +153,69 @@ def make_flag_with_value(key: str, value: Any) -> FeatureFlag: return FeatureFlag({"key": key, "version": 1, "on": True, "fallthrough": {"variation": 0}, "variations": [value]}) +def is_yaml_path(path: str) -> bool: + """ + Whether a data file is a YAML document. The end of the file name decides: a name that ends in + ``.yaml`` or ``.yml``, in any case, is YAML, including a name that is only that, such as + ``.yaml``. Any other file, including one with no extension, is JSON. + """ + return os.path.basename(path).lower().endswith(tuple(YAML_EXTENSIONS)) + + def read_file(path: str) -> Document: - """Reads and parses a single data file, which may be in JSON or YAML format.""" + """Reads and parses a single data file, in the format its file name extension selects.""" try: with open(path, "rb") as f: raw = f.read() except OSError as e: raise FileReadError(path, "unable to read file: %s" % e) try: - return parse_document(raw) + return parse_document(raw, is_yaml_path(path)) except Exception as e: raise FileReadError(path, "error parsing file: %s" % e) -def parse_document(raw: bytes) -> Document: +def has_content(raw: bytes) -> bool: + """Whether the raw content of a file holds anything other than white space and a byte order mark.""" + return raw.removeprefix(codecs.BOM_UTF8).strip() != b"" + + +def parse_document(raw: bytes, is_yaml: bool = False) -> Document: """ - Parses the raw content of a data file, which may be JSON or YAML. The parser is chosen the - way the file data sources choose it: the YAML parser reads both formats when the ``pyyaml`` - package is installed, and the standard JSON parser is used when it is not. An empty - document is valid and holds no entries. + Parses the raw content of a data file as YAML when ``is_yaml`` is true, which requires the + ``pyyaml`` package, and as JSON otherwise. The file name extension decides the format; see + :func:`is_yaml_path`. A byte order mark at the start is ignored. + + Whether a file with no content is valid depends on its format, like the rest of its content. + A JSON file with no content, or only white space, is not a JSON document, so it fails to + parse. A YAML file with no content, or with only comments or a document marker, is a + document with no entries. """ - text = raw.decode("utf-8") - if have_yaml: - parsed = yaml.safe_load(text) # pyyaml correctly parses JSON too + document = _parse_content(raw, is_yaml) + return Document() if document is None else document + + +def _parse_content(raw: bytes, is_yaml: bool) -> Optional[Document]: + """ + The parse behind :func:`parse_document`. Returns None when the YAML parser reads the content + as nothing, as it does for a file with no content, only comments, or ``null``, rather than as + a document, even one with no members such as ``{}``. A JSON document must be an object. + """ + text = raw.decode("utf-8-sig") + if is_yaml: + if not have_yaml: + raise ValueError("the pyyaml package is required to read YAML") + parsed = yaml.safe_load(text) else: + if not has_content(raw): + # Not a JSON document. A file that is rewritten in place can be empty for a moment, + # and the failure keeps the last good data until the file is complete. + raise ValueError("the file has no content, which is not a JSON document") parsed = json.loads(text) + if not isinstance(parsed, dict): + raise ValueError("the document must be an object") if parsed is None: - return Document() + return None if not isinstance(parsed, dict): raise ValueError("the document must be an object") @@ -255,38 +296,79 @@ def load_files(paths: List[str], duplicate_keys_handling: DuplicateKeysHandling, and the handling is FAIL. When ``skip_missing_paths`` is true, a file that does not exist contributes no entries instead of failing the load. """ - merged, _ = _load_files_hashed(paths, duplicate_keys_handling, skip_missing_paths) + merged, _ = _parse_files(_read_files(paths, skip_missing_paths), duplicate_keys_handling) return merged -def _load_files_hashed(paths: List[str], duplicate_keys_handling: DuplicateKeysHandling, skip_missing_paths: bool) -> Tuple[MergeResult, bytes]: +@dataclass +class _FileRead: + """What one load read from one configured file.""" + + path: str + raw: Optional[bytes] + """The raw content, or None when the file does not exist and missing files are skipped.""" + + +def _read_files(paths: List[str], skip_missing_paths: bool) -> List[_FileRead]: """ - The load behind :func:`load_files`. It also returns a digest of the raw file contents, so a - caller can tell whether a later load read the same bytes. One read feeds both the digest - and the parse, so the digest can never disagree with the content that was parsed. + Reads the raw content of all of the given files in order. Raises :class:`FileReadError` + when a file cannot be read, including a missing file unless ``skip_missing_paths`` is true. + """ + return [_read_file(path, skip_missing_paths) for path in paths] + + +def _read_file(path: str, skip_missing_paths: bool) -> _FileRead: + """Reads the raw content of one file, as :func:`_read_files` does.""" + try: + with open(path, "rb") as f: + return _FileRead(path, f.read()) + except FileNotFoundError: + if skip_missing_paths: + log.debug("File %s does not exist; it contributes no data", path) + return _FileRead(path, None) + raise FileReadError(path, "unable to read file: the file does not exist") + except OSError as e: + raise FileReadError(path, "unable to read file: %s" % e) + + +def _parse_file(path: str, raw: bytes) -> Optional[Document]: + """Parses what was read from a file that exists. Returns None when the parser read nothing.""" + is_yaml = is_yaml_path(path) + try: + return _parse_content(raw, is_yaml) + except Exception as e: + raise FileReadError(path, "error parsing file as %s, the format of its name: %s" % ("YAML" if is_yaml else "JSON", e)) + + +def _parse_files(contents: List[_FileRead], duplicate_keys_handling: DuplicateKeysHandling) -> Tuple[MergeResult, bytes]: + """Parses and merges the content that :func:`_read_files` read.""" + return _merge_files([(read, None if read.raw is None else _parse_file(read.path, read.raw)) for read in contents], duplicate_keys_handling) + + +def _merge_files(parsed: List[Tuple[_FileRead, Optional[Document]]], duplicate_keys_handling: DuplicateKeysHandling) -> Tuple[MergeResult, bytes]: + """ + Merges the parsed files. It also returns a digest of the files that were read, so a caller + can tell whether a later load read the same. One read feeds both the digest and the parse, + so the digest can never disagree with the content that was parsed. The digest covers each + path, whether its file exists, and its bytes: the same bytes in a file with another name can + be in another format, and parse differently. """ documents: List[Document] = [] files: List[FileSummary] = [] hasher = hashlib.sha256() - for path in paths: - try: - with open(path, "rb") as f: - raw = f.read() - except FileNotFoundError: - if skip_missing_paths: - log.debug("File %s does not exist; it contributes no data", path) - files.append(FileSummary(path=path)) - continue - raise FileReadError(path, "unable to read file: the file does not exist") - except OSError as e: - raise FileReadError(path, "unable to read file: %s" % e) - hasher.update(raw) - hasher.update(b"\0") - try: - documents.append(parse_document(raw)) - except Exception as e: - raise FileReadError(path, "error parsing file: %s" % e) - files.append(FileSummary(path=path, present=True)) + for read, document in parsed: + path = os.fsencode(read.path) + hasher.update(len(path).to_bytes(8, "big")) + hasher.update(path) + if read.raw is None: + hasher.update(b"-") + files.append(FileSummary(path=read.path)) + continue + hasher.update(b"+") + hasher.update(len(read.raw).to_bytes(8, "big")) + hasher.update(read.raw) + documents.append(Document() if document is None else document) + files.append(FileSummary(path=read.path, present=True)) merged = merge(documents, duplicate_keys_handling) # The documents are the present files in order. Copy their counts onto the file summaries. @@ -300,12 +382,73 @@ def _load_files_hashed(paths: List[str], duplicate_keys_handling: DuplicateKeysH return merged, hasher.digest() +FileState = Optional[Tuple[int, int]] + + +def _observe(path: str) -> FileState: + """ + Observes the state of a file: its modification time and size, through any symbolic link, or + None when it does not exist or cannot be examined. + """ + try: + info = os.stat(path) + return (info.st_mtime_ns, info.st_size) + except OSError: + return None + + +def _observe_all(paths: List[str]) -> List[FileState]: + """Observes the state of each file, as :func:`_observe` does.""" + return [_observe(path) for path in paths] + + +# A file is held back for at most this many retry delays after a load first held it back, until +# a load holds nothing back. +_MAX_HOLD_RETRY_DELAYS = 5 + + +@dataclass +class _EmptyFile: + """A file whose content is in effect and that loads have found with no entries.""" + + since: Optional[float] = None + """ + Since when the file has had no entries without being written, on the reloader's clock, or + None when the last load read it with content, even content that failed to parse, or found + it missing. A read that fails keeps it, as nothing is known about the content then. + """ + state: FileState = None + """The file's state then: its modification time and size, through any symbolic link.""" + first_held: Optional[float] = None + """ + When a load first held the file back, or None. It bounds how long the file is held back. + It is kept while loads hold files back, so files emptied in turn stay bounded, and cleared by + any load that holds nothing back, including one that fails. + """ + + class Reloader: """ Owns the reload cycle for a set of data files. It serializes reloads, debounces change signals, retains the last good result on failure by not calling ``apply``, retries after failures, and skips applications that would change nothing. + A file can be rewritten in place: a writer truncates it and then writes the new content, and + a load in between finds it empty. When retries are enabled, a load that finds a file with no + entries, because it is empty or holds only what its parser reads as nothing (a YAML comment + or document marker), while that file's content is in effect, holds the file back: it keeps + the current data without reporting a failure, and the other files' changes wait too, since a + load applies all files or none. The file is held back until it has stayed so, without being + written, for the retry delay; a change of its modification time or size starts the delay + over. Each file is held back for at most five retry delays after a load first held it back, + until a load holds nothing back, so a file that keeps being rewritten is applied in the end + without cutting short another file's rewrite. A file whose content is not in effect, such as a + placeholder that has always been empty, is never held back. Every file is read and parsed, + even after another file fails, before any is held back, so files emptied together wait + together, and a file that cannot be read or parsed fails the load, with the first failure, + even while another is held back. The clock is read after each file's + own read, so a slow read of another file does not shorten a file's delay. + The worker thread starts on the first :meth:`reload_now` or :meth:`trigger` call, so a reloader that is constructed but never used does not leak a thread. """ @@ -320,6 +463,7 @@ def __init__( debounce_delay: float = 0.0, retry_delay: float = 0.0, skip_unchanged: bool = False, + clock: Callable[[], float] = time.monotonic, ): """ :param paths: the files to load, in order. The order determines which file wins under @@ -336,9 +480,13 @@ def __init__( :param debounce_delay: how long to wait after a trigger for further triggers to settle before reloading. Zero reloads on every trigger. :param retry_delay: how long to wait after a failed reload before retrying it - automatically. Zero disables the automatic retry. + automatically, and how long a file whose content is in effect must have no entries, + without being written, before that is applied. Zero disables the automatic retry, and + an empty file is then parsed as it is. :param skip_unchanged: when true, a load whose raw file contents are identical to the last applied contents does not call ``apply`` + :param clock: the monotonic clock, in seconds, that times how long files are held back. + Tests replace it; the timers always use ``time.monotonic``. """ self._paths = list(paths) self._duplicate_keys_handling = duplicate_keys_handling @@ -348,6 +496,7 @@ def __init__( self._debounce_delay = debounce_delay self._retry_delay = retry_delay self._skip_unchanged = skip_unchanged + self._clock = clock # Guards the deadlines, the closed flag, and the worker start. self._cond = threading.Condition() @@ -355,11 +504,27 @@ def __init__( self._started = False self._debounce_deadline: Optional[float] = None self._retry_deadline: Optional[float] = None + # Whether the pending retry was armed by a load that held files back. + self._retry_after_hold = False - # Serializes the load work between reload_now and the worker thread. + # Serializes the load work between reload_now and the worker thread. The fields below are + # only touched while holding it. self._reload_lock = threading.Lock() self._last_good_digest: Optional[bytes] = None self._last_error_message: Optional[str] = None + # Whether each configured file contributed content to the data in effect. A missing file, + # or one with no entries, contributed none. + self._applied_had_content: Dict[str, bool] = {} + # The files whose content is in effect and that loads have found with no entries. All are + # forgotten when data is applied or a load is skipped as unchanged, and a load that holds + # nothing back, even one that fails, starts every file's bound over. + self._empty_files: Dict[str, _EmptyFile] = {} + # Whether the last load held files back, which the log line of the retry it arms names. + self._held_back = False + # Whether the current load has held a file back, even if it then fails. + self._holding_this_load = False + # The delay of the last retry that was armed, for tests. + self._last_requested_retry_delay: Optional[float] = None def reload_now(self) -> None: """ @@ -367,11 +532,9 @@ def reload_now(self) -> None: arms the same automatic retry as a failed triggered reload. """ self._ensure_started() - if not self._reload() and self._retry_delay > 0: - with self._cond: - if not self._closed and self._retry_deadline is None: - self._retry_deadline = time.monotonic() + self._retry_delay - self._cond.notify() + retry_after, held = self._reload() + if retry_after is not None: + self._arm_retry(retry_after, keep_sooner=True, after_hold=held) def trigger(self) -> None: """ @@ -395,6 +558,25 @@ def close(self) -> None: self._closed = True self._cond.notify_all() + def _arm_retry(self, retry_after: float, keep_sooner: bool, after_hold: bool) -> None: + """ + Arms the automatic retry for the given time from now, unless retries are off. With + ``keep_sooner``, a retry that is already due sooner stays as it is. ``after_hold`` says + that the load held files back, which the retry's log line names. + """ + if self._retry_delay <= 0: + return + with self._cond: + if self._closed: + return + deadline = time.monotonic() + retry_after + if keep_sooner and self._retry_deadline is not None and self._retry_deadline <= deadline: + return + self._retry_deadline = deadline + self._retry_after_hold = after_hold + self._last_requested_retry_delay = retry_after + self._cond.notify() + def _ensure_started(self) -> None: with self._cond: if self._closed or self._started: @@ -406,6 +588,7 @@ def _ensure_started(self) -> None: def _run(self) -> None: while True: is_retry = False + after_hold = False with self._cond: while True: if self._closed: @@ -421,78 +604,245 @@ def _run(self) -> None: continue if self._debounce_deadline is not None and self._debounce_deadline <= now: # A triggered reload supersedes a pending retry. It either succeeds, or - # it fails and arms a fresh retry below. + # it fails or holds files back and arms a fresh retry below. self._debounce_deadline = None self._retry_deadline = None is_retry = False else: self._retry_deadline = None is_retry = True + after_hold = self._retry_after_hold break - if is_retry: + if is_retry and after_hold: + log.debug("Reading flag data again after a file was found empty") + elif is_retry: log.debug("Retrying flag data load after earlier failure") else: log.info("Reloading flag data after detecting a change") try: - ok = self._reload() + retry_after, held = self._reload() except Exception as e: log.exception("Unexpected error while reloading flag data: %s", e) - ok = True - if not ok and self._retry_delay > 0: - with self._cond: - if not self._closed: - self._retry_deadline = time.monotonic() + self._retry_delay + retry_after, held = None, False + if retry_after is not None: + self._arm_retry(retry_after, keep_sooner=False, after_hold=held) def _is_closed(self) -> bool: with self._cond: return self._closed - def _reload(self) -> bool: + def _reload(self) -> Tuple[Optional[float], bool]: """ - Performs one full load of all configured files. Returns whether the load succeeded, - which decides whether a retry is armed. A skipped no-op application counts as success. + Performs one full load of all configured files. Returns the time after which to load + again, or None when the load succeeded, including a skipped no-op application: the retry + delay after a failure, or the time a hold lasts. Also returns whether the load held files + back. """ with self._reload_lock: if self._is_closed(): - return True + return None, False + self._held_back = False + self._holding_this_load = False + reached: List[str] = [] + retry_after = self._load(reached) + if not self._holding_this_load: + # A load that holds nothing back, whether it applies, is skipped or fails, starts the + # bound over for every file that it read or tried to read, including one that + # failed to be read or parsed: such a file is no longer held back, so its next + # rewrite is a new one. + for path in reached: + empty = self._empty_files.get(path) + if empty is not None: + empty.first_held = None + return retry_after, self._held_back + + def _load(self, reached: List[str]) -> Optional[float]: + """ + The body of :meth:`_reload`. It records in ``reached`` each file that it reads or tries to + read, which is every configured file, since a load reads all of them even after a failure. + """ + # Every file is read, parsed and classified, even after another file fails, before the + # load fails, holds files back or merges them. So every file that is found with no + # entries starts its delay in this load, and every file is reached, so a load that holds + # nothing back starts every file's bound over. The load fails with the first failure. + failure: Optional[Exception] = None + + def note_failure(e: Exception) -> None: + nonlocal failure + if failure is None: + failure = e + else: + # The load fails with the first failure. The others are logged, so that a second + # broken file is visible too. + log.debug("Unable to load flag data, besides the first failure: %s", e) + + held_back: List[str] = [] + parsed: List[Tuple[_FileRead, Optional[Document]]] = [] + had_content: Dict[str, bool] = {} + for path in self._paths: + reached.append(path) try: - merged, digest = _load_files_hashed(self._paths, self._duplicate_keys_handling, self._skip_missing_paths) + read = _read_file(path, self._skip_missing_paths) except Exception as e: - return self._fail(e) + # Nothing is known about the content, so the file's empty state, if any, stays. + note_failure(e) + continue + if read.raw is None: + # A missing file is not an empty one. + self._end_empty(read.path) + had_content[read.path] = False + parsed.append((read, None)) + continue + # A file with no content, or with content that its parser reads as nothing, such as + # YAML with only a comment, has no document and so no entries. A read can see that + # much of a file that is being written. Content with no document is invalid JSON, so + # it is only parsed when the file is not held back. {} is a document. + document: Optional[Document] = None + has_document = has_content(read.raw) + if has_document: + try: + document = _parse_file(read.path, read.raw) + except FileReadError as e: + # Content that cannot be parsed is not an empty file either. + self._end_empty(read.path) + note_failure(e) + continue + has_document = document is not None + if has_document: + self._end_empty(read.path) + # The clock is read after this file's own read, so that a slow read of another file + # does not shorten its delay. + elif self._should_hold_back(read.path, self._clock()): + held_back.append(read.path) + self._holding_this_load = True + continue + # The file was read and is not held back, even if it then fails to parse. + had_content[read.path] = has_document + if not has_content(read.raw): + try: + document = _parse_file(read.path, read.raw) + except FileReadError as e: + note_failure(e) + continue + parsed.append((read, document)) + if failure is not None: + return self._fail(failure) + if len(held_back) > 0: + return self._hold_back(held_back, self._clock()) - # A close may have happened while the files were being read. Deliver nothing then. - if self._is_closed(): - return True - - # A success right after a failure applies even when the content is unchanged since - # the last success. The consumer heard about the failure and only an application - # tells it that things are good again. - recovering = self._last_error_message is not None - if self._skip_unchanged and not recovering and digest == self._last_good_digest: - return True - - # Nothing is remembered until the consumer has accepted the result. A result the - # consumer rejects must not become the baseline that skip-unchanged compares - # against, and must not count as a recovery. - try: - self._apply(merged) - except Exception as e: - return self._fail(e) - self._last_error_message = None - self._last_good_digest = digest - return True + try: + merged, digest = _merge_files(parsed, self._duplicate_keys_handling) + except Exception as e: + return self._fail(e) + + # A close may have happened while the files were being read. Deliver nothing then. + if self._is_closed(): + return None + + # A success right after a failure applies even when the content is unchanged since the + # last success. The consumer heard about the failure and only an application tells it + # that things are good again. + recovering = self._last_error_message is not None + if self._skip_unchanged and not recovering and digest == self._last_good_digest: + # The data in effect is confirmed, as by an application, so it is remembered the same + # way, and the bounds start over. + self._remember_applied(had_content) + return None - def _fail(self, err: Exception) -> bool: + # Nothing is remembered until the consumer has accepted the result. A result the + # consumer rejects must not become the baseline that skip-unchanged compares against, + # and must not count as a recovery. + try: + self._apply(merged) + except Exception as e: + return self._fail(e) + self._last_error_message = None + self._last_good_digest = digest + self._remember_applied(had_content) + return None + + def _end_empty(self, path: str) -> None: + """ + Ends the empty state of a file that a load read with content, even content that cannot be + parsed, or found missing, so that it starts a new delay when it is found empty again. Its + bound is kept until a load holds nothing back. + """ + empty = self._empty_files.get(path) + if empty is not None: + empty.since = None + + def _remember_applied(self, had_content: Dict[str, bool]) -> None: + """Records what each file contributed to the data in effect, and forgets the empty files.""" + self._applied_had_content.update(had_content) + self._empty_files.clear() + + def _should_hold_back(self, path: str, now: float) -> bool: + """ + Decides whether a file that a load found with no entries is held back. A file whose + content is in effect and that has none now may be in the middle of an in-place rewrite. + It is held back until it has stayed so, without being written, for the retry delay. A + change of its state, its modification time and size read through any symbolic link, + means that it was written since a load last found it so, for example rewritten and + truncated again between two loads, so the delay starts over. A file that has stayed so + for the whole delay is not held back. Each file is held back for at most five retry + delays after a load first held it back, so that a file that keeps being written while + every load finds it empty, or files that are emptied in turn, are not held back without + end. The bound starts over only at a load that holds nothing back, for every file, whether + the load applies, is skipped or fails. A load reads every file even after one fails. A file whose content is not in effect + is parsed right away, and so is any file when retries are off, since nothing would read it + again. + """ + if self._retry_delay <= 0 or not self._applied_had_content.get(path, False): + return False + state = _observe(path) + empty = self._empty_files.get(path) + if empty is None: + empty = _EmptyFile() + self._empty_files[path] = empty + if empty.since is None or empty.state != state: + empty.since = now + empty.state = state + if now - empty.since >= self._retry_delay: + # The file stayed empty for the whole delay. Its bound starts over only at a load that + # holds nothing back, so that files emptied in turn do not keep each other held back. + return False + if empty.first_held is not None and now - empty.first_held >= _MAX_HOLD_RETRY_DELAYS * self._retry_delay: + return False + if empty.first_held is None: + empty.first_held = now + return True + + def _hold_back(self, held_back: List[str], now: float) -> Optional[float]: + """ + Keeps the current data for a load that holds files back. Whatever the files' format, + nothing is applied or reported. The returned time arms the retry for when the first of + the files has stayed so for the retry delay, or has been held back for as long as it may + be, plus a tenth of the retry delay, so that a timer that fires a little early still + finds the time over. The change signal for a write starts a load sooner. + """ + if self._is_closed(): + return None + until = float("inf") + for path in held_back: + log.debug("File %s has no entries, possibly while it is being rewritten; waiting for it to be written before applying it", path) + empty = self._empty_files[path] + since = now if empty.since is None else empty.since + first_held = now if empty.first_held is None else empty.first_held + until = min(until, since + self._retry_delay, first_held + _MAX_HOLD_RETRY_DELAYS * self._retry_delay) + self._held_back = True + return max(until - now, 0.0) + self._retry_delay / 10 + + def _fail(self, err: Exception) -> Optional[float]: if self._is_closed(): - return True + return None # With automatic retries, a persistent failure would repeat the same log entry and the # same callback on every attempt. Repeats of an identical failure are logged at debug # level and do not call on_error again. message = str(err) if message == self._last_error_message: log.debug("Unable to load flag data: %s", err) - return False + return self._retry_delay self._last_error_message = message log.error("Unable to load flag data: %s", err) if self._on_error is not None: @@ -501,25 +851,7 @@ def _fail(self, err: Exception) -> bool: except Exception as e: # A consumer that raises while it reports a failure must not stop the retry. log.error("Error while reporting a flag data load failure: %s", e) - return False - - -FileState = Optional[Tuple[int, int]] - - -def _observe_all(paths: List[str]) -> List[FileState]: - """ - Observes the state of each file: its modification time and size, or None when it does not - exist or cannot be examined. - """ - states: List[FileState] = [] - for path in paths: - try: - info = os.stat(path) - states.append((info.st_mtime_ns, info.st_size)) - except OSError: - states.append(None) - return states + return self._retry_delay class Poller: diff --git a/ldclient/impl/integrations/overrides/__init__.py b/ldclient/impl/integrations/overrides/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ldclient/impl/integrations/overrides/file_override_source.py b/ldclient/impl/integrations/overrides/file_override_source.py new file mode 100644 index 00000000..220a73b8 --- /dev/null +++ b/ldclient/impl/integrations/overrides/file_override_source.py @@ -0,0 +1,153 @@ +""" +The file-based override source. It reads flag and segment overrides from one or more local +files and reloads them when the files change. +""" + +import threading +from enum import Enum +from typing import List, Optional, Union + +from ldclient.impl.integrations.files.filedata import ( + DEFAULT_DEBOUNCE_DELAY, + DEFAULT_RETRY_DELAY, + DuplicateKeysHandling, + FileSummary, + MergeResult, + Poller, + Reloader, + Watcher +) +from ldclient.impl.util import log +from ldclient.interfaces import OverrideSink, OverrideSource + + +class ChangeDetection(str, Enum): + """ + Selects how the file-based override source learns that a file changed. The two modes are + alternatives. Flag overrides are currently experimental and subject to change. + """ + + POLLING = "polling" + """ + The source examines the files on a fixed interval and reloads when the modification time + or the size of a file changes. Polling works on every file system, including network + mounts and directories whose contents are swapped through symbolic links, as Kubernetes + does for mounted ConfigMaps. It is the default. + """ + + WATCHING = "watching" + """ + The source reloads in response to file system change notifications, using the ``watchdog`` + package. It reacts faster than polling. It depends on notifications, which some file + systems do not deliver reliably. + """ + + +class _FileOverrideSource(OverrideSource): + """ + Reads overrides from the configured files and supplies each successful load to the sink as + a full snapshot. A configured file that does not exist contributes no overrides. A file + that cannot be read or parsed fails that load, the last good snapshot stays in effect, and + the load is retried. A file whose content is in effect and that is found with no entries is + held back until it has stayed so for the retry delay, because it may be in the middle of an + in-place rewrite. + """ + + def __init__(self, paths: List[str], duplicate_keys_handling: DuplicateKeysHandling, change_detection: ChangeDetection, poll_interval: float): + self._paths = paths + self._duplicate_keys_handling = duplicate_keys_handling + self._change_detection = change_detection + self._poll_interval = poll_interval + self._reloader: Optional[Reloader] = None + self._change_detector: Optional[Union[Poller, Watcher]] = None + self._lock = threading.Lock() + self._closed = False + + def start(self, sink: OverrideSink) -> None: + def apply(merged: MergeResult) -> None: + sink.set_overrides(merged.flags, merged.segments) + _log_overrides_in_effect(merged) + + reloader = Reloader( + self._paths, + self._duplicate_keys_handling, + apply=apply, + skip_missing_paths=True, + debounce_delay=DEFAULT_DEBOUNCE_DELAY, + retry_delay=DEFAULT_RETRY_DELAY, + skip_unchanged=True, + ) + # Change detection is set up before the initial load, so an edit made while the files + # are first read is picked up by the reload it triggers instead of waiting for a later + # change. + with self._lock: + if self._closed: + return + self._reloader = reloader + if self._change_detection == ChangeDetection.WATCHING: + self._change_detector = Watcher(self._paths, reloader.trigger) + else: + poller = Poller(self._paths, self._poll_interval, reloader.trigger) + poller.start() + self._change_detector = poller + + # The initial load runs synchronously, so overrides present in the files are in effect + # by the time the client constructor returns. A file that does not exist yet + # contributes no overrides. A file that cannot be read or parsed is not fatal: the + # client runs with no overrides, the failure is logged, and the retry recovers once + # the file is readable. + reloader.reload_now() + + def close(self) -> None: + with self._lock: + if self._closed: + return + self._closed = True + change_detector = self._change_detector + reloader = self._reloader + self._change_detector = None + self._reloader = None + try: + if change_detector is not None: + change_detector.close() + finally: + # The reloader stops even when the change detector fails to close, so no reload + # can run after close. + if reloader is not None: + reloader.close() + + +def _log_overrides_in_effect(merged: MergeResult) -> None: + """ + Reports the overrides now in effect and what each configured file supplied. The reloader + applies a snapshot only when the content changed, so this logs each change once. + """ + details = "; ".join(_file_summary_text(summary) for summary in merged.files) + if len(merged.flags) == 0 and len(merged.segments) == 0: + log.info("Flag overrides: none in effect (%s)", details) + return + log.info("Flag overrides in effect: %s (%s)", _counts_text(len(merged.flags), len(merged.segments)), details) + + +def _file_summary_text(summary: FileSummary) -> str: + if not summary.present: + return "%s: absent" % summary.path + if summary.flags == 0 and summary.segments == 0: + return "%s: no entries" % summary.path + return "%s: %s" % (summary.path, _counts_text(summary.flags, summary.segments)) + + +def _counts_text(flags: int, segments: int) -> str: + """Formats flag and segment counts, for example "2 flags, 1 segment".""" + parts = [] + if flags > 0: + parts.append(_pluralize(flags, "flag")) + if segments > 0: + parts.append(_pluralize(segments, "segment")) + return ", ".join(parts) + + +def _pluralize(count: int, noun: str) -> str: + if count == 1: + return "1 %s" % noun + return "%d %ss" % (count, noun) diff --git a/ldclient/impl/model/feature_flag.py b/ldclient/impl/model/feature_flag.py index ae0b6f75..f7c03f4d 100644 --- a/ldclient/impl/model/feature_flag.py +++ b/ldclient/impl/model/feature_flag.py @@ -1,3 +1,4 @@ +import copy from typing import Any, Dict, List, Optional, Set, Union from ldclient.impl.model.clause import Clause @@ -96,10 +97,12 @@ class FeatureFlag(ModelEntity): '_salt', '_track_events', '_debug_events_until_date', + '_is_override', ] def __init__(self, data: dict): super().__init__(data) + self._is_override = False # In the following logic, we're being somewhat lenient in terms of allowing most properties to # be absent even if they are really required in the schema. That's for backward compatibility # with test logic that constructed incomplete JSON, and also with the file data source which @@ -202,3 +205,24 @@ def exclude_from_summaries(self) -> bool: @property def sampling_ratio(self) -> Optional[int]: return self._sampling_ratio + + @property + def is_override(self) -> bool: + """ + True if this flag definition was supplied by an SDK override source rather than by + LaunchDarkly data. The marker is never part of the JSON representation. Only the SDK + components that manage override entries set it. Components that read a definition + through the store may treat a marked definition the same as any other. + + Flag overrides are currently experimental and subject to change. + """ + return self._is_override + + def with_override_marker(self) -> 'FeatureFlag': + """ + Returns a shallow copy of this flag with the override marker set. The copy shares the + underlying data with this instance, which is left unmarked. + """ + marked = copy.copy(self) + marked._is_override = True + return marked diff --git a/ldclient/impl/model/segment.py b/ldclient/impl/model/segment.py index 6ed1b6a8..709518a2 100644 --- a/ldclient/impl/model/segment.py +++ b/ldclient/impl/model/segment.py @@ -1,3 +1,4 @@ +import copy from typing import Any, List, Optional, Set from ldclient.impl.model.attribute_ref import ( @@ -65,10 +66,12 @@ class Segment(ModelEntity): '_unbounded', '_unbounded_context_kind', '_generation', + '_is_override', ] def __init__(self, data: dict): super().__init__(data) + self._is_override = False # In the following logic, we're being somewhat lenient in terms of allowing most properties to # be absent even if they are really required in the schema. That's for backward compatibility # with test logic that constructed incomplete JSON, and also with the file data source which @@ -137,3 +140,24 @@ def unbounded_context_kind(self) -> Optional[str]: @property def generation(self) -> Optional[int]: return self._generation + + @property + def is_override(self) -> bool: + """ + True if this segment definition was supplied by an SDK override source rather than by + LaunchDarkly data. The marker is never part of the JSON representation. Only the SDK + components that manage override entries set it. Components that read a definition + through the store may treat a marked definition the same as any other. + + Flag overrides are currently experimental and subject to change. + """ + return self._is_override + + def with_override_marker(self) -> 'Segment': + """ + Returns a shallow copy of this segment with the override marker set. The copy shares the + underlying data with this instance, which is left unmarked. + """ + marked = copy.copy(self) + marked._is_override = True + return marked diff --git a/ldclient/impl/overrides/__init__.py b/ldclient/impl/overrides/__init__.py new file mode 100644 index 00000000..b3e7126e --- /dev/null +++ b/ldclient/impl/overrides/__init__.py @@ -0,0 +1,15 @@ +""" +The flag and segment override layer. The layer is a runtime-mutable collection of flag and +segment definitions supplied by an override source. Those definitions take precedence over +LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject to +change. +""" + +from ldclient.impl.overrides.layer import OverrideLayer +from ldclient.impl.overrides.overlay import ( + AsyncOverrideStoreView, + OverrideStoreView +) +from ldclient.impl.overrides.sink import OverrideSinkImpl + +__all__ = ['AsyncOverrideStoreView', 'OverrideLayer', 'OverrideSinkImpl', 'OverrideStoreView'] diff --git a/ldclient/impl/overrides/layer.py b/ldclient/impl/overrides/layer.py new file mode 100644 index 00000000..1ba5ecd3 --- /dev/null +++ b/ldclient/impl/overrides/layer.py @@ -0,0 +1,94 @@ +""" +The override store. It holds the entries an override source has loaded, keyed by flag key or +segment key, and is replaced wholesale on each update. +""" + +from typing import Any, Dict, Mapping, Optional, Set, Tuple + +from ldclient.impl.dependency_tracker import DependencyTracker, KindAndKey +from ldclient.impl.model import ModelEntity +from ldclient.impl.rwlock import ReadWriteLock +from ldclient.versioned_data_kind import FEATURES, SEGMENTS, VersionedDataKind + +LayerContents = Dict[VersionedDataKind, Dict[str, ModelEntity]] + + +def _empty_contents() -> LayerContents: + return {FEATURES: {}, SEGMENTS: {}} + + +class OverrideLayer: + """ + A thread-safe store of override entries, replaced wholesale on each update from an override + source. Each entry is a marked shallow copy of the definition the source supplied, so the + source's own objects are never marked and a source may retain and resupply them. + """ + + def __init__(self): + self._lock = ReadWriteLock() + self._contents: LayerContents = _empty_contents() + # The dependency graph of the override entries, rebuilt with each replacement. The store + # consults it so that a LaunchDarkly change to a prerequisite or a segment also notifies + # the override entries that depend on it. + self._dependencies = DependencyTracker() + # Read without the lock on every evaluation. A single attribute read is atomic, so the + # per-evaluation cost of a configured but unpopulated override layer is negligible. + self._non_empty = False + + def set_all(self, flags: Mapping[str, Any], segments: Mapping[str, Any]) -> Tuple[LayerContents, LayerContents]: + """ + Atomically replaces the entire layer contents. Empty mappings clear the layer. + + Values may be model objects or their JSON dictionary form. A dictionary is decoded with + the model constructor, which raises ``ValueError`` for an invalid definition, and nothing + is replaced in that case. + + :return: the previous and the new contents. The returned dictionaries must not be modified. + """ + replacement: LayerContents = { + FEATURES: {key: FEATURES.decode(item).with_override_marker() for key, item in flags.items()}, + SEGMENTS: {key: SEGMENTS.decode(item).with_override_marker() for key, item in segments.items()}, + } + count = len(replacement[FEATURES]) + len(replacement[SEGMENTS]) + dependencies = DependencyTracker() + for kind, items in replacement.items(): + for key, item in items.items(): + try: + dependencies.update_dependencies_from(kind, key, item) + except Exception: + # An entry whose dependency fields cannot be read contributes no edges. It is + # still served, and the remaining entries are still tracked. + pass + with self._lock.write(): + previous = self._contents + self._contents = replacement + self._dependencies = dependencies + self._non_empty = count > 0 + return previous, replacement + + def add_affected_items(self, items_out: Set[KindAndKey], initial_modified_item: KindAndKey) -> None: + """ + Adds the given item and every override entry that directly or indirectly depends on it + to the set. The item itself need not be an override entry: a LaunchDarkly flag or + segment that an override entry references is a valid starting point. + """ + with self._lock.read(): + dependencies = self._dependencies + dependencies.add_affected_items(items_out, initial_modified_item) + + def get(self, kind: VersionedDataKind, key: str) -> Optional[ModelEntity]: + """Returns the override entry for a key, or None when the layer has none.""" + if not self._non_empty: + return None + with self._lock.read(): + return self._contents.get(kind, {}).get(key) + + def all(self, kind: VersionedDataKind) -> Dict[str, ModelEntity]: + """Returns the entries of the given kind. The returned dictionary must not be modified.""" + with self._lock.read(): + return self._contents.get(kind, {}) + + @property + def is_empty(self) -> bool: + """True when the layer holds no entries.""" + return not self._non_empty diff --git a/ldclient/impl/overrides/overlay.py b/ldclient/impl/overrides/overlay.py new file mode 100644 index 00000000..c1fd0829 --- /dev/null +++ b/ldclient/impl/overrides/overlay.py @@ -0,0 +1,73 @@ +""" +The overlay at the store read boundary. A read for a key returns the override entry when one +exists and the LaunchDarkly entry otherwise. Targeting rules, prerequisites, and segment matches +behave identically for overridden and ordinary data, because they are the same reads through +the same boundary. +""" + +from typing import Any, Callable, Dict, Optional + +from ldclient.impl.overrides.layer import OverrideLayer +from ldclient.interfaces import AsyncReadOnlyStore, ReadOnlyStore +from ldclient.versioned_data_kind import VersionedDataKind + + +def _merge_all(base_items: Dict[str, Any], override_items: Dict[str, Any]) -> Dict[str, Any]: + """The union of the base items and the override items. The override entry wins for a key present in both.""" + if len(override_items) == 0: + return base_items + merged = dict(base_items) + merged.update(override_items) + return merged + + +class OverrideStoreView(ReadOnlyStore): + """ + Merges an override layer over a base read-only store. + + A per-key read serves an override entry whatever the state of the base store, so an + uninitialized base still serves overrides. An enumeration is the union of the base items + and the layer's items, and the override entry wins for a key present in both, including a + key the base holds as a deleted item. A failure of the base enumeration propagates, as it + does without the overlay: the layer's entries alone would look like a complete result. + """ + + def __init__(self, base: ReadOnlyStore, layer: OverrideLayer): + self._base = base + self._layer = layer + + def get(self, kind: VersionedDataKind, key: str, callback: Callable[[Any], Any] = lambda x: x) -> Any: + item = self._layer.get(kind, key) + if item is not None: + return callback(item) + return self._base.get(kind, key, callback) + + def all(self, kind: VersionedDataKind, callback: Callable[[Any], Any] = lambda x: x) -> Any: + base_items = self._base.all(kind, lambda x: x) + if base_items is None: + # A store returns no result when it cannot read. That stays a failure rather than + # becoming the layer's entries alone. + return callback(None) + return callback(_merge_all(base_items, self._layer.all(kind))) + + @property + def initialized(self) -> bool: + # The override layer never affects initialization status or data availability. + return self._base.initialized + + +class AsyncOverrideStoreView(AsyncReadOnlyStore): + """The async counterpart of :class:`OverrideStoreView`, over an async base store.""" + + def __init__(self, base: AsyncReadOnlyStore, layer: OverrideLayer): + self._base = base + self._layer = layer + + async def get(self, kind: VersionedDataKind, key: str) -> Optional[Any]: + item = self._layer.get(kind, key) + if item is not None: + return item + return await self._base.get(kind, key) + + async def all(self, kind: VersionedDataKind) -> Dict[str, Any]: + return _merge_all(await self._base.all(kind), self._layer.all(kind)) diff --git a/ldclient/impl/overrides/sink.py b/ldclient/impl/overrides/sink.py new file mode 100644 index 00000000..f2389628 --- /dev/null +++ b/ldclient/impl/overrides/sink.py @@ -0,0 +1,173 @@ +""" +The sink that applies override snapshots to the layer and notifies flag change listeners of the +flags whose evaluation may have changed. +""" + +import threading +from typing import Any, Callable, Dict, Optional, Set + +from ldclient.impl.dependency_tracker import DependencyTracker, KindAndKey +from ldclient.impl.overrides.layer import LayerContents, OverrideLayer +from ldclient.impl.util import log +from ldclient.interfaces import OverrideSink, ReadOnlyStore +from ldclient.versioned_data_kind import FEATURES, SEGMENTS, VersionedDataKind + +MergedView = Dict[VersionedDataKind, Dict[str, Any]] + +_DIFF_KINDS = (FEATURES, SEGMENTS) + + +class OverrideSinkImpl(OverrideSink): + """ + Applies override layer replacements supplied by an override source, then notifies flag + change listeners of every flag whose merged-view evaluation may have changed. Calls are + serialized, so overlapping updates from a source cannot interleave. + """ + + def __init__( + self, + layer: OverrideLayer, + base: ReadOnlyStore, + notify: Callable[[str], None], + has_listeners: Callable[[], bool], + ): + """ + :param layer: the override layer to write to + :param base: the store holding LaunchDarkly data, without the overlay. Merged-view + snapshots for change computation are built from it plus the layer. + :param notify: receives the key of each affected flag + :param has_listeners: reports whether anything listens for flag changes, so the change + computation can be skipped when nothing does + """ + self._layer = layer + self._base = base + self._notify = notify + self._has_listeners = has_listeners + self._lock = threading.Lock() + self._closed = False # guarded by _lock + + def set_overrides(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None: # type: ignore[override] + with self._lock: + if self._closed: + log.debug("Ignoring an override snapshot supplied after the override source was closed") + return + self._apply(flags, segments) + + def close(self) -> None: + """ + Stops accepting snapshots. Used when the client is closed: a snapshot that the source + supplies while it shuts down is ignored. The layer keeps its contents. + """ + with self._lock: + self._closed = True + + def close_and_clear(self) -> None: + """ + Stops accepting snapshots and clears the layer. Used when the source failed to start: a + snapshot the source supplied before failing is removed, so the client runs without + overrides, and a late snapshot from the closed source is ignored. + """ + with self._lock: + self._closed = True + self._apply({}, {}) + + def _apply(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None: + # Called with _lock held. + if not self._has_listeners(): + self._layer.set_all(flags, segments) + return + + previous, current = self._layer.set_all(flags, segments) + old_merged = snapshot_merged_view(self._base, previous) + new_merged = snapshot_merged_view(self._base, current) + affected = compute_affected_flags(previous, current, old_merged, new_merged) + if len(affected) > 0: + log.debug("Override update affected %d flag(s)", len(affected)) + for key in sorted(affected): + self._notify(key) + + +def snapshot_merged_view(base: ReadOnlyStore, overrides: LayerContents) -> MergedView: + """ + Captures the merged view of a base store and a layer snapshot: base data with the override + entries overlaid. A base read failure for a kind yields just the overrides for that kind. + This degrades the dependency fan-out but never loses the directly changed keys. + """ + view: MergedView = {} + for kind in _DIFF_KINDS: + items: Dict[str, Any] = {} + try: + base_items = base.all(kind, lambda x: x) + if base_items is not None: + items.update(base_items) + except Exception as e: + log.debug("Unable to read %s for override change computation: %s", kind.namespace, e) + items.update(overrides.get(kind, {})) + view[kind] = items + return view + + +def compute_affected_flags( + old_overrides: LayerContents, + new_overrides: LayerContents, + old_merged: MergedView, + new_merged: MergedView, +) -> Set[str]: + """ + Returns the keys of all flags whose merged-view evaluation may have changed when the + override layer was replaced. The result includes the flags whose override entries were + added, removed, or changed. Dependency fan-out adds every flag that depends, directly or + transitively, on any added, removed, or changed entry of either kind. + """ + seeds = diff_overrides(old_overrides, new_overrides) + if len(seeds) == 0: + return set() + + # Dependency edges are computed over both the old and the new merged views, because a + # replacement can rewire dependencies. For example, removing a flag override restores the + # prerequisite edges of the LaunchDarkly definition. Flags that depended on the override's + # references exist as dependents only in the old view. + old_tracker = _tracker_from_view(old_merged) + new_tracker = _tracker_from_view(new_merged) + affected: Set[KindAndKey] = set() + for seed in seeds: + old_tracker.add_affected_items(affected, seed) + new_tracker.add_affected_items(affected, seed) + return {item.key for item in affected if item.kind == FEATURES} + + +def diff_overrides(old_overrides: LayerContents, new_overrides: LayerContents) -> Set[KindAndKey]: + """ + Returns a key for each entry whose override differs between the two layer snapshots. An + added or removed entry is always a change, even when its content matches the underlying + LaunchDarkly data, because the override marker alone changes the served entry. Entries + present in both snapshots are compared by version and definition. The layer is rebuilt + wholesale on every update, so identity comparison would report every retained entry as + changed. + """ + seeds: Set[KindAndKey] = set() + for kind in _DIFF_KINDS: + old_items = old_overrides.get(kind, {}) + new_items = new_overrides.get(kind, {}) + for key, old_item in old_items.items(): + new_item = new_items.get(key) + if new_item is None or not _items_equal(old_item, new_item): + seeds.add(KindAndKey(kind=kind, key=key)) + for key in new_items: + if key not in old_items: + seeds.add(KindAndKey(kind=kind, key=key)) + return seeds + + +def _items_equal(a: Any, b: Any) -> bool: + if a.version != b.version: + return False + return a.to_json_dict() == b.to_json_dict() + + +def _tracker_from_view(view: MergedView) -> DependencyTracker: + tracker = DependencyTracker() + for kind in _DIFF_KINDS: + for key, item in view.get(kind, {}).items(): + tracker.update_dependencies_from(kind, key, item) + return tracker diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py new file mode 100644 index 00000000..0b35cb3e --- /dev/null +++ b/ldclient/integrations/overrides.py @@ -0,0 +1,200 @@ +""" +Override sources for the SDK's flag override capability. Flag overrides are currently +experimental and subject to change. + +Overrides are flag and segment definitions that take precedence over data received from +LaunchDarkly at evaluation time, on a per-key basis. They exist for resilience during an +incident. An operator can force one or more flags to a known state on a running application, +whether or not the application can reach LaunchDarkly. The override stays in effect until the +operator removes it. Flags not present in the override data are completely unaffected. + +This module currently provides one source: :class:`FileOverrideSourceBuilder`, which reads +overrides from local files and reloads them as the files change. Configure it with the data +system builder: +:: + + from ldclient import Config, datasystem + from ldclient.integrations.overrides import FileOverrideSourceBuilder + + source = FileOverrideSourceBuilder(['/etc/launchdarkly/overrides.json']) + config = Config(sdk_key, datasystem_config=datasystem.default().overrides(source).build()) + +An evaluation that an override affects is marked. The marking is direct or transitive. It +applies when the evaluated flag, a prerequisite at any depth, or a segment read during the +evaluation came from the override layer. The evaluation reason's ``overrideAffected`` property +reports the marking. Marked evaluations appear in analytics summary events only, under separate +counters, so LaunchDarkly can distinguish them. They produce no individual evaluation events. +""" + +import math +import os +import threading +from typing import List, Optional, Union + +from ldclient.config import DataSourceBuilderConfig, OverrideSourceBuilder +from ldclient.impl.integrations.files.filedata import ( + DuplicateKeysHandling, + abs_file_paths, + have_watchdog, + have_yaml, + is_yaml_path +) +from ldclient.impl.integrations.overrides.file_override_source import ( + ChangeDetection, + _FileOverrideSource +) +from ldclient.impl.util import log +from ldclient.interfaces import OverrideSource + + +class FileOverrideSourceBuilder(OverrideSourceBuilder): + """ + A builder for a file-based override source. Flag overrides are currently experimental and + subject to change. + + The source reads flag and segment overrides from one or more local files and reloads them + as the files change. The files use the same document format as the file data source: + each file is a JSON or YAML document with optional ``flags``, ``flagValues``, and + ``segments`` members. ``flagValues`` entries are expanded into full flag definitions that + return the given value for every context. A ``flagValues`` entry is not available to + client-side SDKs, so :meth:`ldclient.client.LDClient.all_flags_state` with + ``client_side_only=True`` leaves it out, even when the flag it overrides is available to them. + To override such a flag, define it in ``flags`` with ``"clientSide": true``. + + The file name decides the format of each file: a file whose name ends in ``.yaml`` or + ``.yml`` is YAML, which requires the ``pyyaml`` package, and any other file is JSON. Files + must be UTF-8. + + When multiple files are configured, their entries are combined in the configured order. + The order determines which file wins under the duplicate keys handling. A reload replaces + the entire override set, so removing an entry from the files removes the override. A + configured file that does not exist contributes no overrides: deleting a file removes its + overrides, and deleting every file removes them all. A file that exists but cannot be read + or parsed makes that whole reload fail. The previously loaded overrides stay in effect, the + source logs the failure, retries after a short delay, and recovers on its own once the file + is readable again. + + Files can be rewritten in place: if the source finds a file empty while it is being + rewritten, it keeps the overrides in effect until the new content is written. To remove a + file's overrides, delete the file or write a document with no entries, such as ``{}``. An + empty JSON file is not a valid document. A partly written YAML file can still parse, so for a + YAML file, replacing its contents in one operation, such as by renaming a fully written + temporary file over it, avoids briefly applying a partly written file. + + Whenever the set of overrides in effect changes, including at startup, the source logs the + overrides in effect and what each configured file supplied, at Info level. + + By default the source polls the files for changes once per second. See + :meth:`change_detection` and :meth:`poll_interval`. + """ + + DEFAULT_POLL_INTERVAL = 1.0 + """ + The interval, in seconds, at which the source examines the files for changes in polling + mode when no interval was specified. Because the source reads local files rather than + contacting a service, a short interval keeps an override responsive during an incident at + negligible cost. + """ + + MINIMUM_POLL_INTERVAL = 1.0 + """ + The shortest allowed polling interval, in seconds. A configured interval below this is + raised to it. The minimum exists only to prevent a pathological tight loop over the file + system. + """ + + def __init__(self, paths: Union[str, List[str]]): + """ + :param paths: the files to load overrides from, as a single path or a list of paths. + The order is significant: it determines which file wins under the duplicate keys + handling when the same key appears in more than one file. Relative paths are + resolved against the current working directory when the source is built. + """ + self.__paths: List[str] = [paths] if isinstance(paths, str) else list(paths) + self.__duplicate_keys_handling = DuplicateKeysHandling.FAIL + self.__change_detection = ChangeDetection.POLLING + self.__poll_interval = self.DEFAULT_POLL_INTERVAL + + def duplicate_keys_handling(self, handling: Union[DuplicateKeysHandling, str]) -> 'FileOverrideSourceBuilder': + """ + Specifies how to handle the same key appearing in more than one file. The default is + :attr:`DuplicateKeysHandling.FAIL`, which treats the reload as failed and keeps the + previously loaded overrides. :attr:`DuplicateKeysHandling.IGNORE` keeps the entry from + the first configured file that defines the key and discards the others. + + :param handling: the handling, as the enum or its string value (``"fail"`` or ``"ignore"``) + """ + self.__duplicate_keys_handling = DuplicateKeysHandling(handling) + return self + + def change_detection(self, mode: Union[ChangeDetection, str]) -> 'FileOverrideSourceBuilder': + """ + Selects how the source detects file changes. The default is + :attr:`ChangeDetection.POLLING`. The two modes are alternatives, so setting one replaces + the other. :attr:`ChangeDetection.WATCHING` requires the ``watchdog`` package. + + :param mode: the mode, as the enum or its string value (``"polling"`` or ``"watching"``) + """ + self.__change_detection = ChangeDetection(mode) + return self + + def poll_interval(self, seconds: float) -> 'FileOverrideSourceBuilder': + """ + Sets the interval between examinations of the files in polling mode. Watching mode + ignores it. The default is :attr:`DEFAULT_POLL_INTERVAL`. An interval below + :attr:`MINIMUM_POLL_INTERVAL` is raised to the minimum. + + :param seconds: the interval in seconds + :raises TypeError: when the interval is not a number + :raises ValueError: when the interval is not a finite number, or is longer than the + runtime can wait + """ + if isinstance(seconds, bool) or not isinstance(seconds, (int, float)): + raise TypeError("the poll interval must be a number of seconds") + if not math.isfinite(seconds): + raise ValueError("the poll interval must be a finite number of seconds") + if seconds > threading.TIMEOUT_MAX: + raise ValueError("the poll interval must not exceed %s seconds" % threading.TIMEOUT_MAX) + self.__poll_interval = seconds + return self + + def build(self, config: DataSourceBuilderConfig) -> OverrideSource: # pylint: disable=unused-argument + """ + Builds the override source. This is called internally by the SDK. It raises + ``ValueError`` when no file paths were specified, when watching mode was selected + without the ``watchdog`` package, or when a YAML file was specified without the + ``pyyaml`` package. + """ + if len(self.__paths) == 0: + raise ValueError("no file paths were specified for the file-based override source") + if self.__change_detection == ChangeDetection.WATCHING and not have_watchdog: + raise ValueError("the file-based override source cannot watch files for changes because the watchdog package is not installed; install it or use polling") + if not have_yaml: + for path in self.__paths: + if is_yaml_path(path): + raise ValueError("the file-based override source cannot read the YAML file %s because the pyyaml package is not installed; install it or use JSON" % path) + + poll_interval = self.__poll_interval + if self.__change_detection == ChangeDetection.POLLING and poll_interval < self.MINIMUM_POLL_INTERVAL: + log.warning("Poll interval %s is below the minimum for the file-based override source; using %s", poll_interval, self.MINIMUM_POLL_INTERVAL) + poll_interval = self.MINIMUM_POLL_INTERVAL + + # A path listed more than once would make every entry of that file a duplicate of itself, + # so the repeats are dropped and the first occurrence keeps its place in the order. Paths + # are compared the way the platform compares file names: without regard to case on + # Windows, exactly elsewhere. + all_paths = abs_file_paths(self.__paths) + paths: List[str] = [] + seen = set() + for path in all_paths: + key = os.path.normcase(path) + if key not in seen: + seen.add(key) + paths.append(path) + if len(paths) != len(all_paths): + log.warning("The file-based override source lists the same file more than once; the repeats are ignored") + + return _FileOverrideSource(paths, self.__duplicate_keys_handling, self.__change_detection, poll_interval) + + +__all__ = ['ChangeDetection', 'DuplicateKeysHandling', 'FileOverrideSourceBuilder'] diff --git a/ldclient/interfaces.py b/ldclient/interfaces.py index 2c9c245f..9fec157e 100644 --- a/ldclient/interfaces.py +++ b/ldclient/interfaces.py @@ -2128,6 +2128,81 @@ def stop(self): raise NotImplementedError +class OverrideSink(Protocol): # pylint: disable=too-few-public-methods + """ + OverrideSink receives the contents of the SDK's flag and segment override layer. The SDK + implements it and passes it to an :class:`OverrideSource` when the source starts. Override + sources call it, they do not implement it. + + Flag overrides are currently experimental and subject to change. + """ + + @abstractmethod + def set_overrides(self, flags: Mapping[str, Any], segments: Mapping[str, Any]) -> None: + """ + Replaces the entire override layer with the given flag and segment definitions. Each call + is a full snapshot: entries absent from the call are removed from the layer, and empty + mappings clear the layer. The new contents are visible to evaluations when the call + returns. + + The mappings are keyed by flag key and segment key. Each value is a full definition, + either a :class:`ldclient.impl.model.FeatureFlag` or :class:`ldclient.impl.model.Segment` + model object or its JSON dictionary form. A dictionary that is not a valid definition + raises ``ValueError``. The SDK itself marks the entries as overrides. + + This method is safe to call from any thread. Calls are serialized by the SDK. + + :param flags: the flag definitions, keyed by flag key + :param segments: the segment definitions, keyed by segment key + """ + raise NotImplementedError + + +class OverrideSource(Protocol): + """ + OverrideSource supplies flag and segment overrides that take precedence over LaunchDarkly data + at evaluation time, on a per-key basis. Overrides exist for resilience during an incident. + They let an operator force one or more flags to a known state on a running client, whether + or not the client can reach LaunchDarkly. + + An override source is not a data source. It does not participate in the data system's + initializer and synchronizer pipeline. The override layer it populates has no effect on the + client's initialization status, data availability, or data source status. + + To configure an override source, use :meth:`ldclient.datasystem.ConfigBuilder.overrides`. + + Flag overrides are currently experimental and subject to change. + """ + + @abstractmethod + def start(self, sink: OverrideSink) -> None: + """ + Begins supplying overrides to the sink and returns without blocking on long-running + work. Implementations typically perform an initial load synchronously, then push a full + replacement snapshot to the sink whenever their backing data changes, until + :meth:`close` is called. A failed load leaves the previously supplied layer untouched by + not calling the sink. + + This method is not intended to fail. A source that cannot load its data logs the failure + and keeps retrying, as the SDK's file-based source does. A configuration error belongs in + the builder's ``build``, where it is raised when the client is created. If this method does + raise, the client still starts: the SDK logs the exception, closes the source, removes any + snapshot the source supplied, and the client runs without overrides. + + :param sink: the sink that receives each snapshot + """ + raise NotImplementedError + + @abstractmethod + def close(self) -> None: + """ + Stops the source and releases any resources it holds. An exception raised from this + method is logged and does not stop the client from shutting down. A snapshot that the + source supplies after the client begins to close it is ignored. + """ + raise NotImplementedError + + class AsyncInitializer(Protocol): # pylint: disable=too-few-public-methods """ AsyncInitializer represents a component capable of retrieving a single data result diff --git a/ldclient/testing/file_util.py b/ldclient/testing/file_util.py new file mode 100644 index 00000000..25f5a10a --- /dev/null +++ b/ldclient/testing/file_util.py @@ -0,0 +1,26 @@ +import time + + +def rewrite_in_place(path: str, content: str, pause: float) -> None: + """ + Rewrites a file the way applications and editors often do, without replacing it atomically: + the file is truncated, and the new content is written after the pause. A read during the + pause finds the file empty. A file that does not exist is created with the content. + """ + try: + f = open(path, 'r+') + except FileNotFoundError: + with open(path, 'w') as created: + created.write(content) + return + with f: + f.truncate(0) + f.flush() + time.sleep(pause) + f.write(content) + + +def truncate(path: str) -> None: + """Empties a file in place, which is how an in-place rewrite starts.""" + with open(path, 'r+') as f: + f.truncate(0) diff --git a/ldclient/testing/impl/datasystem/test_store_external_dependencies.py b/ldclient/testing/impl/datasystem/test_store_external_dependencies.py new file mode 100644 index 00000000..87422a6a --- /dev/null +++ b/ldclient/testing/impl/datasystem/test_store_external_dependencies.py @@ -0,0 +1,137 @@ +""" +Tests for the store's use of a second dependency graph: a LaunchDarkly change to a flag or +segment that an override entry depends on notifies the override entry's flag as well. +""" +from typing import List, Optional + +from ldclient.impl.datasystem.store import Store +from ldclient.impl.dependency_tracker import KindAndKey +from ldclient.impl.listeners import Listeners +from ldclient.impl.overrides import OverrideLayer +from ldclient.interfaces import ( + Change, + ChangeSet, + ChangeType, + FlagChange, + IntentCode, + ObjectKind, + Selector +) +from ldclient.testing.builders import ( + FlagBuilder, + FlagRuleBuilder, + SegmentBuilder, + make_clause_matching_segment_key +) +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + + +def _flag(key: str, version: int, prerequisite: Optional[str] = None, segment: Optional[str] = None) -> dict: + builder = FlagBuilder(key).version(version).on(True).variations(True, False).fallthrough_variation(1) + if prerequisite is not None: + builder.prerequisite(prerequisite, 0) + if segment is not None: + builder.rules(FlagRuleBuilder().id('r').variation(0).clauses(make_clause_matching_segment_key(segment)).build()) + return builder.build().to_json_dict() + + +def _segment(key: str, version: int) -> dict: + return SegmentBuilder(key).version(version).build().to_json_dict() + + +def _put(kind: ObjectKind, item: dict) -> Change: + return Change(action=ChangeType.PUT, kind=kind, key=item['key'], version=item['version'], object=item) + + +def _full(*changes: Change) -> ChangeSet: + return ChangeSet(intent_code=IntentCode.TRANSFER_FULL, changes=list(changes), selector=Selector.no_selector()) + + +def _delta(*changes: Change) -> ChangeSet: + return ChangeSet(intent_code=IntentCode.TRANSFER_CHANGES, changes=list(changes), selector=Selector.no_selector()) + + +class Fixture: + def __init__(self, layer: Optional[OverrideLayer] = None): + self.notified: List[str] = [] + listeners = Listeners() + listeners.add(lambda change: self.notified.append(change.key)) + self.store = Store(listeners, Listeners()) + if layer is not None: + self.store.set_external_dependencies(layer) + + def take_notified(self) -> List[str]: + result = sorted(self.notified) + del self.notified[:] + return result + + +def test_delta_to_a_prerequisite_notifies_the_override_flag_that_depends_on_it(): + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, prerequisite='ld-flag')}, {}) + f = Fixture(layer) + f.store.apply(_full(_put(ObjectKind.FLAG, _flag('ld-flag', 1))), False) + f.take_notified() + + f.store.apply(_delta(_put(ObjectKind.FLAG, _flag('ld-flag', 2))), False) + assert f.take_notified() == ['ld-flag', 'override-flag'] + + +def test_full_data_set_replacement_notifies_the_override_flags_that_depend_on_changed_items(): + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, segment='ld-segment')}, {}) + f = Fixture(layer) + f.store.apply(_full(_put(ObjectKind.SEGMENT, _segment('ld-segment', 1)), _put(ObjectKind.FLAG, _flag('unrelated', 1))), False) + f.take_notified() + + f.store.apply(_full(_put(ObjectKind.SEGMENT, _segment('ld-segment', 2)), _put(ObjectKind.FLAG, _flag('unrelated', 1))), False) + # The segment is not a flag, so only the dependent override flag is notified. + assert f.take_notified() == ['override-flag'] + + +def test_dependency_chain_crosses_between_the_store_and_the_layer(): + # LaunchDarkly flag -> override flag -> LaunchDarkly segment: the chain enters the layer + # and comes back out, and a segment change must reach the LaunchDarkly flag at the top. + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, segment='ld-segment')}, {}) + f = Fixture(layer) + f.store.apply(_full( + _put(ObjectKind.SEGMENT, _segment('ld-segment', 1)), + _put(ObjectKind.FLAG, _flag('ld-flag', 1, prerequisite='override-flag')), + ), False) + f.take_notified() + + f.store.apply(_delta(_put(ObjectKind.SEGMENT, _segment('ld-segment', 2))), False) + assert f.take_notified() == ['ld-flag', 'override-flag'] + + +def test_layer_replacement_changes_which_flags_a_store_change_notifies(): + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, prerequisite='ld-flag')}, {}) + f = Fixture(layer) + f.store.apply(_full(_put(ObjectKind.FLAG, _flag('ld-flag', 1))), False) + f.take_notified() + + layer.set_all({}, {}) + f.store.apply(_delta(_put(ObjectKind.FLAG, _flag('ld-flag', 2))), False) + assert f.take_notified() == ['ld-flag'] + + +def test_store_without_external_dependencies_notifies_only_its_own_dependents(): + f = Fixture() + f.store.apply(_full(_put(ObjectKind.FLAG, _flag('ld-flag', 1)), _put(ObjectKind.FLAG, _flag('top', 1, prerequisite='ld-flag'))), False) + f.take_notified() + + f.store.apply(_delta(_put(ObjectKind.FLAG, _flag('ld-flag', 2))), False) + assert f.take_notified() == ['ld-flag', 'top'] + + +def test_external_dependencies_do_not_hide_the_store_s_own_dependents(): + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, prerequisite='ld-flag')}, {}) + f = Fixture(layer) + f.store.apply(_full(_put(ObjectKind.FLAG, _flag('ld-flag', 1)), _put(ObjectKind.FLAG, _flag('top', 1, prerequisite='ld-flag'))), False) + f.take_notified() + + f.store.apply(_delta(_put(ObjectKind.FLAG, _flag('ld-flag', 2))), False) + assert f.take_notified() == ['ld-flag', 'override-flag', 'top'] diff --git a/ldclient/testing/impl/events/test_event_overrides.py b/ldclient/testing/impl/events/test_event_overrides.py new file mode 100644 index 00000000..4f4345a3 --- /dev/null +++ b/ldclient/testing/impl/events/test_event_overrides.py @@ -0,0 +1,188 @@ +""" +Tests for how analytics events treat override-affected evaluations: they appear in summary +counters only, under a counter that carries the override-affected marker, and they produce no +individual feature event and no debug event. +""" +import json +import time +from typing import Any, Dict, List + +import pytest + +from ldclient.async_config import AsyncConfig +from ldclient.config import Config +from ldclient.context import Context +from ldclient.impl.events.async_event_processor import ( + DefaultAsyncEventProcessor +) +from ldclient.impl.events.event_processor import DefaultEventProcessor +from ldclient.impl.events.event_processor_common import EventOutputFormatter +from ldclient.impl.events.event_summarizer import ( + EventSummarizer, + EventSummaryCounter +) +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.testing.builders import FlagBuilder +from ldclient.testing.impl.events.test_async_event_processor import MockAioHttp +from ldclient.testing.stub_util import MockHttp + +context = Context.builder('userkey').name('Red').build() +timestamp = 10000 + + +def tracked_flag(key: str = 'flagkey', version: int = 2): + """A flag that requests individual feature events and debug events far into the future.""" + return FlagBuilder(key).version(version).track_events(True).debug_events_until_date(int(time.time() * 1000) + 100000).build() + + +def evaluation(flag, value: str = 'value', variation: int = 1, override_affected: bool = False, track_events: bool = True) -> EventInputEvaluation: + return EventInputEvaluation(timestamp, context, flag.key, flag, variation, value, None, 'default', None, track_events, override_affected) + + +def events_of_kind(output: List[Dict[str, Any]], kind: str) -> List[Dict[str, Any]]: + return [e for e in output if e['kind'] == kind] + + +# --------------------------------------------------------------------------- +# Summarizer and output formatter +# --------------------------------------------------------------------------- + +def test_summarizer_keeps_override_affected_counters_separate(): + flag = tracked_flag() + es = EventSummarizer() + es.summarize_event(evaluation(flag)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(evaluation(flag)) + counters = es.snapshot().flags[flag.key].counters + assert counters == { + (1, flag.version, False): EventSummaryCounter(2, 'value'), + (1, flag.version, True): EventSummaryCounter(2, 'value'), + } + + +def test_summary_output_carries_the_marker_only_on_override_affected_counters(): + flag = tracked_flag() + es = EventSummarizer() + es.summarize_event(evaluation(flag)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(EventInputEvaluation(timestamp, context, 'unknown-flag', None, None, 'default', None, 'default', None, False, False)) + output = EventOutputFormatter(Config('SDK_KEY')).make_summary_event(es.snapshot()) + + counters = output['features'][flag.key]['counters'] + assert len(counters) == 2 + by_marker = {c.get('overrideAffected'): c for c in counters} + assert by_marker[None] == {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version} + assert by_marker[True] == {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True} + assert 'overrideAffected' not in json.dumps(output['features']['unknown-flag']) + + +def test_event_input_evaluation_defaults_to_not_override_affected(): + event = EventInputEvaluation(timestamp, context, 'flag', None, None, 'default', None, 'default') + assert event.override_affected is False + assert event.to_debugging_dict()['override_affected'] is False + + +# --------------------------------------------------------------------------- +# Sync event processor +# --------------------------------------------------------------------------- + +def flush_and_get_events(ep: DefaultEventProcessor, mock_http: MockHttp) -> List[Dict[str, Any]]: + ep.flush() + ep._wait_until_inactive() + assert mock_http.request_data is not None, 'Expected to get an HTTP request but did not get one' + return json.loads(mock_http.request_data) + + +def make_processor(mock_http: MockHttp) -> DefaultEventProcessor: + return DefaultEventProcessor(Config('SDK_KEY', diagnostic_opt_out=True), mock_http) + + +def test_override_affected_evaluation_produces_no_feature_or_debug_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + output = flush_and_get_events(ep, mock_http) + + assert [e['kind'] for e in output] == ['index', 'summary'] + counters = output[1]['features'][flag.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}] + + +def test_ordinary_evaluation_still_produces_feature_and_debug_events(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag)) + output = flush_and_get_events(ep, mock_http) + + assert sorted(e['kind'] for e in output) == ['debug', 'feature', 'index', 'summary'] + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}] + + +def test_mixed_evaluations_of_one_flag_split_into_two_counters_and_one_feature_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag)) + output = flush_and_get_events(ep, mock_http) + + assert len(events_of_kind(output, 'feature')) == 1 + assert len(events_of_kind(output, 'debug')) == 1 + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert sorted(counters, key=lambda c: c['count']) == [ + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}, + {'count': 2, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}, + ] + + +def test_override_affected_prerequisite_record_produces_no_feature_event(): + parent = tracked_flag('parent', 1) + prereq = tracked_flag('prereq', 7) + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(EventInputEvaluation(timestamp, context, prereq.key, prereq, 1, 'value', None, None, parent, True, True)) + output = flush_and_get_events(ep, mock_http) + + assert [e['kind'] for e in output] == ['index', 'summary'] + counters = output[1]['features'][prereq.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': prereq.version, 'overrideAffected': True}] + + +def test_override_affected_evaluation_still_produces_an_index_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + output = flush_and_get_events(ep, mock_http) + assert output[0]['kind'] == 'index' + assert output[0]['context'] == context.to_dict() + + +# --------------------------------------------------------------------------- +# Async event processor (shares the dispatch logic) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_async_processor_suppresses_individual_events_for_override_affected_evaluations(): + flag = tracked_flag() + mock_http = MockAioHttp() + ep = DefaultAsyncEventProcessor(AsyncConfig('SDK_KEY', diagnostic_opt_out=True), mock_http) + try: + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag)) + assert await ep.flush_and_wait(5) is True + output = json.loads(mock_http.request_data) + finally: + await ep.stop() + + assert sorted(e['kind'] for e in output) == ['debug', 'feature', 'index', 'summary'] + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert sorted(counters, key=lambda c: 'overrideAffected' in c) == [ + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}, + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}, + ] diff --git a/ldclient/testing/impl/events/test_event_summarizer.py b/ldclient/testing/impl/events/test_event_summarizer.py index d7cf4ad1..3b2a2f82 100644 --- a/ldclient/testing/impl/events/test_event_summarizer.py +++ b/ldclient/testing/impl/events/test_event_summarizer.py @@ -41,8 +41,8 @@ def test_summarize_event_increments_counters(): data = es.snapshot() expected = { - 'flag1': EventSummaryFlag({'user'}, 'default1', {(1, flag1.version): EventSummaryCounter(2, 'value1'), (2, flag1.version): EventSummaryCounter(1, 'value2')}), - 'flag2': EventSummaryFlag({'user'}, 'default2', {(1, flag2.version): EventSummaryCounter(1, 'value99')}), - 'badkey': EventSummaryFlag({'user'}, 'default3', {(None, None): EventSummaryCounter(1, 'default3')}), + 'flag1': EventSummaryFlag({'user'}, 'default1', {(1, flag1.version, False): EventSummaryCounter(2, 'value1'), (2, flag1.version, False): EventSummaryCounter(1, 'value2')}), + 'flag2': EventSummaryFlag({'user'}, 'default2', {(1, flag2.version, False): EventSummaryCounter(1, 'value99')}), + 'badkey': EventSummaryFlag({'user'}, 'default3', {(None, None, False): EventSummaryCounter(1, 'default3')}), } assert data.flags == expected diff --git a/ldclient/testing/impl/test_async_evaluator_overrides.py b/ldclient/testing/impl/test_async_evaluator_overrides.py new file mode 100644 index 00000000..e193d3bf --- /dev/null +++ b/ldclient/testing/impl/test_async_evaluator_overrides.py @@ -0,0 +1,154 @@ +""" +Tests for the override marking performed by the async evaluator. These mirror the key +scenarios of the sync evaluator override tests. +""" +import pytest + +from ldclient import Context +from ldclient.impl.events.types import EventFactory +from ldclient.testing.builders import * +from ldclient.testing.impl.test_async_evaluator import AsyncEvaluatorBuilder +from ldclient.testing.impl.test_evaluator_overrides import ( + assert_override_affected, + assert_reason_override_affected, + make_override_test_flag, + prereq_records, + require_prereq_record +) + +override_test_context = Context.create('userkey') +event_factory_with_reasons = EventFactory(True, lambda: 0) + + +@pytest.mark.asyncio +async def test_override_flag_marks_evaluation(): + flag = make_override_test_flag('feature').with_override_marker() + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_plain_flag_is_not_marked(): + flag = make_override_test_flag('feature') + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert_override_affected(False, result) + + +@pytest.mark.asyncio +async def test_malformed_override_flag_error_result_is_marked(): + flag = FlagBuilder('feature').on(True).fallthrough_variation(99).variations('off', 'on').build().with_override_marker() + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_prerequisite_cycle_through_an_override_flag_is_marked(): + prereq = make_override_test_flag('prereq', 'feature').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = AsyncEvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + assert prereq_records(result) == [] + + +@pytest.mark.asyncio +async def test_prerequisite_cycle_below_an_override_flag_keeps_the_flag_marking(): + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_override_prerequisite_marks_prerequisite_record_and_top_level(): + prereq = make_override_test_flag('prereq').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(True, require_prereq_record(result, 'prereq').reason) + + +@pytest.mark.asyncio +async def test_override_flag_does_not_mark_unaffected_prerequisite_record(): + prereq = make_override_test_flag('prereq') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +@pytest.mark.asyncio +async def test_unaffected_sibling_prerequisite_record_stays_unmarked(): + s1 = SegmentBuilder('s1').included(override_test_context.key).build() + d = make_override_test_flag('d').with_override_marker() + b = make_override_test_flag('b', 'd') + c = FlagBuilder('c').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(s1.key)).build() + ).build() + a = make_override_test_flag('a', 'b', 'c') + evaluator = AsyncEvaluatorBuilder().with_flag(b).with_flag(c).with_flag(d).with_segment(s1).build() + result = await evaluator.evaluate(a, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert len(prereq_records(result)) == 3 + assert_reason_override_affected(True, require_prereq_record(result, 'd').reason) + assert_reason_override_affected(True, require_prereq_record(result, 'b').reason) + assert_reason_override_affected(False, require_prereq_record(result, 'c').reason) + + +@pytest.mark.asyncio +async def test_override_segment_read_without_matching_marks_evaluation(): + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = AsyncEvaluatorBuilder().with_segment(segment).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_override_segment_referenced_by_segment_rule_marks_evaluation(): + nested = SegmentBuilder('nested-segment').included(override_test_context.key).build().with_override_marker() + outer = SegmentBuilder('outer-segment').rules(SegmentRuleBuilder().clauses(make_clause_matching_segment_key(nested.key)).build()).build() + flag = make_boolean_flag_matching_segment(outer) + evaluator = AsyncEvaluatorBuilder().with_segment(outer).with_segment(nested).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.value is True + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_missing_definitions_do_not_mark_evaluation(): + unrelated = make_override_test_flag('unrelated').with_override_marker() + flag = make_override_test_flag('feature', 'missing') + evaluator = AsyncEvaluatorBuilder().with_flag(unrelated).with_unknown_flag('missing').build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': 'missing'} + assert_override_affected(False, result) + + segment_flag = make_boolean_flag_with_clauses(make_clause_matching_segment_key('missing')) + evaluator = AsyncEvaluatorBuilder().with_unknown_segment('missing').build() + result = await evaluator.evaluate(segment_flag, override_test_context, event_factory_with_reasons) + assert_override_affected(False, result) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("flag_override,prereq_override", [(False, False), (True, False), (False, True), (True, True)]) +async def test_result_override_affected_matches_reason_indicator(flag_override: bool, prereq_override: bool): + prereq = make_override_test_flag('prereq') + if prereq_override: + prereq = prereq.with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + if flag_override: + flag = flag.with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(flag_override or prereq_override, result) + assert_reason_override_affected(prereq_override, require_prereq_record(result, 'prereq').reason) diff --git a/ldclient/testing/impl/test_evaluator_overrides.py b/ldclient/testing/impl/test_evaluator_overrides.py new file mode 100644 index 00000000..309c3a6b --- /dev/null +++ b/ldclient/testing/impl/test_evaluator_overrides.py @@ -0,0 +1,325 @@ +""" +Tests for the override marking performed by the evaluator. An evaluation is override-affected +when any flag or segment definition it reads carries the override marker. The marking +propagates upward from prerequisites to the flags that depend on them and never sideways or +downward. +""" +from typing import List, Optional + +import pytest + +from ldclient import Context +from ldclient.impl.evaluator_common import EvalResult, is_override_definition +from ldclient.impl.events.types import EventFactory, EventInputEvaluation +from ldclient.testing.builders import * +from ldclient.testing.impl.evaluator_util import EvaluatorBuilder + +override_test_context = Context.create('userkey') + +# Events carry their reasons, so a prerequisite record's marking is visible on its reason. +event_factory_with_reasons = EventFactory(True, lambda: 0) + + +def assert_override_affected(expected: bool, result: EvalResult) -> None: + """Checks the reason indicator and the result scalar together. Both report the same marking.""" + assert result.override_affected is expected, "result.override_affected" + assert_reason_override_affected(expected, result.detail.reason) + + +def assert_reason_override_affected(expected: bool, reason: Optional[dict]) -> None: + assert reason is not None + if expected: + assert reason.get('overrideAffected') is True, "reason indicator: %s" % reason + else: + assert 'overrideAffected' not in reason, "reason indicator: %s" % reason + + +def make_override_test_flag(key: str, *prereq_keys: str) -> FeatureFlag: + """A flag that is on and serves variation 1 ("on") by fallthrough, with prerequisites that must each serve variation 1.""" + builder = FlagBuilder(key).on(True).fallthrough_variation(1).off_variation(0).variations('off', 'on') + for prereq_key in prereq_keys: + builder.prerequisite(prereq_key, 1) + return builder.build() + + +def prereq_records(result: EvalResult) -> List[EventInputEvaluation]: + return result.events or [] + + +def require_prereq_record(result: EvalResult, prereq_key: str) -> EventInputEvaluation: + for event in prereq_records(result): + if event.flag is not None and event.flag.key == prereq_key: + return event + raise AssertionError("no record for prerequisite %s" % prereq_key) + + +def test_override_flag_marks_evaluation_when_off(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'OFF' + assert result.detail.value == 'off' + assert_override_affected(True, result) + + +def test_override_flag_marks_evaluation_on_fallthrough(): + flag = make_override_test_flag('feature').with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + +def test_override_flag_marks_evaluation_on_rule_match(): + rule = FlagRuleBuilder().id('rule-id').variation(1).clauses(make_clause_matching_context(override_test_context)).build() + flag = FlagBuilder('feature').on(True).fallthrough_variation(0).variations('off', 'on').rules(rule).build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.reason['ruleId'] == 'rule-id' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + +def test_plain_flag_alone_is_not_marked(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'OFF'} + assert_override_affected(False, result) + + +def test_plain_flag_with_prerequisite_and_segment_is_not_marked(): + segment = SegmentBuilder('segment').included(override_test_context.key).build() + prereq = FlagBuilder('prereq').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(segment.key)).build() + ).build() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert result.detail.value == 'on' + assert_override_affected(False, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +def test_malformed_override_flag_error_result_is_marked(): + flag = FlagBuilder('feature').on(True).fallthrough_variation(99).variations('off', 'on').build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'ERROR' + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert result.detail.value is None + assert_override_affected(True, result) + + +def test_prerequisite_cycle_through_an_override_flag_is_marked(): + # feature -> prereq -> feature; only prereq is an override + prereq = make_override_test_flag('prereq', 'feature').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'ERROR' + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + assert prereq_records(result) == [] + + +def test_prerequisite_cycle_below_an_override_flag_keeps_the_flag_marking(): + # feature -> prereq -> feature; only feature is an override. The failure happens inside the + # plain prerequisite's subtree, and the top-level marking from the flag's own read survives it. + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +def test_prerequisite_cycle_through_plain_flags_is_not_marked(): + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(False, result) + + +def test_override_prerequisite_marks_prerequisite_record_and_top_level(): + prereq = make_override_test_flag('prereq').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + record = require_prereq_record(result, 'prereq') + assert record.reason is not None + assert record.reason['kind'] == 'FALLTHROUGH' + assert_reason_override_affected(True, record.reason) + + +def test_override_flag_does_not_mark_unaffected_prerequisite_record(): + # The marking propagates upward only. The top-level flag's own marker does not leak into + # the record of a prerequisite whose subtree read no override definition. + prereq = make_override_test_flag('prereq') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +def test_override_prerequisite_at_depth_two_marks_all_affected_scopes(): + prereq2 = make_override_test_flag('prereq2').with_override_marker() + prereq1 = make_override_test_flag('prereq1', 'prereq2') + flag = make_override_test_flag('feature', 'prereq1') + evaluator = EvaluatorBuilder().with_flag(prereq1).with_flag(prereq2).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + # The nested record is produced first, during the evaluation of prereq1. + records = prereq_records(result) + assert [r.flag.key for r in records] == ['prereq2', 'prereq1'] + assert_reason_override_affected(True, records[0].reason) + assert_reason_override_affected(True, records[1].reason) + + +def test_unaffected_sibling_prerequisite_record_stays_unmarked(): + # Flag a has prerequisites b and c. Only d, a prerequisite of b, is an override. The marking + # reaches a, b, and d. It does not reach the sibling c, and the plain segments s1 and s2 + # mark nothing. + s1 = SegmentBuilder('s1').included(override_test_context.key).build() + s2 = SegmentBuilder('s2').included(override_test_context.key).build() + d = make_override_test_flag('d').with_override_marker() + b = make_override_test_flag('b', 'd') + c = FlagBuilder('c').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(s1.key)).build() + ).build() + a = FlagBuilder('a').on(True).fallthrough_variation(0).off_variation(0).prerequisite('b', 1).prerequisite('c', 1).rules( + FlagRuleBuilder().id('rule-s2').variation(1).clauses(make_clause_matching_segment_key(s2.key)).build() + ).variations('off', 'on').build() + + evaluator = EvaluatorBuilder().with_flag(b).with_flag(c).with_flag(d).with_segment(s1).with_segment(s2).build() + result = evaluator.evaluate(a, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + assert len(prereq_records(result)) == 3 + assert_reason_override_affected(True, require_prereq_record(result, 'd').reason) + assert_reason_override_affected(True, require_prereq_record(result, 'b').reason) + assert_reason_override_affected(False, require_prereq_record(result, 'c').reason) + + +def test_override_segment_referenced_by_flag_rule_marks_evaluation(): + segment = SegmentBuilder('segment').included(override_test_context.key).build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_read_without_matching_marks_evaluation(): + # A read is enough to mark the evaluation. The segment does not need to match. + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert result.detail.value is False + assert_override_affected(True, result) + + +def test_override_segment_read_through_negated_clause_marks_evaluation(): + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_with_clauses(negate_clause(make_clause_matching_segment_key(segment.key))) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_referenced_by_segment_rule_marks_evaluation(): + # The outer segment is plain. A rule of the outer segment reads a nested override segment. + nested = SegmentBuilder('nested-segment').included(override_test_context.key).build().with_override_marker() + outer = SegmentBuilder('outer-segment').rules(SegmentRuleBuilder().clauses(make_clause_matching_segment_key(nested.key)).build()).build() + flag = make_boolean_flag_matching_segment(outer) + evaluator = EvaluatorBuilder().with_segment(outer).with_segment(nested).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_referenced_by_prerequisite_marks_prerequisite_record_and_top_level(): + segment = SegmentBuilder('segment').included(override_test_context.key).build().with_override_marker() + prereq = FlagBuilder('prereq').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(segment.key)).build() + ).build() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + assert_reason_override_affected(True, require_prereq_record(result, 'prereq').reason) + + +def test_missing_prerequisite_does_not_mark_evaluation(): + # A definition that cannot be resolved contributes nothing, because nothing was read. The + # store holds an unrelated override definition to show that only reads count. + unrelated = make_override_test_flag('unrelated').with_override_marker() + flag = make_override_test_flag('feature', 'missing') + evaluator = EvaluatorBuilder().with_flag(unrelated).with_unknown_flag('missing').build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': 'missing'} + assert result.detail.value == 'off' + assert_override_affected(False, result) + assert prereq_records(result) == [] + + +def test_missing_segment_does_not_mark_evaluation(): + unrelated = SegmentBuilder('unrelated').included(override_test_context.key).build().with_override_marker() + flag = make_boolean_flag_with_clauses(make_clause_matching_segment_key('missing')) + evaluator = EvaluatorBuilder().with_segment(unrelated).with_unknown_segment('missing').build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert result.detail.value is False + assert_override_affected(False, result) + + +@pytest.mark.parametrize("flag_override,prereq_override", [(False, False), (True, False), (False, True), (True, True)]) +def test_result_override_affected_matches_reason_indicator(flag_override: bool, prereq_override: bool): + prereq = make_override_test_flag('prereq') + if prereq_override: + prereq = prereq.with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + if flag_override: + flag = flag.with_override_marker() + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(flag_override or prereq_override, result) + assert_reason_override_affected(prereq_override, require_prereq_record(result, 'prereq').reason) + + +def test_marking_does_not_disturb_other_reason_properties(): + segment = SegmentBuilder('segment').unbounded(True).generation(1).build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).with_no_big_segments_for_key(override_test_context.key).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH', 'bigSegmentsStatus': 'HEALTHY', 'overrideAffected': True} + + +def test_reason_indicator_is_absent_when_events_do_not_carry_reasons(): + # The marking does not depend on the event factory. The reason on the result is marked either way. + flag = make_override_test_flag('feature').with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, EventFactory(False, lambda: 0)) + assert_override_affected(True, result) + + +def test_is_override_definition_reads_the_marker_and_never_fails(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build() + assert is_override_definition(flag.with_override_marker()) is True + assert is_override_definition(flag) is False + # An item without the marker attribute, such as a raw dict, reads as unmarked. + assert is_override_definition(flag.to_json_dict()) is False + assert is_override_definition(None) is False diff --git a/ldclient/testing/impl/test_model_override_marker.py b/ldclient/testing/impl/test_model_override_marker.py new file mode 100644 index 00000000..203e0fc5 --- /dev/null +++ b/ldclient/testing/impl/test_model_override_marker.py @@ -0,0 +1,76 @@ +""" +Tests for the override marker carried on the flag and segment models. The marker lives on the +model type and never in the JSON representation. +""" +import json + +import pytest + +from ldclient.impl.model import FeatureFlag, ModelEncoder, Segment +from ldclient.testing.builders import FlagBuilder, SegmentBuilder +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + +flag_data = {'key': 'flag1', 'version': 2, 'on': True, 'variations': [True, False], 'fallthrough': {'variation': 0}, 'salt': 'x'} +segment_data = {'key': 'seg1', 'version': 3, 'included': ['user1'], 'salt': 'y'} + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marker_defaults_to_false(kind, data): + entity = kind.decode(dict(data)) + assert entity.is_override is False + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marked_copy_is_marked_and_source_is_not(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.is_override is True + assert entity.is_override is False + assert type(marked) is type(entity) + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marker_is_never_serialized(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.to_json_dict() == data + assert kind.encode(marked) == data + assert json.loads(ModelEncoder().encode(marked)) == data + assert 'is_override' not in marked.to_json_dict() + assert 'override' not in json.dumps(marked.to_json_dict()).lower() + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marked_copy_shares_the_definition(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.key == entity.key + assert marked.version == entity.version + assert marked.to_json_dict() is entity.to_json_dict() + assert marked == entity + + +def test_marked_flag_keeps_its_parsed_properties(): + flag = FlagBuilder('flag1').version(7).on(True).variations('a', 'b').fallthrough_variation(1).prerequisite('p', 0).track_events(True).build() + marked = flag.with_override_marker() + assert marked.on is True + assert marked.variations == ['a', 'b'] + assert marked.fallthrough.variation == 1 + assert [p.key for p in marked.prerequisites] == ['p'] + assert marked.track_events is True + assert marked.version == 7 + + +def test_marked_segment_keeps_its_parsed_properties(): + segment = SegmentBuilder('seg1').version(4).included('a').excluded('b').build() + marked = segment.with_override_marker() + assert marked.included == {'a'} + assert marked.excluded == {'b'} + assert marked.version == 4 + + +def test_marking_a_marked_copy_is_a_marked_copy(): + flag = FeatureFlag(dict(flag_data)).with_override_marker() + again = flag.with_override_marker() + assert again.is_override is True + assert again is not flag diff --git a/ldclient/testing/impl/test_overrides.py b/ldclient/testing/impl/test_overrides.py new file mode 100644 index 00000000..cfef2d9a --- /dev/null +++ b/ldclient/testing/impl/test_overrides.py @@ -0,0 +1,458 @@ +""" +Tests for the override layer, the overlay at the store read boundary, and the sink that applies +snapshots and notifies flag change listeners. +""" +from typing import Any, Dict, List, Optional + +import pytest + +from ldclient.impl.datasystem.store import InMemoryFeatureStore +from ldclient.impl.dependency_tracker import KindAndKey +from ldclient.impl.overrides import ( + AsyncOverrideStoreView, + OverrideLayer, + OverrideSinkImpl, + OverrideStoreView +) +from ldclient.testing.builders import ( + FlagBuilder, + FlagRuleBuilder, + SegmentBuilder, + make_clause_matching_segment_key +) +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + + +class FakeBaseStore: + """A minimal read-only store with controllable contents, initialization state, and failures.""" + + def __init__(self, flags: Optional[Dict[str, Any]] = None, segments: Optional[Dict[str, Any]] = None, initialized: bool = True): + self.flags = dict(flags or {}) + self.segments = dict(segments or {}) + self._initialized = initialized + self.fail_all = False + + def _items(self, kind): + return self.flags if kind == FEATURES else self.segments + + def get(self, kind, key, callback=lambda x: x): + return callback(self._items(kind).get(key)) + + def all(self, kind, callback=lambda x: x): + if self.fail_all: + raise RuntimeError("store failure") + return callback(dict(self._items(kind))) + + @property + def initialized(self) -> bool: + return self._initialized + + +class FakeAsyncBaseStore: + def __init__(self, base: FakeBaseStore): + self._base = base + + async def get(self, kind, key): + return self._base.get(kind, key) + + async def all(self, kind): + return self._base.all(kind) + + +def flag(key: str, version: int = 1): + return FlagBuilder(key).version(version).on(True).variations(False, True).fallthrough_variation(1).off_variation(0).build() + + +def segment(key: str, version: int = 1): + return SegmentBuilder(key).version(version).build() + + +# --------------------------------------------------------------------------- +# Layer +# --------------------------------------------------------------------------- + +def test_layer_marks_copies_without_mutating_source(): + layer = OverrideLayer() + f = flag('flag1', 2) + s = segment('segment1', 3) + layer.set_all({'flag1': f}, {'segment1': s}) + + assert f.is_override is False + assert s.is_override is False + stored_flag = layer.get(FEATURES, 'flag1') + assert stored_flag is not None + assert stored_flag.is_override is True + assert stored_flag.version == 2 + stored_segment = layer.get(SEGMENTS, 'segment1') + assert stored_segment is not None + assert stored_segment.is_override is True + + +def test_layer_decodes_dictionaries_and_marks_them(): + layer = OverrideLayer() + layer.set_all({'flag1': flag('flag1').to_json_dict()}, {'segment1': segment('segment1').to_json_dict()}) + stored_flag = layer.get(FEATURES, 'flag1') + assert stored_flag is not None + assert stored_flag.is_override is True + assert stored_flag.on is True + stored_segment = layer.get(SEGMENTS, 'segment1') + assert stored_segment is not None + assert stored_segment.is_override is True + + +def test_layer_rejects_invalid_definitions_and_keeps_its_contents(): + layer = OverrideLayer() + layer.set_all({'flag1': flag('flag1')}, {}) + with pytest.raises(ValueError): + layer.set_all({'flag2': {'key': 'flag2', 'version': 'not a number'}}, {}) + assert layer.get(FEATURES, 'flag1') is not None + assert layer.get(FEATURES, 'flag2') is None + with pytest.raises(ValueError): + layer.set_all({}, {'seg': {'key': 'seg', 'version': 1, 'included': 'not a list'}}) + assert layer.get(FEATURES, 'flag1') is not None + + +def test_layer_replacement_semantics(): + layer = OverrideLayer() + assert layer.is_empty is True + assert layer.get(FEATURES, 'flag1') is None + + layer.set_all({'flag1': flag('flag1')}, {}) + assert layer.is_empty is False + assert layer.get(FEATURES, 'flag1') is not None + + # A replacement is a full snapshot: entries absent from it are removed. + layer.set_all({'flag2': flag('flag2')}, {}) + assert layer.get(FEATURES, 'flag1') is None + assert layer.get(FEATURES, 'flag2') is not None + + layer.set_all({}, {}) + assert layer.is_empty is True + assert layer.get(FEATURES, 'flag2') is None + + +def test_layer_set_all_returns_previous_and_current_contents(): + layer = OverrideLayer() + previous, current = layer.set_all({'flag1': flag('flag1')}, {'seg': segment('seg')}) + assert previous == {FEATURES: {}, SEGMENTS: {}} + assert list(current[FEATURES].keys()) == ['flag1'] + assert list(current[SEGMENTS].keys()) == ['seg'] + previous, current = layer.set_all({}, {}) + assert list(previous[FEATURES].keys()) == ['flag1'] + assert current == {FEATURES: {}, SEGMENTS: {}} + + +def test_layer_all_returns_entries_of_a_kind(): + layer = OverrideLayer() + layer.set_all({'a': flag('a'), 'b': flag('b')}, {'s': segment('s')}) + assert sorted(layer.all(FEATURES).keys()) == ['a', 'b'] + assert list(layer.all(SEGMENTS).keys()) == ['s'] + assert all(item.is_override for item in layer.all(FEATURES).values()) + + +# --------------------------------------------------------------------------- +# Overlay +# --------------------------------------------------------------------------- + +def test_layer_add_affected_items_reaches_the_entries_that_depend_on_an_item(): + top = FlagBuilder('top').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('mid', 0).build() + mid = FlagBuilder('mid').version(1).on(True).variations(True, False).fallthrough_variation(1).rules( + FlagRuleBuilder().id('r').variation(0).clauses(make_clause_matching_segment_key('seg')).build() + ).build() + layer = OverrideLayer() + layer.set_all({'top': top, 'mid': mid, 'other': flag('other')}, {}) + + # The segment is not an override entry; the chain still starts from it. + affected = set() + layer.add_affected_items(affected, KindAndKey(kind=SEGMENTS, key='seg')) + assert affected == {KindAndKey(SEGMENTS, 'seg'), KindAndKey(FEATURES, 'mid'), KindAndKey(FEATURES, 'top')} + + affected = set() + layer.add_affected_items(affected, KindAndKey(kind=FEATURES, key='other')) + assert affected == {KindAndKey(FEATURES, 'other')} + + +def test_layer_dependency_graph_follows_each_replacement(): + top = FlagBuilder('top').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('leaf', 0).build() + layer = OverrideLayer() + layer.set_all({'top': top}, {}) + affected = set() + layer.add_affected_items(affected, KindAndKey(kind=FEATURES, key='leaf')) + assert affected == {KindAndKey(FEATURES, 'leaf'), KindAndKey(FEATURES, 'top')} + + layer.set_all({}, {}) + affected = set() + layer.add_affected_items(affected, KindAndKey(kind=FEATURES, key='leaf')) + assert affected == {KindAndKey(FEATURES, 'leaf')} + + +def test_overlay_get_precedence(): + base = FakeBaseStore(flags={'both': flag('both', 1), 'base-only': flag('base-only', 1)}) + layer = OverrideLayer() + layer.set_all({'both': flag('both', 99), 'override-only': flag('override-only', 1)}, {}) + overlay = OverrideStoreView(base, layer) + + item = overlay.get(FEATURES, 'both') + assert item.version == 99 + assert item.is_override is True + + item = overlay.get(FEATURES, 'base-only') + assert item.is_override is False + + item = overlay.get(FEATURES, 'override-only') + assert item.is_override is True + + assert overlay.get(FEATURES, 'nowhere') is None + assert overlay.get(FEATURES, 'both', lambda x: x.version) == 99 + + +def test_overlay_get_serves_overrides_from_uninitialized_base(): + base = FakeBaseStore(initialized=False) + layer = OverrideLayer() + layer.set_all({'flag1': flag('flag1')}, {}) + overlay = OverrideStoreView(base, layer) + item = overlay.get(FEATURES, 'flag1') + assert item.is_override is True + assert overlay.initialized is False + + +def test_overlay_get_all_union(): + memory = InMemoryFeatureStore() + memory.set_basis({ + FEATURES: { + 'both': flag('both', 1).to_json_dict(), + 'base-only': flag('base-only', 1).to_json_dict(), + 'tombstone': {'key': 'tombstone', 'version': 5, 'deleted': True}, + }, + SEGMENTS: {}, + }) + layer = OverrideLayer() + layer.set_all({'both': flag('both', 99), 'tombstone': flag('tombstone', 1), 'override-only': flag('override-only', 1)}, {}) + overlay = OverrideStoreView(memory, layer) + + items = overlay.all(FEATURES) + assert sorted(items.keys()) == ['base-only', 'both', 'override-only', 'tombstone'] + assert items['both'].version == 99 + assert items['both'].is_override is True + assert items['base-only'].is_override is False + assert items['tombstone'].is_override is True, "an override wins over a deleted item" + assert items['override-only'].is_override is True + assert overlay.all(FEATURES, lambda x: len(x)) == 4 + + +def test_overlay_get_all_with_empty_layer_is_passthrough(): + base = FakeBaseStore(flags={'flag1': flag('flag1')}) + overlay = OverrideStoreView(base, OverrideLayer()) + assert list(overlay.all(FEATURES).keys()) == ['flag1'] + base.fail_all = True + with pytest.raises(RuntimeError): + overlay.all(FEATURES) + + +def test_overlay_get_all_propagates_a_base_failure_when_the_layer_holds_entries(): + # The layer's entries alone would look like a complete result to a caller such as the + # all-flags state, so the failure propagates as it does without the overlay. A per-key read + # still serves the override. + base = FakeBaseStore() + base.fail_all = True + layer = OverrideLayer() + layer.set_all({'override-1': flag('override-1', 1)}, {}) + overlay = OverrideStoreView(base, layer) + with pytest.raises(RuntimeError): + overlay.all(FEATURES) + assert overlay.get(FEATURES, 'override-1').is_override is True + + +def test_overlay_get_all_keeps_a_missing_base_result_missing(): + # A store returns no result when it cannot read. The overlay does not turn that into the + # layer's entries alone. + class NoResultStore(FakeBaseStore): + def all(self, kind, callback=lambda x: x): + return callback(None) + + layer = OverrideLayer() + layer.set_all({'override-1': flag('override-1', 1)}, {}) + assert OverrideStoreView(NoResultStore(), layer).all(FEATURES) is None + + +def test_overlay_initialized_follows_the_base(): + base = FakeBaseStore(initialized=True) + overlay = OverrideStoreView(base, OverrideLayer()) + assert overlay.initialized is True + base._initialized = False + assert overlay.initialized is False + + +@pytest.mark.asyncio +async def test_async_overlay_get_and_all(): + base = FakeBaseStore(flags={'both': flag('both', 1), 'base-only': flag('base-only', 1)}, initialized=False) + layer = OverrideLayer() + layer.set_all({'both': flag('both', 99), 'override-only': flag('override-only', 1)}, {'seg': segment('seg')}) + overlay = AsyncOverrideStoreView(FakeAsyncBaseStore(base), layer) + + both = await overlay.get(FEATURES, 'both') + assert both.version == 99 and both.is_override is True + base_only = await overlay.get(FEATURES, 'base-only') + assert base_only.is_override is False + assert (await overlay.get(FEATURES, 'nowhere')) is None + assert (await overlay.get(SEGMENTS, 'seg')).is_override is True + + items = await overlay.all(FEATURES) + assert sorted(items.keys()) == ['base-only', 'both', 'override-only'] + assert items['both'].version == 99 + + +@pytest.mark.asyncio +async def test_async_overlay_all_when_base_fails(): + base = FakeBaseStore() + base.fail_all = True + layer = OverrideLayer() + overlay = AsyncOverrideStoreView(FakeAsyncBaseStore(base), layer) + with pytest.raises(RuntimeError): + await overlay.all(FEATURES) + # The failure propagates whatever the layer holds. + layer.set_all({'override-1': flag('override-1')}, {}) + with pytest.raises(RuntimeError): + await overlay.all(FEATURES) + assert (await overlay.get(FEATURES, 'override-1')).is_override is True + + +# --------------------------------------------------------------------------- +# Sink +# --------------------------------------------------------------------------- + +class SinkFixture: + def __init__(self, base: FakeBaseStore): + self.base = base + self.layer = OverrideLayer() + self.notified: List[str] = [] + self.listening = True + self.sink = OverrideSinkImpl(self.layer, base, lambda key: self.notified.append(key), lambda: self.listening) + + def take_notified(self) -> List[str]: + result = sorted(self.notified) + del self.notified[:] + return result + + +def test_sink_notifies_on_add_change_remove(): + base = FakeBaseStore(flags={'flag1': flag('flag1', 1)}) + f = SinkFixture(base) + + # Adding an override is a change even though flag1 also exists in base data. + f.sink.set_overrides({'flag1': flag('flag1', 1), 'flag2': flag('flag2', 1)}, {}) + assert f.take_notified() == ['flag1', 'flag2'] + + # An identical replacement (rebuilt from scratch, new objects) changes nothing. + f.sink.set_overrides({'flag1': flag('flag1', 1), 'flag2': flag('flag2', 1)}, {}) + assert f.take_notified() == [] + + # Changing one entry notifies only that entry. + f.sink.set_overrides({'flag1': flag('flag1', 1), 'flag2': flag('flag2', 2)}, {}) + assert f.take_notified() == ['flag2'] + + # A change in content at the same version is a change. + f.sink.set_overrides({'flag1': flag('flag1', 1), 'flag2': FlagBuilder('flag2').version(2).on(False).build()}, {}) + assert f.take_notified() == ['flag2'] + + # Removing overrides notifies them: flag1 reverts to base data, flag2 to not-found. + f.sink.set_overrides({}, {}) + assert f.take_notified() == ['flag1', 'flag2'] + + +def test_sink_segment_override_fans_out_to_dependent_flags(): + dependent = FlagBuilder('dependent').version(1).on(True).variations(True, False).fallthrough_variation(1).rules( + FlagRuleBuilder().id('r').variation(0).clauses(make_clause_matching_segment_key('segment1')).build() + ).build() + base = FakeBaseStore( + flags={'dependent': dependent, 'unrelated': flag('unrelated')}, + segments={'segment1': segment('segment1', 1)}, + ) + f = SinkFixture(base) + f.sink.set_overrides({}, {'segment1': segment('segment1', 99)}) + # The segment itself is not a flag, so only the dependent flag is notified. + assert f.take_notified() == ['dependent'] + + +def test_sink_prerequisite_override_fans_out_through_the_chain(): + top = FlagBuilder('top').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('mid', 0).build() + mid = FlagBuilder('mid').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('leaf', 0).build() + base = FakeBaseStore(flags={'top': top, 'mid': mid, 'leaf': flag('leaf'), 'other': flag('other')}) + f = SinkFixture(base) + f.sink.set_overrides({'leaf': flag('leaf', 5)}, {}) + assert f.take_notified() == ['leaf', 'mid', 'top'] + + +def test_sink_removing_a_segment_override_notifies_flags_that_reference_it(): + # The override of "dependent" references segment "s". Removing the override of "s" alone + # changes what "dependent" evaluates to, so "dependent" is notified through the dependency + # fan-out even though its own override did not change. + dependent_override = FlagBuilder('dependent').version(2).on(True).variations(True, False).fallthrough_variation(1).rules( + FlagRuleBuilder().variation(0).clauses(make_clause_matching_segment_key('s')).build() + ).build() + base = FakeBaseStore(flags={'dependent': flag('dependent')}, segments={'s': segment('s')}) + f = SinkFixture(base) + f.sink.set_overrides({'dependent': dependent_override}, {'s': segment('s', 2)}) + f.take_notified() + f.sink.set_overrides({'dependent': dependent_override}, {}) + assert f.take_notified() == ['dependent'] + + +def test_sink_skips_change_computation_when_nothing_listens(): + base = FakeBaseStore(flags={'flag1': flag('flag1')}) + f = SinkFixture(base) + f.listening = False + f.sink.set_overrides({'flag1': flag('flag1', 2)}, {}) + assert f.take_notified() == [] + assert f.layer.get(FEATURES, 'flag1').version == 2 + + +def test_sink_close_keeps_the_layer_and_ignores_later_snapshots(): + f = SinkFixture(FakeBaseStore()) + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + assert f.take_notified() == ['flag1'] + + f.sink.close() + assert f.layer.get(FEATURES, 'flag1') is not None + assert f.take_notified() == [] + + # A snapshot after closing changes nothing. + f.sink.set_overrides({'flag2': flag('flag2')}, {}) + assert f.layer.get(FEATURES, 'flag1') is not None + assert f.layer.get(FEATURES, 'flag2') is None + assert f.take_notified() == [] + + +def test_sink_close_and_clear_clears_the_layer_and_ignores_later_snapshots(): + f = SinkFixture(FakeBaseStore()) + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + assert f.take_notified() == ['flag1'] + + # Clearing removes what the source supplied, and the removal is a change. + f.sink.close_and_clear() + assert f.layer.is_empty is True + assert f.take_notified() == ['flag1'] + + # A snapshot after closing changes nothing. + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + assert f.layer.is_empty is True + assert f.take_notified() == [] + + +def test_sink_notifies_directly_changed_keys_when_base_read_fails(): + base = FakeBaseStore(flags={'flag1': flag('flag1')}) + base.fail_all = True + f = SinkFixture(base) + f.sink.set_overrides({'flag1': flag('flag1', 2)}, {}) + assert f.take_notified() == ['flag1'] + + +def test_sink_invalid_snapshot_raises_and_changes_nothing(): + base = FakeBaseStore() + f = SinkFixture(base) + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + f.take_notified() + with pytest.raises(ValueError): + f.sink.set_overrides({'flag1': {'key': 'flag1', 'version': 'bad'}}, {}) + assert f.layer.get(FEATURES, 'flag1').version == 1 + assert f.take_notified() == [] diff --git a/ldclient/testing/integrations/test_file_override_source.py b/ldclient/testing/integrations/test_file_override_source.py new file mode 100644 index 00000000..abfff9b8 --- /dev/null +++ b/ldclient/testing/integrations/test_file_override_source.py @@ -0,0 +1,732 @@ +""" +Tests for the file-based override source: its builder, initial load, multi-file merging, +absent files, change detection in both modes, failure retention, and its Info log. +""" +import logging +import ntpath +import os +import threading +import time +from queue import Empty, Queue +from typing import Any, Dict, List, Optional, Set, Tuple + +import pytest + +from ldclient.client import Config, Context, LDClient +from ldclient.config import Config as SDKConfig +from ldclient.datasystem import custom +from ldclient.impl.integrations.files import filedata +from ldclient.impl.integrations.overrides import file_override_source +from ldclient.impl.integrations.overrides.file_override_source import ( + _FileOverrideSource +) +from ldclient.integrations.overrides import ( + ChangeDetection, + DuplicateKeysHandling, + FileOverrideSourceBuilder +) +from ldclient.testing.file_util import rewrite_in_place, truncate +from ldclient.testing.mock_components import HangingSynchronizer +from ldclient.testing.stub_util import MockEventProcessor + +TEST_TIMEOUT = 10.0 +QUIET_PERIOD = 0.3 + +watchdog_required = pytest.mark.skipif(not filedata.have_watchdog, reason="watchdog is not installed") +yaml_required = pytest.mark.skipif(not filedata.have_yaml, reason="pyyaml is not installed") + +Snapshot = Tuple[Dict[str, Any], Dict[str, Any]] + + +class CapturingSink: + """Records every override snapshot it receives.""" + + def __init__(self): + self.snapshots: Queue = Queue() + + def set_overrides(self, flags, segments) -> None: + self.snapshots.put((dict(flags), dict(segments))) + + def require_snapshot(self, timeout: float = TEST_TIMEOUT) -> Snapshot: + try: + return self.snapshots.get(timeout=timeout) + except Empty: + pytest.fail("timed out waiting for an override snapshot") + + def require_no_snapshot(self, duration: float = QUIET_PERIOD) -> None: + try: + snapshot = self.snapshots.get(timeout=duration) + except Empty: + return + pytest.fail("received an unexpected override snapshot: %r" % (snapshot,)) + + +def write_file(path: str, content: str) -> None: + with open(path, 'w') as f: + f.write(content) + + +def flag_values(snapshot: Snapshot) -> Dict[str, Any]: + """The single variation of each flag in a snapshot, keyed by flag key.""" + return {key: flag.variations[0] for key, flag in snapshot[0].items()} + + +@pytest.fixture +def sources(): + started: List[_FileOverrideSource] = [] + yield started + for source in started: + source.close() + + +def build_source(sources, paths, configure=None) -> Tuple[_FileOverrideSource, CapturingSink]: + builder = FileOverrideSourceBuilder(paths) + if configure is not None: + configure(builder) + source = builder.build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + sink = CapturingSink() + source.start(sink) + sources.append(source) + return source, sink + + +def info_lines(caplog) -> List[str]: + return [r.getMessage() for r in caplog.records if r.levelno == logging.INFO] + + +def require_info_line(caplog, expected: str) -> None: + """Waits for an Info line equal to the expected text. The source logs after it hands the snapshot to the sink.""" + deadline = time.time() + TEST_TIMEOUT + while True: + if expected in info_lines(caplog): + return + if time.time() > deadline: + pytest.fail("timed out waiting for the Info line %r; Info output: %r" % (expected, info_lines(caplog))) + time.sleep(0.01) + + +# --------------------------------------------------------------------------- +# Builder +# --------------------------------------------------------------------------- + +def test_builder_requires_paths(): + with pytest.raises(ValueError): + FileOverrideSourceBuilder([]).build(SDKConfig('SDK_KEY')) + + +def test_builder_accepts_a_single_path_string(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + source = FileOverrideSourceBuilder(path).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [path] + + +def test_builder_resolves_relative_paths(): + source = FileOverrideSourceBuilder(['relative/overrides.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [os.path.abspath('relative/overrides.json')] + + +def test_builder_drops_a_repeated_path_and_warns(caplog): + # Under the default fail handling the same file listed twice would make every one of its keys + # a duplicate of itself. The repeat is ignored and the first occurrence keeps its place. + with caplog.at_level(logging.WARNING): + source = FileOverrideSourceBuilder(['/a.json', '/b.json', '/a.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [os.path.abspath('/a.json'), os.path.abspath('/b.json')] + assert any('more than once' in r.getMessage() for r in caplog.records) + + +def test_builder_treats_a_path_repeated_in_different_case_as_a_repeat_only_where_file_names_ignore_case(): + source = FileOverrideSourceBuilder(['/a.json', '/A.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + if os.name == 'nt': + assert source._paths == [os.path.abspath('/a.json')] + else: + assert source._paths == [os.path.abspath('/a.json'), os.path.abspath('/A.json')] + + +def test_builder_drops_a_path_repeated_in_different_case_on_windows(monkeypatch): + # Windows file names ignore case, so two spellings of one path name the same file. The path + # comparison of Windows is used here on any platform. + monkeypatch.setattr(os.path, 'normcase', ntpath.normcase) + source = FileOverrideSourceBuilder(['/a.json', '/b.json', '/A.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [os.path.abspath('/a.json'), os.path.abspath('/b.json')] + + +def test_builder_polls_by_default(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + source = FileOverrideSourceBuilder([path]).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._change_detection == ChangeDetection.POLLING + assert source._poll_interval == FileOverrideSourceBuilder.DEFAULT_POLL_INTERVAL + assert source._duplicate_keys_handling == DuplicateKeysHandling.FAIL + + +def test_builder_rejects_unknown_change_detection(): + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['a']).change_detection('notify') + assert 'notify' in str(excinfo.value) + + +def test_builder_rejects_unknown_duplicate_keys_handling(): + with pytest.raises(ValueError): + FileOverrideSourceBuilder(['a']).duplicate_keys_handling('merge') + + +def test_builder_accepts_string_values(): + builder = FileOverrideSourceBuilder(['a']).change_detection('polling').duplicate_keys_handling('ignore') + source = builder.build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._change_detection == ChangeDetection.POLLING + assert source._duplicate_keys_handling == DuplicateKeysHandling.IGNORE + + +def test_builder_clamps_poll_interval_to_the_minimum(caplog): + with caplog.at_level(logging.WARNING): + source = FileOverrideSourceBuilder(['a']).poll_interval(0.001).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._poll_interval == FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL + assert any('below the minimum' in r.getMessage() for r in caplog.records) + + source = FileOverrideSourceBuilder(['a']).poll_interval(2.5).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._poll_interval == 2.5 + + +@pytest.mark.parametrize('seconds', ['1', None, True], ids=['string', 'none', 'bool']) +def test_builder_rejects_a_poll_interval_that_is_not_a_number(seconds): + with pytest.raises(TypeError): + FileOverrideSourceBuilder(['a']).poll_interval(seconds) + + +def test_builder_rejects_a_poll_interval_longer_than_the_runtime_can_wait(): + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['a']).poll_interval(1e20) + assert 'must not exceed' in str(excinfo.value) + + +@pytest.mark.parametrize('seconds', [float('nan'), float('inf'), float('-inf')], ids=['nan', 'inf', '-inf']) +def test_builder_rejects_a_poll_interval_that_is_not_finite(seconds): + with pytest.raises(ValueError): + FileOverrideSourceBuilder(['a']).poll_interval(seconds) + + +def test_builder_rejects_watching_without_watchdog(monkeypatch): + import ldclient.integrations.overrides as module + monkeypatch.setattr(module, 'have_watchdog', False) + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['a']).change_detection(ChangeDetection.WATCHING).build(SDKConfig('SDK_KEY')) + assert 'watchdog' in str(excinfo.value) + + +@pytest.mark.parametrize('name', ['overrides.yaml', 'overrides.YML']) +def test_builder_rejects_a_yaml_file_without_pyyaml(monkeypatch, name): + # A YAML file can never load without a YAML parser, so it is a configuration error rather + # than a failure that every reload would report. + import ldclient.integrations.overrides as module + monkeypatch.setattr(module, 'have_yaml', False) + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['/overrides.json', '/' + name]).build(SDKConfig('SDK_KEY')) + assert 'pyyaml' in str(excinfo.value) + assert name in str(excinfo.value) + + +def test_builder_accepts_json_files_without_pyyaml(monkeypatch): + import ldclient.integrations.overrides as module + monkeypatch.setattr(module, 'have_yaml', False) + source = FileOverrideSourceBuilder(['/overrides.json', '/overrides']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + + +# --------------------------------------------------------------------------- +# Loading +# --------------------------------------------------------------------------- + +def test_source_loads_initial_data_synchronously(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true, "flag2": "x"}, "segments": {"seg": {"key": "seg"}}}') + source, sink = build_source(sources, [path]) + # The snapshot was delivered before start returned. + snapshot = sink.snapshots.get_nowait() + assert flag_values(snapshot) == {'flag1': True, 'flag2': 'x'} + assert list(snapshot[1].keys()) == ['seg'] + assert snapshot[0]['flag1'].is_override is False, "the source supplies unmarked definitions; the SDK marks them" + + +@yaml_required +def test_source_loads_yaml(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.yaml') + write_file(path, 'flagValues:\n yaml-flag: "override-value"\n') + _, sink = build_source(sources, [path]) + assert flag_values(sink.require_snapshot()) == {'yaml-flag': 'override-value'} + + +def test_source_merges_files_in_configured_order(tmp_path, sources): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"shared": "first", "only-first": 1}}') + write_file(second, '{"flagValues": {"shared": "second", "only-second": 2}}') + _, sink = build_source(sources, [first, second], lambda b: b.duplicate_keys_handling(DuplicateKeysHandling.IGNORE)) + assert flag_values(sink.require_snapshot()) == {'shared': 'first', 'only-first': 1, 'only-second': 2} + + +def test_source_duplicate_keys_fail_by_default(tmp_path, sources, caplog): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"shared": "first"}}') + write_file(second, '{"flagValues": {"shared": "second"}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [first, second]) + # The load failed, so no snapshot was supplied and the failure was logged. + sink.require_no_snapshot() + assert any("is specified by multiple files" in r.getMessage() for r in caplog.records) + + +def test_source_starts_with_missing_file(tmp_path, sources): + path = os.path.join(str(tmp_path), 'not-yet.json') + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + # A missing file contributes no overrides. The initial snapshot is empty. + assert sink.require_snapshot() == ({}, {}) + # Once the file appears, the change detection picks it up. + write_file(path, '{"flagValues": {"flag1": true}}') + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + +def test_source_missing_file_contributes_no_entries(tmp_path, sources): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"from-first": true}}') + + # Step 1: one configured file exists and one does not. The existing file applies. + _, sink = build_source(sources, [first, second], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'from-first': True} + + # Step 2: the second file appears. Both apply. + write_file(second, '{"flagValues": {"from-second": true}}') + assert flag_values(sink.require_snapshot()) == {'from-first': True, 'from-second': True} + + # Step 3: the second file is deleted. Its overrides are removed. + os.remove(second) + assert flag_values(sink.require_snapshot()) == {'from-first': True} + + # Step 4: the last file is deleted. The layer is cleared. + os.remove(first) + assert sink.require_snapshot() == ({}, {}) + + +def test_source_logs_overrides_in_effect_on_each_change(tmp_path, sources, caplog): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"flag1": true, "flag2": false}, "segments": {"seg": {"key": "seg"}}}') + + with caplog.at_level(logging.INFO): + # Step 1: at startup, one file supplies entries and the other is absent. + _, sink = build_source(sources, [first, second], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides in effect: 2 flags, 1 segment (%s: 2 flags, 1 segment; %s: absent)" % (first, second)) + + # Step 2: the absent file appears with one entry. + write_file(second, '{"flagValues": {"flag3": true}}') + sink.require_snapshot() + require_info_line(caplog, "Flag overrides in effect: 3 flags, 1 segment (%s: 2 flags, 1 segment; %s: 1 flag)" % (first, second)) + + # Step 3: both files are deleted. Nothing is in effect. + os.remove(first) + os.remove(second) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides: none in effect (%s: absent; %s: absent)" % (first, second)) + + +def test_source_logs_none_in_effect_at_startup_without_files(tmp_path, sources, caplog): + path = os.path.join(str(tmp_path), 'overrides.json') + with caplog.at_level(logging.INFO): + _, sink = build_source(sources, [path]) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides: none in effect (%s: absent)" % path) + + +def test_source_logs_a_present_file_with_no_entries(tmp_path, sources, caplog): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + with caplog.at_level(logging.INFO): + _, sink = build_source(sources, [path]) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides: none in effect (%s: no entries)" % path) + + +def wait_for_log(caplog, text: str) -> None: + deadline = time.time() + TEST_TIMEOUT + while not any(text in r.getMessage() for r in caplog.records): + assert time.time() < deadline, "timed out waiting for a log line containing %r" % text + time.sleep(0.01) + + +def test_source_keeps_the_overrides_when_a_json_file_is_emptied(tmp_path, sources, caplog): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + # An empty JSON file is not a document. The overrides stay in effect and, once the file + # has been read again, the failure is logged. + truncate(path) + wait_for_log(caplog, 'no content') + sink.require_no_snapshot() + + # A document with no entries removes the overrides. + write_file(path, '{}') + assert sink.require_snapshot() == ({}, {}) + + +@yaml_required +@watchdog_required +def test_source_removes_the_overrides_of_an_emptied_yaml_file_after_the_retry_delay(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.yaml') + write_file(path, 'flagValues:\n flag1: true\n') + _, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.WATCHING)) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + # An empty YAML file is a document with no entries. The watch notices the truncation at + # once, and the file is read again after the retry delay before it is applied, in case it + # is in the middle of an in-place rewrite. + emptied = time.monotonic() + truncate(path) + assert sink.require_snapshot() == ({}, {}) + assert time.monotonic() - emptied >= filedata.DEFAULT_RETRY_DELAY + + +# --------------------------------------------------------------------------- +# Change detection +# --------------------------------------------------------------------------- + +@watchdog_required +def test_watching_mode_is_quiet_when_file_is_absent(tmp_path, sources): + path = os.path.join(str(tmp_path), 'not-yet.json') + _, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.WATCHING)) + assert sink.require_snapshot() == ({}, {}) + sink.require_no_snapshot() + write_file(path, '{"flagValues": {"flag1": true}}') + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + +@watchdog_required +def test_watching_mode_reloads_on_change(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.WATCHING)) + assert isinstance(source._change_detector, filedata.Watcher) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + write_file(path, '{"flagValues": {"flag1": false}}') + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +def test_polling_mode_reloads_on_change(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.POLLING).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert isinstance(source._change_detector, filedata.Poller) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + write_file(path, '{"flagValues": {"flag1": false}}') + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +@pytest.mark.parametrize("mode", [ChangeDetection.POLLING, pytest.param(ChangeDetection.WATCHING, marks=watchdog_required)]) +def test_source_retains_last_good_data_across_malformed_edit(tmp_path, sources, caplog, mode): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + + # A malformed edit produces no snapshot: the previously applied overrides stay in + # effect because the sink is never called. The failure is logged. + write_file(path, '{"flagValues"') + sink.require_no_snapshot(1.5) + deadline = time.time() + TEST_TIMEOUT + while not any('Unable to load flag data' in r.getMessage() for r in caplog.records): + assert time.time() < deadline, "the load failure was not logged" + time.sleep(0.01) + + # Fixing the file recovers, through the change notification or the failure retry. + write_file(path, '{"flagValues": {"flag1": false}}') + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +def test_source_retries_a_failed_load_without_a_change_signal(tmp_path, sources, caplog): + # The fixed content has the same size as the malformed content and the file keeps its + # modification time, so the poller sees no change. Only the automatic retry can recover. + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + + malformed = '{"flagValues": {"flag1": false}' + fixed = '{"flagValues": {"flag1":false}}' + assert len(malformed) == len(fixed) + write_file(path, malformed) + deadline = time.time() + TEST_TIMEOUT + while not any('Unable to load flag data' in r.getMessage() for r in caplog.records): + assert time.time() < deadline, "the load failure was not logged" + time.sleep(0.01) + observed = os.stat(path) + write_file(path, fixed) + os.utime(path, ns=(observed.st_atime_ns, observed.st_mtime_ns)) + assert os.stat(path).st_size == observed.st_size + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +@pytest.mark.parametrize("mode", [ChangeDetection.POLLING, pytest.param(ChangeDetection.WATCHING, marks=watchdog_required)]) +def test_source_detects_an_edit_made_during_the_initial_load(tmp_path, sources, monkeypatch, mode): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": "initial"}}') + original_reload_now = filedata.Reloader.reload_now + + def load_then_edit(reloader: filedata.Reloader) -> None: + # The edit lands after the files were read and before start returns. + original_reload_now(reloader) + write_file(path, '{"flagValues": {"flag1": "edited-after-the-first-read"}}') + + monkeypatch.setattr(filedata.Reloader, 'reload_now', load_then_edit) + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + + # The initial snapshot holds what the first read saw. The edit is detected and applied + # without any further change to the file. + assert flag_values(sink.require_snapshot()) == {'flag1': 'initial'} + assert flag_values(sink.require_snapshot()) == {'flag1': 'edited-after-the-first-read'} + + +# --------------------------------------------------------------------------- +# In-place rewrites +# --------------------------------------------------------------------------- + +IN_PLACE_REWRITE_CASES = [ + pytest.param('json', ChangeDetection.POLLING, id='json-polling'), + pytest.param('yaml', ChangeDetection.POLLING, marks=yaml_required, id='yaml-polling'), + pytest.param('json', ChangeDetection.WATCHING, marks=watchdog_required, id='json-watching'), + pytest.param('yaml', ChangeDetection.WATCHING, marks=[watchdog_required, yaml_required], id='yaml-watching'), +] + + +def override_document(file_format: str, value: str) -> str: + if file_format == 'json': + return '{"flagValues": {"flag1": "%s"}}' % value + return 'flagValues:\n flag1: "%s"\n' % value + + +def configure_rewrite_timing(monkeypatch, mode: ChangeDetection) -> float: + """ + Returns the pause of a slow in-place rewrite and sets a retry delay longer than it, so that + the rewrite completes before the retry, even on a slow machine. A watch notices the + truncation at once. A poll notices it at the next examination, so in polling mode the pause + is longer than the poll interval, which makes a poll find the file empty. + """ + if mode == ChangeDetection.WATCHING: + pause, retry_delay = 0.3, 1.5 + else: + pause, retry_delay = FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL + 0.6, 3.0 + monkeypatch.setattr(file_override_source, 'DEFAULT_RETRY_DELAY', retry_delay) + return pause + + +@pytest.mark.parametrize('file_format, mode', IN_PLACE_REWRITE_CASES) +def test_source_never_supplies_a_snapshot_without_the_override_during_an_in_place_rewrite(tmp_path, sources, monkeypatch, file_format, mode): + pause = configure_rewrite_timing(monkeypatch, mode) + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, override_document(file_format, 'a')) + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': 'a'} + + # The file is empty for the whole pause, and a load finds it empty. The next snapshot holds + # the rewritten override, and the retry that the empty file armed supplies nothing more. + rewrite_in_place(path, override_document(file_format, 'b'), pause) + assert flag_values(sink.require_snapshot()) == {'flag1': 'b'} + sink.require_no_snapshot(file_override_source.DEFAULT_RETRY_DELAY) + + +@pytest.mark.parametrize('file_format, mode', IN_PLACE_REWRITE_CASES) +def test_source_never_supplies_a_snapshot_without_the_override_during_successive_in_place_rewrites(tmp_path, sources, file_format, mode): + # The source's own timing: the default retry delay and a one-second poll interval. An + # operator rewrites the file several times in a row, in place. A watch sees each write and + # the next truncation together; a poll can see a truncation without the content written + # before it. No snapshot may lack the override. + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, override_document(file_format, 'a')) + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': 'a'} + + values = ['b', 'c', 'd', 'e'] + for value in values: + rewrite_in_place(path, override_document(file_format, value), 0.3) + if mode == ChangeDetection.POLLING: + # The rewrites start about a second apart. + time.sleep(0.7) + + supplied: List[Dict[str, Any]] = [] + while len(supplied) == 0 or supplied[-1] != {'flag1': values[-1]}: + supplied.append(flag_values(sink.require_snapshot())) + assert all(list(snapshot.keys()) == ['flag1'] for snapshot in supplied), supplied + sink.require_no_snapshot(filedata.DEFAULT_RETRY_DELAY + 0.5) + + +@pytest.mark.parametrize('file_format, mode', IN_PLACE_REWRITE_CASES) +def test_client_serves_the_override_throughout_an_in_place_rewrite(tmp_path, monkeypatch, file_format, mode): + # An operator changes an override by rewriting its file in place. A client that evaluates the + # flag throughout serves the old value and then the new one, never the default. + pause = configure_rewrite_timing(monkeypatch, mode) + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, override_document(file_format, 'a')) + source = FileOverrideSourceBuilder([path]).change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + user = Context.create('user-key') + + with LDClient(config, start_wait=0) as client: + served: List[Any] = [] + stop = threading.Event() + + def sample() -> None: + while not stop.is_set(): + value = client.variation('flag1', user, 'default') + if len(served) == 0 or served[-1] != value: + served.append(value) + time.sleep(0.001) + + sampler = threading.Thread(target=sample, daemon=True) + sampler.start() + try: + rewrite_in_place(path, override_document(file_format, 'b'), pause) + deadline = time.time() + TEST_TIMEOUT + while served[-1] != 'b': + assert time.time() < deadline, "the rewritten override was not served; served %r" % served + time.sleep(0.01) + time.sleep(0.5) + finally: + stop.set() + sampler.join(TEST_TIMEOUT) + assert served == ['a', 'b'] + + +def reloader_threads() -> Set[threading.Thread]: + return {t for t in threading.enumerate() if t.name == 'ldclient.filedata.reloader'} + + +def test_source_close_stops_the_reloader_when_the_change_detector_fails_to_close(tmp_path, sources, monkeypatch): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + before = reloader_threads() + source, sink = build_source(sources, [path]) + sink.require_snapshot() + started = reloader_threads() - before + assert len(started) == 1 + + detector: Any = source._change_detector + original_close = detector.close + + def failing_close() -> None: + original_close() + raise RuntimeError("detector close failure") + + monkeypatch.setattr(detector, 'close', failing_close) + + # The failure propagates, and the reloader is stopped all the same: its worker ends. + with pytest.raises(RuntimeError): + source.close() + worker = started.pop() + worker.join(TEST_TIMEOUT) + assert not worker.is_alive() + + +def test_source_does_not_supply_snapshots_after_close(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + source.close() + write_file(path, '{"flagValues": {"flag1": false}}') + sink.require_no_snapshot(2.5) + + +def test_source_close_is_idempotent(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + source, _ = build_source(sources, [path]) + source.close() + source.close() + + +def test_source_start_after_close_does_nothing(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source = FileOverrideSourceBuilder([path]).build(SDKConfig('SDK_KEY')) + source.close() + sink = CapturingSink() + source.start(sink) + sink.require_no_snapshot(0.2) + + +# --------------------------------------------------------------------------- +# Through the client +# --------------------------------------------------------------------------- + +def test_file_overrides_end_to_end(tmp_path): + # An operator writes, edits, and empties an override file. Evaluations follow without any + # client restart, even though the client never obtains data from LaunchDarkly. + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + user = Context.create('user-key') + source = FileOverrideSourceBuilder([path]).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + + def eventually(predicate, message: str) -> None: + deadline = time.time() + TEST_TIMEOUT + while not predicate(): + assert time.time() < deadline, message + time.sleep(0.05) + + with LDClient(config, start_wait=0) as client: + # Not initialized and no override present: the default is served. + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is False + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + + # An operator adds an override. The running client picks it up. + write_file(path, '{"flagValues": {"overridden-flag": true}}') + eventually(lambda: client.variation('overridden-flag', user, False) is True, "the override was not picked up") + + # The override changes value. + write_file(path, '{"flagValues": {"overridden-flag": false}}') + + def changed() -> bool: + detail = client.variation_detail('overridden-flag', user, True) + return detail.value is False and detail.reason.get('overrideAffected') is True + + eventually(changed, "the changed override was not picked up") + + # The override is removed. The not-initialized short-circuit returns. + write_file(path, '{}') + eventually(lambda: client.variation_detail('overridden-flag', user, False).reason.get('errorKind') == 'CLIENT_NOT_READY', "the removed override was not picked up") + + +def test_client_side_all_flags_state_includes_a_full_definition_with_client_side_but_not_a_value_entry(tmp_path): + # A flagValues entry is not available to client-side SDKs. A full definition that sets + # clientSide is, so that is how an operator overrides a flag that client-side SDKs use. + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"value-flag": true}, ' + '"flags": {"full-flag": {"key": "full-flag", "version": 1, "on": true, "variations": [true], ' + '"fallthrough": {"variation": 0}, "clientSide": true}}}') + source = FileOverrideSourceBuilder([path]) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + + with LDClient(config, start_wait=0) as client: + user = Context.create('user-key') + assert set(client.all_flags_state(user).to_values_map().keys()) == {'value-flag', 'full-flag'} + state = client.all_flags_state(user, client_side_only=True) + assert state.valid + assert state.to_values_map() == {'full-flag': True} diff --git a/ldclient/testing/integrations/test_filedata.py b/ldclient/testing/integrations/test_filedata.py index 7f9875d5..4e5454cb 100644 --- a/ldclient/testing/integrations/test_filedata.py +++ b/ldclient/testing/integrations/test_filedata.py @@ -3,7 +3,7 @@ import threading import time from queue import Empty, Queue -from typing import Any, Dict, List, Set, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple from unittest import mock import pytest @@ -27,6 +27,7 @@ abs_file_paths, have_watchdog, have_yaml, + is_yaml_path, load_files, make_flag_with_value, merge, @@ -34,6 +35,7 @@ read_file ) from ldclient.impl.model import FeatureFlag, Segment +from ldclient.testing.file_util import rewrite_in_place, truncate TEST_TIMEOUT = 5.0 @@ -42,6 +44,9 @@ SHORT_DELAY = 0.05 QUIET_PERIOD = 0.3 +# The pause of the fixture's in-place rewrites, between truncating the file and writing it. +REWRITE_PAUSE = 0.03 + def write_file(path: str, content: str) -> None: with open(path, 'w') as f: @@ -78,30 +83,67 @@ def test_parse_json_document(): assert document.segments['seg1'].included == {'user1'} -def test_parse_chooses_the_parser_the_way_the_file_data_source_does(): - # The parser depends only on whether pyyaml is installed, never on the content. A tab inside - # a string is invalid JSON but valid YAML, so with pyyaml this document parses, and a parser - # chosen from the leading brace would reject it. - if not have_yaml: - pytest.skip("pyyaml is not installed") - document = parse_document(b' \n {"flagValues": {"flag1": "a\tb"}}') - assert document.flag_values == {'flag1': 'a\tb'} +@pytest.mark.parametrize('path', ['/data.yaml', '/data.yml', '/data.YAML', '/data.Yml', 'relative/data.yaml', '/x/.yaml', '/x/.YML']) +def test_a_yaml_extension_selects_yaml(path): + assert is_yaml_path(path) + + +@pytest.mark.parametrize('path', ['/data.json', '/data', '/data.txt', '/data.yaml.json', '/yaml', '/data.yaml/file', '/data.yamlx']) +def test_any_other_file_is_json(path): + assert not is_yaml_path(path) +def test_parse_chooses_the_parser_by_the_extension_not_the_content(): + # A tab inside a string is valid YAML but not valid JSON. Installing pyyaml does not make a + # JSON file YAML: the extension decides, whatever the content looks like. + content = b' \n {"flagValues": {"flag1": "a\tb"}}' + with pytest.raises(ValueError): + parse_document(content) + with pytest.raises(ValueError): + parse_document(b'flagValues:\n flag1: true\n') + if have_yaml: + assert parse_document(content, is_yaml=True).flag_values == {'flag1': 'a\tb'} + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") def test_parse_yaml_document(): - if not have_yaml: - pytest.skip("pyyaml is not installed") - document = parse_document(b'---\nflags:\n flag1:\n key: flag1\n "on": true\nflagValues:\n flag2: value2\nsegments:\n seg1:\n key: seg1\n') + document = parse_document(b'---\nflags:\n flag1:\n key: flag1\n "on": true\nflagValues:\n flag2: value2\nsegments:\n seg1:\n key: seg1\n', is_yaml=True) assert document.flags['flag1'].on is True assert document.flag_values == {'flag2': 'value2'} assert list(document.segments.keys()) == ['seg1'] -def test_parse_empty_document_has_no_entries(): - document = parse_document(b'') - assert document == Document() - document = parse_document(b'{}') - assert document == Document() +def test_parse_yaml_without_pyyaml_fails(monkeypatch): + monkeypatch.setattr(filedata, 'have_yaml', False) + with pytest.raises(ValueError) as excinfo: + parse_document(b'flagValues:\n flag1: true\n', is_yaml=True) + assert 'pyyaml' in str(excinfo.value) + + +@pytest.mark.parametrize('content', [b'', b' \n\t\r\n', b'\xef\xbb\xbf', b'\xef\xbb\xbf \n'], ids=['empty', 'white-space', 'byte-order-mark', 'byte-order-mark-and-white-space']) +def test_parse_empty_json_is_not_a_document(content): + # A file that is rewritten in place is empty for a moment. As JSON it is invalid, so the load + # fails and the last good data stays. + with pytest.raises(ValueError) as excinfo: + parse_document(content) + assert str(excinfo.value) == 'the file has no content, which is not a JSON document' + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +@pytest.mark.parametrize('content', [b'', b' \n\n', b'---\n', b'# no entries yet\n', b'\xef\xbb\xbf'], ids=['empty', 'white-space', 'document-start', 'comment', 'byte-order-mark']) +def test_parse_empty_yaml_has_no_entries(content): + assert parse_document(content, is_yaml=True) == Document() + + +def test_parse_json_object_with_no_members_has_no_entries(): + assert parse_document(b'{}') == Document() + assert parse_document(b' {} \n') == Document() + + +def test_parse_ignores_a_byte_order_mark(): + assert parse_document(b'\xef\xbb\xbf{"flagValues": {"flag1": true}}').flag_values == {'flag1': True} + if have_yaml: + assert parse_document(b'\xef\xbb\xbfflagValues:\n flag1: true\n', is_yaml=True).flag_values == {'flag1': True} def test_parse_fills_in_missing_key_and_version(): @@ -112,10 +154,25 @@ def test_parse_fills_in_missing_key_and_version(): assert document.segments['seg1'].version == 1 +@pytest.mark.parametrize('content', [b'null', b' null\n', b'["a", "b"]', b'1', b'"text"', b'true']) +def test_parse_rejects_a_json_document_that_is_not_an_object(content): + with pytest.raises(ValueError) as excinfo: + parse_document(content) + assert str(excinfo.value) == 'the document must be an object' + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_parse_reads_yaml_null_as_no_entries(): + assert parse_document(b'null\n', is_yaml=True) == Document() + assert parse_document(b'~\n', is_yaml=True) == Document() + + def test_parse_rejects_documents_that_are_not_objects(): if have_yaml: with pytest.raises(ValueError): - parse_document(b'- a\n- b\n') + parse_document(b'- a\n- b\n', is_yaml=True) + with pytest.raises(ValueError): + parse_document(b'["a", "b"]') with pytest.raises(ValueError): parse_document(b'{"flags": ["not", "an", "object"]}') with pytest.raises(ValueError): @@ -127,12 +184,16 @@ def test_parse_rejects_documents_that_are_not_objects(): def test_parse_rejects_malformed_json(): - # With pyyaml the YAML parser reports the error, as in the file data sources. - expected = (ValueError, yaml.YAMLError) if have_yaml else (ValueError,) - with pytest.raises(expected): + with pytest.raises(ValueError): parse_document(b'{"flagValues"') +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_parse_rejects_malformed_yaml(): + with pytest.raises(yaml.YAMLError): + parse_document(b'flagValues: [', is_yaml=True) + + def test_parse_validates_definition_property_types(): with pytest.raises(ValueError): parse_document(b'{"flags": {"flag1": {"key": "flag1", "version": "not a number"}}}') @@ -140,6 +201,32 @@ def test_parse_validates_definition_property_types(): parse_document(b'{"segments": {"seg1": {"key": "seg1", "version": 1, "included": "not a list"}}}') +@pytest.mark.parametrize('content', [ + '{"flagValues": {"flag1": true}}'.encode('utf-16'), + '{"flagValues": {"flag1": true}}'.encode('utf-32'), + b'{"flagValues": {"flag1": "\xff"}}', +], ids=['utf-16', 'utf-32', 'invalid-utf-8']) +@pytest.mark.parametrize('is_yaml', [False, pytest.param(True, marks=pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed"))], ids=['json', 'yaml']) +def test_parse_rejects_content_that_is_not_utf_8(content, is_yaml): + with pytest.raises(ValueError): + parse_document(content, is_yaml=is_yaml) + + +def test_reloader_reports_a_file_that_is_not_utf_8_and_keeps_working(make_fixture): + # A file in another encoding fails to load like any other content that cannot be parsed, and + # the reloader keeps loading the files when they change. + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader.reload_now() + f.require_applied() + with open(f.path, 'wb') as out: + out.write('{"flagValues": {"flag1": false}}'.encode('utf-16')) + f.reloader.trigger() + assert isinstance(f.require_errored(), FileReadError) + write_file(f.path, '{"flagValues": {"flag1": false}}') + f.reloader.trigger() + assert f.require_applied().flags['flag1'].variations == [False] + + def test_read_file_json_and_errors(tmp_path): path = os.path.join(str(tmp_path), 'data.json') write_file(path, '{"flagValues": {"flag1": true}}') @@ -156,6 +243,20 @@ def test_read_file_json_and_errors(tmp_path): assert 'error parsing file' in str(excinfo.value) assert path in str(excinfo.value) + write_file(path, '') + with pytest.raises(FileReadError) as excinfo: + read_file(path) + assert 'no content' in str(excinfo.value) + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_read_file_yaml(tmp_path): + path = os.path.join(str(tmp_path), 'data.yaml') + write_file(path, 'flagValues:\n flag1: true\n') + assert read_file(path).flag_values == {'flag1': True} + write_file(path, '') + assert read_file(path) == Document() + def test_abs_file_paths(): paths = abs_file_paths(['relative/data.json', '/absolute/data.json']) @@ -269,6 +370,39 @@ def test_load_files_reports_parse_error_with_path(tmp_path): assert excinfo.value.path == path +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_load_files_parses_each_file_in_the_format_of_its_extension(tmp_path): + json_path = os.path.join(str(tmp_path), 'first.json') + yaml_path = os.path.join(str(tmp_path), 'second.YML') + write_file(json_path, '{"flagValues": {"flag1": "json"}}') + write_file(yaml_path, 'flagValues:\n flag2: yaml\n') + result = load_files([json_path, yaml_path], DuplicateKeysHandling.FAIL) + assert {key: flag.variations for key, flag in result.flags.items()} == {'flag1': ['json'], 'flag2': ['yaml']} + + # The same YAML content in a file without a YAML extension is not JSON. + write_file(json_path, 'flagValues:\n flag1: json\n') + with pytest.raises(FileReadError) as excinfo: + load_files([json_path, yaml_path], DuplicateKeysHandling.FAIL) + assert excinfo.value.path == json_path + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_load_files_reads_an_empty_yaml_file_as_no_entries_and_fails_on_an_empty_json_file(tmp_path): + json_path = os.path.join(str(tmp_path), 'data.json') + yaml_path = os.path.join(str(tmp_path), 'data.yaml') + write_file(json_path, '{"flagValues": {"flag1": true}}') + write_file(yaml_path, '') + result = load_files([json_path, yaml_path], DuplicateKeysHandling.FAIL) + assert list(result.flags.keys()) == ['flag1'] + assert result.files[1] == FileSummary(path=yaml_path, present=True) + + write_file(json_path, '') + with pytest.raises(FileReadError) as excinfo: + load_files([json_path, yaml_path], DuplicateKeysHandling.FAIL) + assert excinfo.value.path == json_path + assert 'no content' in str(excinfo.value) + + def test_load_files_reports_duplicate_keys(tmp_path): first = os.path.join(str(tmp_path), 'first.json') second = os.path.join(str(tmp_path), 'second.json') @@ -287,8 +421,8 @@ def test_load_files_reports_duplicate_keys(tmp_path): # --------------------------------------------------------------------------- class ReloaderFixture: - def __init__(self, tmp_path, initial_content: str, **kwargs): - self.path = os.path.join(str(tmp_path), 'data.json') + def __init__(self, tmp_path, initial_content: str, name: str = 'data.json', **kwargs): + self.path = os.path.join(str(tmp_path), name) self.applied: Queue = Queue() self.errored: Queue = Queue() write_file(self.path, initial_content) @@ -302,13 +436,16 @@ def __init__(self, tmp_path, initial_content: str, **kwargs): self.reloader = Reloader(**options) def write(self, content: str) -> None: - write_file(self.path, content) + # Applications rewrite data files in place, which is not atomic. The fixture does too, + # deliberately slowly: the file is truncated and written a little later, so a load can + # find it empty, as it would with an application's writes, instead of only by chance. + rewrite_in_place(self.path, content, REWRITE_PAUSE) - def require_applied(self) -> MergeResult: - return take(self.applied) + def require_applied(self, timeout: float = TEST_TIMEOUT) -> MergeResult: + return take(self.applied, timeout) - def require_errored(self) -> Exception: - return take(self.errored) + def require_errored(self, timeout: float = TEST_TIMEOUT) -> Exception: + return take(self.errored, timeout) def require_quiet(self, duration: float = QUIET_PERIOD) -> None: deadline = time.time() + duration @@ -450,16 +587,17 @@ def test_reloader_reports_identical_failure_only_once(make_fixture): # The automatic retries keep failing in the same way. They do not report again. f.require_quiet() - # A different failure is reported. The write is ordinary, not atomic, and a retry that runs - # while the file is being rewritten reads it empty or half written. An empty document is - # valid, so such a retry succeeds and ends the retries; a half-written file is a failure of - # its own. The final content is therefore loaded on demand, and everything that was reported - # is inspected as a whole: the new failure is among the reports, and once it is the current - # failure, the retries that repeat it stay silent. + # A different failure is reported. The write is in place, so a retry can run while the file + # is empty. That retry reads the file again instead of reporting it, and a retry that finds + # it still empty reports that it has no content. The final content is loaded on demand, and + # every report is checked: each one is the new failure or the empty file, the new failure is + # among them, and once it is the current failure, the retries that repeat it stay silent. f.write('{"flags": {"flag1": "not an object"}}') f.reloader.reload_now() - _, errored = f.drain() + applied, errored = f.drain() + assert applied == [] assert any('must be an object' in str(e) for e in errored), errored + assert all('must be an object' in str(e) or 'no content' in str(e) for e in errored), errored f.require_quiet() @@ -537,7 +675,9 @@ def test_reloader_retries_after_failed_initial_load(make_fixture): def test_reloader_stops_retrying_after_success(make_fixture): - f = make_fixture('{"flagValues"', retry_delay=SHORT_DELAY, skip_unchanged=True) + # The retry delay is well over the pause of the fixture's in-place writes, so that a retry is + # unlikely to read the file in the middle of one on a slow machine. + f = make_fixture('{"flagValues"', retry_delay=0.2, skip_unchanged=True) f.reloader.reload_now() f.require_errored() @@ -695,6 +835,1308 @@ def on_error(err: Exception) -> None: f.require_applied() +# --------------------------------------------------------------------------- +# Reloader: files that are found empty +# --------------------------------------------------------------------------- + +yaml_required = pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") + +# The retry delay of the tests of a file that is found empty. The retry comes long enough after a +# trigger for a test to finish rewriting the file before it, even on a slow machine. +EMPTY_FILE_RETRY_DELAY = 1.5 + +# Shorter than EMPTY_FILE_RETRY_DELAY, so a load that happens within it after a trigger is not +# the retry. +BEFORE_THE_RETRY = 1.0 + +DEFERRAL_LOG = 'waiting for it to be written' + +YAML_FLAG1 = 'flagValues:\n flag1: true\n' +YAML_FLAG2 = 'flagValues:\n flag2: true\n' + + +def flag_keys(result: MergeResult) -> List[str]: + return list(result.flags.keys()) + + +def deferrals_logged(caplog) -> int: + return len([r for r in caplog.records if r.levelno == logging.DEBUG and DEFERRAL_LOG in r.getMessage()]) + + +def wait_for_deferrals(caplog, count: int) -> None: + deadline = time.monotonic() + TEST_TIMEOUT + while deferrals_logged(caplog) < count: + assert time.monotonic() < deadline, "the empty file was not held back" + time.sleep(0.01) + + +def change_modification_time(path: str) -> None: + """ + Makes sure that the file's modification time differs from what it was, as a rewrite changes + it, even on a file system whose timestamps are too coarse to tell two quick writes apart. + """ + modified = os.stat(path).st_mtime_ns + 1000000 + os.utime(path, ns=(modified, modified)) + + +class FakeClock: + """A monotonic clock that only moves when a test advances it.""" + + def __init__(self): + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +# The retry delay of the tests that time a hold on a fake clock. The timers use the real clock, +# so no retry fires during such a test, and each load is started with reload_now. +HOUR = 3600.0 + + +def make_held_fixture(make_fixture, content: str, name: str = 'data.yaml', **kwargs) -> Tuple[ReloaderFixture, FakeClock]: + clock = FakeClock() + f = make_fixture(content, name=name, retry_delay=HOUR, clock=clock, **kwargs) + f.reloader.reload_now() + f.require_applied() + return f, clock + + +def make_two_file_fixture(make_fixture, tmp_path, first: str, second: str, **kwargs) -> Tuple[ReloaderFixture, FakeClock, str]: + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.yaml') + write_file(other, second) + f = make_fixture(first, name='data.yaml') + f.reloader = Reloader([f.path, other], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock, **kwargs) + f.reloader.reload_now() + f.require_applied() + return f, clock, other + + +def load(f: ReloaderFixture) -> Optional[MergeResult]: + """Loads the files at once, and returns what was applied, or None when nothing was.""" + f.reloader.reload_now() + try: + return f.applied.get_nowait() + except Empty: + return None + + +def require_held_back(f: ReloaderFixture) -> None: + assert load(f) is None, "the load applied the file instead of holding it back" + assert f.errored.empty(), "the load reported %r" % f.errored.get_nowait() + + +@yaml_required +def test_reloader_holds_back_a_file_until_it_has_stayed_empty_for_exactly_the_retry_delay(make_fixture, caplog): + # A writer that rewrites the file in place truncates it and then writes the new content. A + # load that finds the file empty while its content is in effect keeps the data. In YAML an + # empty file is valid, so applying it would briefly remove every entry. + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + assert deferrals_logged(caplog) == 1 + + clock.advance(0.95 * HOUR) + require_held_back(f) + + clock.advance(0.05 * HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +@yaml_required +def test_reloader_restarts_the_delay_when_the_held_back_file_is_written(make_fixture): + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + + # Rewritten and truncated again between two loads: its state changed, so it is a new rewrite. + clock.advance(0.6 * HOUR) + write_file(f.path, YAML_FLAG2) + truncate(f.path) + change_modification_time(f.path) + require_held_back(f) + + clock.advance(0.6 * HOUR) + require_held_back(f) + + clock.advance(0.4 * HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +@yaml_required +def test_reloader_observes_a_held_back_file_through_a_symbolic_link(make_fixture, tmp_path): + # The configured path is a link to the file that is rewritten, as with a mounted ConfigMap. + # Writing the file changes its state as seen through the link, so the delay starts over. + target = os.path.join(str(tmp_path), 'target.yaml') + link = os.path.join(str(tmp_path), 'link.yaml') + write_file(target, YAML_FLAG1) + try: + os.symlink(target, link) + except (OSError, NotImplementedError): + pytest.skip("symbolic links are not available") + clock = FakeClock() + f = make_fixture(YAML_FLAG1, name='unused.yaml') + f.reloader = Reloader([link], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(target) + require_held_back(f) + clock.advance(0.6 * HOUR) + write_file(target, YAML_FLAG2) + truncate(target) + change_modification_time(target) + require_held_back(f) + clock.advance(0.6 * HOUR) + require_held_back(f) + + +@yaml_required +def test_reloader_forgets_an_emptying_once_the_file_is_read_with_content(make_fixture): + # A file that was held back, then applied with content, and emptied again much later starts a + # new delay, even if it ends up with the same modification time as at the first emptying, as + # when a tool restores the times of the files it writes. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + first = os.stat(f.path) + require_held_back(f) + write_file(f.path, YAML_FLAG2) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + clock.advance(10 * HOUR) + truncate(f.path) + os.utime(f.path, ns=(first.st_atime_ns, first.st_mtime_ns)) + require_held_back(f) + + +@yaml_required +def test_reloader_holds_back_a_file_emptied_again_after_a_read_with_content_whose_state_looks_the_same(make_fixture, tmp_path): + # On a file system that records modification times coarsely, a file emptied a second time can + # look exactly as it did the first time. The read of its content in between still starts its + # delay over. The other file is kept written while empty, so that no load applies. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + first_emptying = os.stat(f.path) + truncate(other) + require_held_back(f) + + clock.advance(50 / 60 * HOUR) + write_file(f.path, YAML_FLAG1) + change_modification_time(other) + require_held_back(f) + + clock.advance(20 / 60 * HOUR) + truncate(f.path) + os.utime(f.path, ns=(first_emptying.st_atime_ns, first_emptying.st_mtime_ns)) + change_modification_time(other) + require_held_back(f) + + # The other file gets its content back while this one is still within its new delay. + clock.advance(10 / 60 * HOUR) + write_file(other, YAML_FLAG2) + require_held_back(f) + + +@yaml_required +def test_reloader_holds_back_a_file_rewritten_with_the_same_content_again_later(make_fixture): + # A tool rewrites the file in place with the same content now and then. The load after each + # rewrite is skipped as unchanged, which confirms the data in effect like an application, so + # a later rewrite is held back on its own and not cut short by the first one's bound. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1, skip_unchanged=True) + truncate(f.path) + require_held_back(f) + write_file(f.path, YAML_FLAG1) + assert load(f) is None + + clock.advance(6 * HOUR) + truncate(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_holds_back_a_file_emptied_again_after_unparsable_content_whose_state_looks_the_same(make_fixture): + # The file is emptied and held back, then saved with content that cannot be parsed, then + # emptied again with the modification time of its first emptying, as a coarse file system can + # make it look. Reading content ended its empty state, so it starts a new delay. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + first_emptying = os.stat(f.path) + require_held_back(f) + write_file(f.path, 'flagValues: [') + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + + clock.advance(70 / 60 * HOUR) + truncate(f.path) + os.utime(f.path, ns=(first_emptying.st_atime_ns, first_emptying.st_mtime_ns)) + require_held_back(f) + + +@yaml_required +def test_reloader_applies_the_written_content_of_a_held_back_file_at_once(make_fixture): + # Two quick rewrites in place: the second truncation follows the first write before any load + # reads the new content, as when both fall in one debounce window. The file is empty at every + # load, and its emptiness is not applied; the content that is finally written is. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + write_file(f.path, YAML_FLAG2) + truncate(f.path) + change_modification_time(f.path) + clock.advance(0.2 * HOUR) + require_held_back(f) + + write_file(f.path, 'flagValues:\n flag3: true\n') + result = load(f) + assert result is not None and flag_keys(result) == ['flag3'] + + +@yaml_required +@pytest.mark.parametrize('content', [' \n\n', '# no entries yet\n', '---\n', 'null\n'], ids=['white-space', 'comment', 'document-marker', 'null']) +def test_reloader_holds_back_a_yaml_file_that_has_no_entries(make_fixture, content): + # A read can see only the start of a YAML file that is being written: white space, a comment + # or a document marker, which the parser reads as nothing. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + write_file(f.path, content) + require_held_back(f) + write_file(f.path, YAML_FLAG2) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_applies_a_yaml_object_with_no_members_at_once(make_fixture, caplog): + # {} is a document with no entries, written on purpose to remove them, not part of a file. + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + write_file(f.path, '{}\n') + result = load(f) + assert result is not None and result.flags == {} + assert deferrals_logged(caplog) == 0 + + +@yaml_required +def test_reloader_holds_back_files_emptied_together_for_one_retry_delay(make_fixture, tmp_path, caplog): + # Every file is read and classified before any is held back, so two files that are emptied + # together start their delays in the same load and are applied together. + caplog.set_level(logging.DEBUG) + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + truncate(other) + require_held_back(f) + assert deferrals_logged(caplog) == 2 + + clock.advance(HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +@yaml_required +def test_reloader_holds_back_an_emptied_file_while_another_file_changes(make_fixture, tmp_path): + # A change to one file starts a load while another is in the middle of a rewrite. The load + # applies all files or none, so the change waits too. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + + write_file(other, 'flagValues:\n flag2: false\n') + clock.advance(0.3 * HOUR) + require_held_back(f) + + write_file(f.path, YAML_FLAG1) + result = load(f) + assert result is not None and flag_keys(result) == ['flag1', 'flag2'] + assert result.flags['flag2'].variations == [False] + + +def test_reloader_fails_on_a_file_that_cannot_be_parsed_while_another_is_held_back(make_fixture, tmp_path): + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.json') + write_file(other, '{"flagValues": {"flag2": true}}') + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader = Reloader([f.path, other], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + # One file is mid-rewrite and the other is broken. The broken file is reported at once. + truncate(f.path) + write_file(other, '{"flagValues"') + assert load(f) is None + err = f.errored.get_nowait() + assert isinstance(err, FileReadError) + assert err.path == other + + +@yaml_required +def test_reloader_stops_holding_back_a_file_that_keeps_being_emptied_after_five_retry_delays(make_fixture): + # Every load finds the file empty with a new modification time, so its delay keeps starting + # over. The run of loads that hold it back ends after five retry delays. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + for _ in range(9): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + require_held_back(f) + + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + result = load(f) + assert result is not None and result.flags == {} + + +@yaml_required +def test_reloader_stops_holding_back_files_rewritten_in_turn_after_five_retry_delays(make_fixture, tmp_path): + # Two files are rewritten in turn, so every load finds one of them empty, and each is read + # with content in between. The run of loads that hold files back still ends after five retry + # delays. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + paths = [f.path, other] + contents = [YAML_FLAG1, YAML_FLAG2] + truncate(f.path) + require_held_back(f) + for i in range(1, 10): + clock.advance(0.5 * HOUR) + write_file(paths[(i - 1) % 2], contents[(i - 1) % 2]) + truncate(paths[i % 2]) + require_held_back(f) + + clock.advance(0.5 * HOUR) + write_file(paths[1], contents[1]) + truncate(paths[0]) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_measures_a_file_from_its_own_read_when_an_earlier_file_reads_slowly(make_fixture, tmp_path, monkeypatch): + # The read of the earlier file stalls while the later file is being rewritten in place. The + # later file's delay counts from when it was read, not from when the load began. + clock = FakeClock() + slow = os.path.join(str(tmp_path), 'slow.yaml') + write_file(slow, YAML_FLAG2) + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([slow, f.path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + original = filedata._read_file + stall = [True] + + def read_file(path: str, skip_missing_paths: bool): + read = original(path, skip_missing_paths) + if path == slow and stall[0]: + truncate(f.path) + clock.advance(0.9 * HOUR) + return read + + monkeypatch.setattr(filedata, '_read_file', read_file) + require_held_back(f) + stall[0] = False + clock.advance(0.2 * HOUR) + require_held_back(f) + + write_file(f.path, 'flagValues:\n flag3: true\n') + result = load(f) + assert result is not None and flag_keys(result) == ['flag2', 'flag3'] + + +@yaml_required +def test_reloader_measures_a_file_from_its_own_read_when_a_later_file_reads_slowly(make_fixture, tmp_path, monkeypatch): + # The file is found empty before the read of a later file stalls. Its delay counts from its + # own read, so it is over one retry delay after that read, and the retry the hold asks for + # comes then. + clock = FakeClock() + slow = os.path.join(str(tmp_path), 'slow.yaml') + write_file(slow, YAML_FLAG2) + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, slow], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + original = filedata._read_file + stall = [True] + + def read_file(path: str, skip_missing_paths: bool): + read = original(path, skip_missing_paths) + if path == slow and stall[0]: + clock.advance(0.9 * HOUR) + return read + + monkeypatch.setattr(filedata, '_read_file', read_file) + truncate(f.path) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.1 * HOUR + 0.1 * HOUR) + stall[0] = False + clock.advance(0.1 * HOUR) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_a_failing_other_file_does_not_restart_the_bound(make_fixture, tmp_path): + # The file keeps being written while every load finds it empty, and the other file fails to + # parse every other load. The failures do not start the file's bound over. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + for i in range(1, 10): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + write_file(other, 'flagValues: [' if i % 2 == 1 else YAML_FLAG2) + assert load(f) is None + + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + write_file(other, YAML_FLAG2) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_another_files_bound_does_not_cut_a_rewrite_short(make_fixture, tmp_path): + # One file keeps being written while every load finds it empty, until its bound is over. + # The other file is a quarter of a retry delay into an ordinary rewrite then, and stays held + # back until it is written. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + for _ in range(9): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + require_held_back(f) + + clock.advance(0.25 * HOUR) + change_modification_time(f.path) + truncate(other) + require_held_back(f) + + clock.advance(0.25 * HOUR) + change_modification_time(f.path) + require_held_back(f) + + write_file(other, 'flagValues:\n flag2: false\n') + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + assert result.flags['flag2'].variations == [False] + + +@yaml_required +def test_reloader_applies_files_emptied_in_staggered_turns_after_the_bound(make_fixture, tmp_path): + # Two files stay empty and are truncated again in turn, so that whenever one of them has + # stayed empty for the whole delay, the other is still within its delay and holds the load + # back. A file that settles so does not get a fresh bound while the other is held back. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + applied = False + for i in range(1, 15): + clock.advance(54 / 60 * HOUR) + if i == 1: + truncate(other) + change_modification_time(other if i % 2 == 1 else f.path) + applied = load(f) is not None + if applied: + break + # The retry that the hold arms, a little after the other file's delay is over. + clock.advance(12 / 60 * HOUR) + applied = load(f) is not None + clock.advance(-12 / 60 * HOUR) + if applied: + break + assert applied, "the files were held back without end" + + +@yaml_required +def test_reloader_bound_of_a_file_held_back_during_a_failure_starts_over_once_it_is_read_again(make_fixture, tmp_path): + # While the other file fails to load, this file is rewritten once and its gap is held back. + # Much later, the other file is fixed while this one is in the middle of another rewrite, + # which is held back with a bound of its own. + clock = FakeClock() + broken = os.path.join(str(tmp_path), 'other.json') + write_file(broken, '{"flagValues": {"flag3": false}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, broken], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + write_file(broken, '{bad') + truncate(f.path) + assert load(f) is None + write_file(f.path, YAML_FLAG2) + assert load(f) is None + clock.advance(6 * HOUR) + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + assert f.errored.empty(), "an identical failure was reported again" + + write_file(broken, '{"flagValues": {"flag3": false}}') + truncate(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_hold_asks_for_the_retry_from_the_end_of_the_load(make_fixture, tmp_path, monkeypatch): + # The read of the file after the held one takes longer than what is left of the held file's + # delay, so the retry is due right away, after the margin. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + + original = filedata._read_file + stall = [True] + + def read_file(path: str, skip_missing_paths: bool): + read = original(path, skip_missing_paths) + if path == other and stall[0]: + stall[0] = False + clock.advance(6 / 60 * HOUR) + return read + + monkeypatch.setattr(filedata, '_read_file', read_file) + clock.advance(57 / 60 * HOUR) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.1 * HOUR) + + +@pytest.mark.parametrize('name, good, broken, fixed', [ + pytest.param('data.yaml', YAML_FLAG1, 'flagValues: [', 'flagValues:\n flag1: false\n', marks=yaml_required, id='yaml'), + pytest.param('data.json', '{"flagValues": {"flag1": true}}', '{"flagValues"', '{"flagValues": {"flag1": false}}', id='json'), +]) +def test_reloader_holds_back_the_fix_of_a_file_saved_in_place_with_an_error(make_fixture, name, good, broken, fixed): + # An operator saves a typo in place, and the fix in place much later. The truncation of the + # second save is held back like the first, so the fix is applied and the file never empty. + f, clock = make_held_fixture(make_fixture, good, name=name) + truncate(f.path) + require_held_back(f) + write_file(f.path, broken) + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + for _ in range(6): + clock.advance(HOUR) + assert load(f) is None + assert f.errored.empty(), "an identical failure was reported again" + + truncate(f.path) + require_held_back(f) + write_file(f.path, fixed) + result = load(f) + assert result is not None and result.flags['flag1'].variations == [False] + + +@yaml_required +def test_reloader_file_kept_written_while_empty_is_applied_after_its_bound_while_an_earlier_file_fails(make_fixture, tmp_path): + # The file is kept written while empty. A file before it fails to parse on every other load. + # Those loads still read and hold back this file, so they do not start its bound over. + clock = FakeClock() + first = os.path.join(str(tmp_path), 'first.json') + write_file(first, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([first, f.path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + require_held_back(f) + for i in range(1, 10): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + write_file(first, '{"flagValues"' if i % 2 == 1 else '{"flagValues": {"flag3": true}}') + assert load(f) is None + + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + write_file(first, '{"flagValues": {"flag3": true}}') + result = load(f) + assert result is not None and flag_keys(result) == ['flag3'] + + +@yaml_required +def test_reloader_bound_of_a_file_after_a_failing_file_starts_over_once_it_is_read_again(make_fixture, tmp_path): + # The file is held back, then a file configured before it breaks while this one gets its + # content back. The failing loads still read this file, and they hold nothing back, so its + # bound starts over. Much later the other file is fixed while this one is rewritten in place, + # and the truncation is held back. + clock = FakeClock() + first = os.path.join(str(tmp_path), 'first.json') + write_file(first, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([first, f.path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + require_held_back(f) + write_file(first, '{"flagValues"') + write_file(f.path, YAML_FLAG2) + assert load(f) is None + for _ in range(6): + clock.advance(HOUR) + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + assert f.errored.empty() + + write_file(first, '{"flagValues": {"flag3": true}}') + truncate(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_reads_a_file_after_one_that_cannot_be_read(make_fixture, tmp_path): + # A file configured before this one goes missing, and missing files are not skipped, so it + # cannot be read. The failing loads still read this file, which got its content back, and + # they hold nothing back, so its bound starts over. + clock = FakeClock() + first = os.path.join(str(tmp_path), 'first.json') + write_file(first, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([first, f.path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + require_held_back(f) + os.remove(first) + write_file(f.path, YAML_FLAG2) + assert load(f) is None + assert f.errored.get_nowait().path == first + clock.advance(6 * HOUR) + assert load(f) is None + + write_file(first, '{"flagValues": {"flag3": true}}') + truncate(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_starts_the_bound_of_a_file_that_cannot_be_read_over(make_fixture, monkeypatch): + # The file is held back, then cannot be read for a long time, then is readable again and + # empty after a new write. The failing loads hold nothing back and tried to read the file, + # so its bound started over, and the new emptying is held back. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + + original = filedata._read_file + unreadable = [True] + + def read_file(path: str, skip_missing_paths: bool): + if path == f.path and unreadable[0]: + raise FileReadError(path, "unable to read file: permission denied") + return original(path, skip_missing_paths) + + monkeypatch.setattr(filedata, '_read_file', read_file) + for _ in range(6): + assert load(f) is None + clock.advance(HOUR) + assert isinstance(f.errored.get_nowait(), FileReadError) + + unreadable[0] = False + change_modification_time(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_keeps_the_delay_of_a_held_back_file_through_a_failed_read(make_fixture, monkeypatch): + # A read that fails says nothing about the content, so the file's delay goes on. Once it can + # be read again, still empty and unmodified, it has been empty for the whole delay. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + + original = filedata._read_file + unreadable = [True] + + def read_file(path: str, skip_missing_paths: bool): + if path == f.path and unreadable[0]: + raise FileReadError(path, "unable to read file: permission denied") + return original(path, skip_missing_paths) + + monkeypatch.setattr(filedata, '_read_file', read_file) + clock.advance(0.5 * HOUR) + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + + unreadable[0] = False + clock.advance(0.5 * HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +def test_reloader_holds_back_a_json_file_that_holds_only_a_byte_order_mark(make_fixture, caplog): + # A byte order mark is not content, so a JSON file in effect that holds only one, as a read + # can see of a file being written, is held back quietly. + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, '{"flagValues": {"flag1": true}}', name='data.json') + with open(f.path, 'wb') as out: + out.write(b'\xef\xbb\xbf') + require_held_back(f) + assert deferrals_logged(caplog) == 1 + + +def test_reloader_reports_a_json_file_that_is_not_an_object(make_fixture): + f, clock = make_held_fixture(make_fixture, '{"flagValues": {"flag1": true}}', name='data.json') + write_file(f.path, 'null') + assert load(f) is None + err = f.errored.get_nowait() + assert isinstance(err, FileReadError) + assert 'must be an object' in str(err) + assert 'as JSON, the format of its name' in str(err) + + +def test_reloader_reports_the_first_failure_when_an_empty_file_also_fails(make_fixture, tmp_path): + # The first file cannot be parsed, and the second is empty, which as JSON fails too. The load + # reports the first failure, and logs the other at Debug level. + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues"') + write_file(second, '') + f = make_fixture('{}') + f.reloader = Reloader([first, second], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put) + f.reloader.reload_now() + err = f.require_errored() + assert isinstance(err, FileReadError) + assert err.path == first + + +def test_reloader_logs_the_failures_after_the_first(make_fixture, tmp_path, caplog): + caplog.set_level(logging.DEBUG) + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues"') + write_file(second, '{"flags"') + f = make_fixture('{}') + f.reloader = Reloader([first, second], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put) + f.reloader.reload_now() + assert f.require_errored().path == first + assert any(r.levelno == logging.DEBUG and second in r.getMessage() for r in caplog.records) + + +def test_reloader_failing_load_reports_the_first_failure(make_fixture, tmp_path): + # Every file is read even after one fails, and the load reports the first failure. + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues"') + write_file(second, '{"flags"') + f = make_fixture('{}') + f.reloader = Reloader([first, second], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put) + f.reloader.reload_now() + err = f.require_errored() + assert isinstance(err, FileReadError) + assert err.path == first + + +@yaml_required +def test_reloader_applies_the_same_bytes_moved_to_a_file_of_another_format(make_fixture, tmp_path): + # The format comes from the extension, so the same bytes moved from x.yaml to x.json parse + # differently: YAML reads 1e3 as a string, JSON as a number. The load is not skipped as + # unchanged. + yaml_path = os.path.join(str(tmp_path), 'x.yaml') + json_path = os.path.join(str(tmp_path), 'x.json') + content = '{"flagValues": {"flag1": 1e3}}' + write_file(yaml_path, content) + f = make_fixture('{}') + f.reloader = Reloader([yaml_path, json_path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, skip_unchanged=True) + f.reloader.reload_now() + assert f.require_applied().flags['flag1'].variations == ['1e3'] + + os.replace(yaml_path, json_path) + result = load(f) + assert result is not None and result.flags['flag1'].variations == [1000.0] + + +@yaml_required +def test_reloader_load_that_holds_a_file_back_and_then_fails_does_not_start_bounds_over(make_fixture, tmp_path): + # Two files are rewritten in turn, and a third fails to parse on every other load, after one of + # the two has been held back. Those loads hold a file back, so they do not start the bounds + # over, and the files are applied in the end. + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.yaml') + last = os.path.join(str(tmp_path), 'last.json') + write_file(other, YAML_FLAG2) + write_file(last, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, other, last], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + require_held_back(f) + applied = False + for i in range(1, 13): + clock.advance(0.5 * HOUR) + if i % 2 == 1: + write_file(f.path, YAML_FLAG1) + truncate(other) + write_file(last, '{"flagValues"') + else: + write_file(other, YAML_FLAG2) + truncate(f.path) + write_file(last, '{"flagValues": {"flag3": true}}') + if load(f) is not None: + applied = True + break + assert applied, "the files were held back without end" + + +@yaml_required +def test_reloader_file_found_missing_starts_a_new_delay_when_it_is_created_empty_again(make_fixture, tmp_path): + # The file is emptied and held back, then deleted, then created empty again with the + # modification time of its first emptying, while another file fails to parse, so nothing is + # applied meanwhile. Being found missing ended its empty state, so it starts a new delay. + clock = FakeClock() + broken = os.path.join(str(tmp_path), 'other.json') + write_file(broken, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, broken], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + write_file(broken, '{bad') + truncate(f.path) + first_emptying = os.stat(f.path) + assert load(f) is None + os.remove(f.path) + assert load(f) is None + + clock.advance(70 / 60 * HOUR) + write_file(f.path, '') + os.utime(f.path, ns=(first_emptying.st_atime_ns, first_emptying.st_mtime_ns)) + write_file(broken, '{"flagValues": {"flag3": true}}') + while not f.errored.empty(): + f.errored.get_nowait() + require_held_back(f) + + +@yaml_required +def test_reloader_hold_asks_for_the_retry_after_the_delay_and_a_tenth(make_fixture): + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(1.1 * HOUR) + + +@yaml_required +def test_reloader_later_hold_asks_for_the_rest_of_the_delay(make_fixture): + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + clock.advance(0.4 * HOUR) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.6 * HOUR + 0.1 * HOUR) + + +@yaml_required +def test_reloader_hold_asks_for_the_retry_when_the_bound_is_over(make_fixture): + # The file was written again, so its delay would end one retry delay from now, but its bound + # ends sooner. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + for _ in range(9): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.5 * HOUR + 0.1 * HOUR) + + +@yaml_required +@pytest.mark.parametrize('due_first', ['first', 'last']) +def test_reloader_hold_asks_for_the_retry_when_the_first_held_file_is_due(make_fixture, tmp_path, due_first): + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + earlier, later = (f.path, other) if due_first == 'first' else (other, f.path) + truncate(earlier) + require_held_back(f) + clock.advance(0.5 * HOUR) + truncate(later) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.5 * HOUR + 0.1 * HOUR) + + +@yaml_required +def test_reloader_hold_after_a_change_signal_asks_for_the_same_retry(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + + def wait_for_requested_delay(expected: float) -> None: + deadline = time.monotonic() + TEST_TIMEOUT + while f.reloader._last_requested_retry_delay != pytest.approx(expected): + assert time.monotonic() < deadline, "asked for %r" % f.reloader._last_requested_retry_delay + time.sleep(0.01) + + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + wait_for_requested_delay(1.1 * HOUR) + clock.advance(0.4 * HOUR) + f.reloader.trigger() + wait_for_deferrals(caplog, 2) + wait_for_requested_delay(0.7 * HOUR) + + +def test_reloader_failure_asks_for_the_retry_delay(make_fixture): + f = make_fixture('{"flagValues"', retry_delay=HOUR, clock=FakeClock()) + f.reloader.reload_now() + f.require_errored() + assert f.reloader._last_requested_retry_delay == HOUR + + +@yaml_required +def test_reloader_holds_back_a_file_that_is_emptied_again_after_its_content_was_applied(make_fixture): + # A load that holds no file back ends the run, so a later rewrite is held back on its own. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + write_file(f.path, YAML_FLAG2) + clock.advance(0.5 * HOUR) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + clock.advance(10 * HOUR) + truncate(f.path) + require_held_back(f) + clock.advance(HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +def test_reloader_holds_back_a_json_file_that_was_empty_for_long_when_it_is_rewritten(make_fixture, caplog): + # A JSON file that stays empty fails to load once the delay is over, and keeps failing. The + # failures end the run of holds, so when the file is rewritten in place much later, the new + # truncation is held back like any other, without another report. + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, '{"flagValues": {"flag1": true}}', name='data.json') + truncate(f.path) + require_held_back(f) + + clock.advance(HOUR) + assert load(f) is None + assert 'no content' in str(f.errored.get_nowait()) + clock.advance(HOUR) + assert load(f) is None + assert f.errored.empty() + + clock.advance(10 * HOUR) + held_before = deferrals_logged(caplog) + truncate(f.path) + change_modification_time(f.path) + require_held_back(f) + assert deferrals_logged(caplog) == held_before + 1 + + write_file(f.path, '{"flagValues": {"flag1": false}}') + result = load(f) + assert result is not None and result.flags['flag1'].variations == [False] + + +@yaml_required +def test_reloader_does_not_hold_back_a_file_that_is_empty_from_the_start(make_fixture, tmp_path, caplog): + # An empty YAML file next to the data, such as a placeholder for later entries, has never had + # content. It does not hold back the changes of the other file. + caplog.set_level(logging.DEBUG) + clock = FakeClock() + placeholder = os.path.join(str(tmp_path), 'placeholder.yaml') + write_file(placeholder, '') + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader = Reloader([f.path, placeholder], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag1'] + + write_file(f.path, '{"flagValues": {"flag1": false}}') + result = load(f) + assert result is not None and result.flags['flag1'].variations == [False] + assert deferrals_logged(caplog) == 0 + + +@yaml_required +def test_reloader_applies_a_file_that_is_deleted_and_then_created_empty_at_once(make_fixture): + f, clock = make_held_fixture(make_fixture, YAML_FLAG1, skip_missing_paths=True) + + # Deleting a file that had content removes its data at once. + os.remove(f.path) + result = load(f) + assert result is not None and result.flags == {} + + # A missing file contributed no content, so the empty file that replaces it is applied at once. + write_file(f.path, '') + result = load(f) + assert result is not None and result.flags == {} + assert result.files == [FileSummary(path=f.path, present=True)] + + +@yaml_required +def test_reloader_does_not_hold_back_a_file_whose_content_moved_away(make_fixture, tmp_path): + # The content moves from one file to the other. The file that no longer exists contributes no + # content, so when it is created empty, that is applied at once. + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.yaml') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, other], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, retry_delay=HOUR, skip_unchanged=True, clock=clock) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag1'] + + write_file(other, YAML_FLAG1) + os.remove(f.path) + result = load(f) + assert result is not None and result.files == [FileSummary(path=f.path), FileSummary(path=other, present=True, flags=1)] + + write_file(f.path, '') + result = load(f) + assert result is not None and flag_keys(result) == ['flag1'] + assert result.files == [FileSummary(path=f.path, present=True), FileSummary(path=other, present=True, flags=1)] + + +@yaml_required +def test_reloader_holds_back_a_file_whose_content_was_moved_to_it_from_another(make_fixture, tmp_path): + # The file is renamed to the other configured path (mv data.yaml other.yaml). The other + # path's content is in effect now, so an in-place rewrite of it is held back. + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.yaml') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, other], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, retry_delay=HOUR, skip_unchanged=True, clock=clock) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag1'] + + os.replace(f.path, other) + result = load(f) + assert result is not None and result.files == [FileSummary(path=f.path), FileSummary(path=other, present=True, flags=1)] + + truncate(other) + require_held_back(f) + write_file(other, YAML_FLAG2) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_applies_an_emptied_file_at_once_when_retries_are_off(make_fixture, caplog): + # Nothing would load the file later, so the empty file is parsed as it is. + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + result = load(f) + assert result is not None and result.flags == {} + assert deferrals_logged(caplog) == 0 + + +def test_reloader_reports_an_emptied_json_file_at_once_when_retries_are_off(make_fixture): + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + assert load(f) is None + assert 'no content' in str(f.errored.get_nowait()) + + +def test_reloader_hold_is_not_a_failure(make_fixture, caplog): + # A rewrite in place that writes the same content again changes nothing. The load that finds + # the file empty reports nothing, and is not a failure that a later load recovers from, so + # the unchanged content is not applied again. + caplog.set_level(logging.DEBUG) + content = '{"flagValues": {"flag1": true}}' + f, clock = make_held_fixture(make_fixture, content, name='data.json', skip_unchanged=True) + truncate(f.path) + require_held_back(f) + write_file(f.path, content) + assert load(f) is None + assert f.errored.empty() + assert not any(r.levelno >= logging.WARNING for r in caplog.records), [r.getMessage() for r in caplog.records] + + +@yaml_required +def test_reloader_initial_load_reads_an_empty_yaml_file_as_no_entries(make_fixture): + f = make_fixture('', name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + result = f.require_applied() + assert result.flags == {} + assert result.files == [FileSummary(path=f.path, present=True)] + + +def test_reloader_initial_load_fails_on_an_empty_json_file(make_fixture): + f = make_fixture('', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + assert 'no content' in str(f.require_errored()) + f.require_quiet(0.1) + + +# Real-clock tests: the retries that holds arm do fire. + +@yaml_required +def test_reloader_holds_back_an_emptied_file_until_its_write_triggers_a_load(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY, skip_unchanged=True) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag1'] + + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + f.require_quiet(0.1) + + # The write triggers a load that applies the new content. Nothing was applied in between. + write_file(f.path, YAML_FLAG2) + f.reloader.trigger() + assert flag_keys(f.require_applied(BEFORE_THE_RETRY)) == ['flag2'] + f.require_quiet(0.1) + + +@yaml_required +def test_reloader_applies_an_emptied_yaml_file_that_stays_empty_at_the_retry(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + started = time.monotonic() + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + + # The retry that the hold armed comes when the file has stayed empty and unmodified for the + # retry delay, so that first retry applies it as a document with no entries. + assert f.require_applied().flags == {} + elapsed = time.monotonic() - started + assert EMPTY_FILE_RETRY_DELAY <= elapsed < 2 * EMPTY_FILE_RETRY_DELAY + assert deferrals_logged(caplog) == 1 + + +@yaml_required +def test_reloader_a_further_hold_does_not_put_off_the_retry(make_fixture, caplog): + # A load that holds the file back again arms the retry for the time that is left of the + # delay, so the file is still applied one retry delay after it was found empty. + caplog.set_level(logging.DEBUG) + retry_delay = 3.0 + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=retry_delay) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + started = time.monotonic() + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + time.sleep(1.5) + f.reloader.trigger() + wait_for_deferrals(caplog, 2) + + # A whole retry delay from the second hold would come at 4.5 s. + assert f.require_applied().flags == {} + elapsed = time.monotonic() - started + assert retry_delay <= elapsed < retry_delay + 1.0 + + +def test_reloader_says_why_it_loads_the_files_again(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture('{"flagValues": {"flag1": true}}', retry_delay=0.2) + f.reloader.reload_now() + f.require_applied() + + # The retry that a hold arms reads the files again because a file was found empty. It finds + # the file still empty, which fails as JSON, and the retry after that failure says so. + truncate(f.path) + f.reloader.trigger() + for expected in ('Reading flag data again after a file was found empty', 'Retrying flag data load after earlier failure'): + deadline = time.monotonic() + TEST_TIMEOUT + while not any(r.getMessage() == expected for r in caplog.records): + assert time.monotonic() < deadline, "no log line %r" % expected + time.sleep(0.01) + + +def test_reloader_reports_an_emptied_json_file_only_at_the_retry_and_keeps_the_data(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture('{"flagValues": {"flag1": true}}', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + f.require_applied() + + # The load that the trigger starts holds the file back instead of reporting it. The retry + # finds it still empty and unmodified, and an empty file is not a JSON document. + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + f.require_quiet(0.1) + err = f.require_errored() + assert isinstance(err, FileReadError) + assert 'no content' in str(err) + f.require_quiet(0.1) + + +@yaml_required +def test_reloader_holds_back_a_second_rewrite_that_the_retry_of_the_first_finds_empty(make_fixture, caplog): + # The first rewrite is held back. Its content is written and the file is truncated again by + # a second rewrite before any load reads that content, as when a polled file is rewritten + # about a second apart and the poll sees only the second truncation. The retry that the + # first rewrite armed finds the file empty again, and must not apply it. + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + + # No trigger: only the retry reads the file. + write_file(f.path, YAML_FLAG2) + truncate(f.path) + change_modification_time(f.path) + wait_for_deferrals(caplog, 2) + f.require_quiet(0.2) + + write_file(f.path, 'flagValues:\n flag3: true\n') + f.reloader.trigger() + assert flag_keys(f.require_applied(BEFORE_THE_RETRY)) == ['flag3'] + + +@yaml_required +def test_reloader_retry_that_finds_a_file_emptied_holds_it_back(make_fixture, caplog): + # A retry left over from a failure can run while the file is being rewritten in place. It + # holds the file back too, rather than applying it empty. + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + f.require_applied() + write_file(f.path, 'flagValues: [') + f.reloader.trigger() + f.require_errored() + + # No trigger: only the retry reads the file. + truncate(f.path) + wait_for_deferrals(caplog, 1) + write_file(f.path, YAML_FLAG2) + assert flag_keys(f.require_applied()) == ['flag2'] + f.require_quiet(0.1) + + +@yaml_required +def test_reloader_retry_left_over_from_a_hold_applies_nothing_after_a_load_on_demand(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY, skip_unchanged=True) + f.reloader.reload_now() + f.require_applied() + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + + # A load on demand reads the rewritten file before the retry does. + write_file(f.path, YAML_FLAG2) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag2'] + + # The retry still runs. It finds the content that was just applied, so it applies and + # reports nothing. + f.require_quiet(EMPTY_FILE_RETRY_DELAY + 0.5) + + # --------------------------------------------------------------------------- # Poller # --------------------------------------------------------------------------- diff --git a/ldclient/testing/mock_components.py b/ldclient/testing/mock_components.py index ad93b32b..a569f829 100644 --- a/ldclient/testing/mock_components.py +++ b/ldclient/testing/mock_components.py @@ -1,9 +1,16 @@ +import threading import time -from typing import Callable +from typing import Any, Callable, Dict, Optional +from ldclient.impl.util import _Success from ldclient.interfaces import ( + Basis, BigSegmentStore, BigSegmentStoreMetadata, + ChangeSetBuilder, + IntentCode, + ObjectKind, + OverrideSink, Selector ) @@ -54,3 +61,142 @@ def __init__(self, selector: Selector): def selector(self) -> Selector: return self._selector + + +class MockOverrideSource: + """ + An override source for tests. It pushes its current contents to the sink when started and + on every later call to set_overrides, and records its lifecycle calls. + """ + + def __init__( + self, + flags: Optional[Dict[str, Any]] = None, + segments: Optional[Dict[str, Any]] = None, + start_error: Optional[Exception] = None, + supply_before_start_error: bool = False, + close_error: Optional[Exception] = None, + ): + """ + :param start_error: when given, start raises it instead of supplying the contents + :param supply_before_start_error: with start_error, start supplies the contents and then raises + :param close_error: when given, close raises it after recording the call + """ + self._flags: Dict[str, Any] = dict(flags or {}) + self._segments: Dict[str, Any] = dict(segments or {}) + self._start_error = start_error + self._supply_before_start_error = supply_before_start_error + self._close_error = close_error + self._sink: Optional[OverrideSink] = None + self.start_count = 0 + self.close_count = 0 + + def start(self, sink: OverrideSink) -> None: + self.start_count += 1 + if self._start_error is not None: + if self._supply_before_start_error: + self._sink = sink + sink.set_overrides(self._flags, self._segments) + raise self._start_error + self._sink = sink + sink.set_overrides(self._flags, self._segments) + + def close(self) -> None: + self.close_count += 1 + if self._close_error is not None: + raise self._close_error + + def set_overrides(self, flags: Optional[Dict[str, Any]] = None, segments: Optional[Dict[str, Any]] = None) -> None: + """Replaces the source's contents and, once started, pushes them to the sink.""" + self._flags = dict(flags or {}) + self._segments = dict(segments or {}) + if self._sink is not None: + self._sink.set_overrides(self._flags, self._segments) + + @property + def builder(self) -> 'MockOverrideSourceBuilder': + return MockOverrideSourceBuilder(self) + + +class MockOverrideSourceBuilder: + """Wraps a MockOverrideSource so it can be passed to the data system configuration.""" + + def __init__(self, source: MockOverrideSource): + self._source = source + self.build_count = 0 + + def build(self, config) -> MockOverrideSource: + self.build_count += 1 + return self._source + + +class FailingOverrideSourceBuilder: + """A builder whose build raises, to simulate invalid override source configuration.""" + + def build(self, config): + raise ValueError("invalid override source configuration") + + +class HangingSynchronizer: + """ + A synchronizer that connects but never yields data, so the client stays uninitialized + until it is stopped. + """ + + def __init__(self): + self._stop = threading.Event() + + @property + def name(self) -> str: + return "HangingSynchronizer" + + def sync(self, ss): + self._stop.wait() + yield from () + + def stop(self): + self._stop.set() + + @property + def builder(self) -> 'MockDataSourceBuilder': + return MockDataSourceBuilder(self) + + +class StaticInitializer: + """ + An initializer that supplies fixed flag and segment definitions as a full-transfer basis + with a defined selector, so the client reports full data availability once it has applied + the data. + """ + + def __init__(self, flags: Optional[Dict[str, dict]] = None, segments: Optional[Dict[str, dict]] = None): + self._flags = dict(flags or {}) + self._segments = dict(segments or {}) + + @property + def name(self) -> str: + return "StaticInitializer" + + def fetch(self, ss): + builder = ChangeSetBuilder() + builder.start(IntentCode.TRANSFER_FULL) + for key, flag in self._flags.items(): + builder.add_put(ObjectKind.FLAG, key, flag.get('version', 1), flag) + for key, segment in self._segments.items(): + builder.add_put(ObjectKind.SEGMENT, key, segment.get('version', 1), segment) + change_set = builder.finish(Selector.new_selector('test-state', 1)) + return _Success(Basis(change_set=change_set, persist=False, environment_id=None)) + + @property + def builder(self) -> 'MockDataSourceBuilder': + return MockDataSourceBuilder(self) + + +class MockDataSourceBuilder: + """Wraps a ready-made data source so it can be passed to the data system configuration.""" + + def __init__(self, source): + self._source = source + + def build(self, config): + return self._source diff --git a/ldclient/testing/test_async_client_override_vectors.py b/ldclient/testing/test_async_client_override_vectors.py new file mode 100644 index 00000000..3eab7e3f --- /dev/null +++ b/ldclient/testing/test_async_client_override_vectors.py @@ -0,0 +1,71 @@ +""" +Runs the OVERRIDE specification test vectors through the async client. The vectors and the +checks are those of the sync runner: each vector sets up LaunchDarkly data, an override layer, +and an initialization state, and the test evaluates one flag through the full client stack. +""" +from typing import Any, Dict + +import pytest + +from ldclient.async_client import AsyncLDClient +from ldclient.async_config import AsyncConfig, AsyncDataSystemConfig +from ldclient.context import Context +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.testing.mock_async_components import MockAsyncEventProcessor +from ldclient.testing.mock_components import ( + MockDataSourceBuilder, + MockOverrideSource +) +from ldclient.testing.test_async_client_overrides import ( + AsyncHangingSynchronizer, + AsyncStaticInitializer +) +from ldclient.testing.test_ldclient_override_vectors import ( + assert_reason, + load_vectors, + override_flags, + vector_id +) + + +async def make_client(vector: Dict[str, Any]) -> AsyncLDClient: + source = MockOverrideSource(flags=override_flags(vector['overrides']), segments=dict(vector['overrides'].get('segments', {}))) + ld_data = vector['launchDarklyData'] + if ld_data['initialized']: + initializer = AsyncStaticInitializer(ld_data.get('flags', {}), ld_data.get('segments', {})) + datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder) + start_wait = 5 + else: + # With no sources at all, the client would consider cached data available rather than + # applying its not-initialized handling. A synchronizer that never delivers avoids that. + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder) + start_wait = 0 + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=start_wait) + return client + + +@pytest.mark.asyncio +@pytest.mark.parametrize("vector", load_vectors(), ids=vector_id) +async def test_override_spec_vector(vector: Dict[str, Any]): + client = await make_client(vector) + try: + assert await client.is_initialized() is vector['launchDarklyData']['initialized'] + evaluate = vector['evaluate'] + detail = await client.variation_detail(evaluate['flagKey'], Context.from_dict(evaluate['context']), evaluate['defaultValue']) + + expect = vector['expect'] + assert detail.value == expect['value'], "value" + assert detail.variation_index == expect['variationIndex'], "variationIndex" + assert_reason(expect['reason'], detail.reason) + + # summaryOverrideAffected is the marking the client hands to the event processor for this + # evaluation. The event processor keys individual-event suppression and the summary + # counter marker on that scalar, not on the reason. + if 'summaryOverrideAffected' in expect: + records = [e for e in client._event_processor.events if isinstance(e, EventInputEvaluation) and e.key == evaluate['flagKey']] + assert len(records) == 1, "expected exactly one evaluation record for the flag" + assert records[0].override_affected is expect['summaryOverrideAffected'], "summaryOverrideAffected" + finally: + await client.close() diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py new file mode 100644 index 00000000..8fe69458 --- /dev/null +++ b/ldclient/testing/test_async_client_overrides.py @@ -0,0 +1,650 @@ +""" +Tests for flag overrides through the async client. These mirror the key scenarios of the sync +client tests. +""" +import asyncio +import json +import logging +import threading +import time +from typing import Any, Dict, List, Optional + +import pytest + +from ldclient.async_client import AsyncLDClient +from ldclient.async_config import AsyncConfig, AsyncDataSystemConfig +from ldclient.async_feature_store import AsyncInMemoryFeatureStore +from ldclient.context import Context +from ldclient.evaluation import EvaluationDetail +from ldclient.hook import AsyncHook, EvaluationSeriesContext, Metadata +from ldclient.impl.aio.concurrency import AsyncEvent +from ldclient.impl.events.async_event_processor import ( + DefaultAsyncEventProcessor +) +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.impl.integrations.files.filedata import make_flag_with_value +from ldclient.interfaces import AsyncFeatureStore, DataStoreMode +from ldclient.migrations import Stage +from ldclient.testing.builders import FlagBuilder +from ldclient.testing.impl.events.test_async_event_processor import MockAioHttp +from ldclient.testing.mock_async_components import MockAsyncEventProcessor +from ldclient.testing.mock_components import ( + FailingOverrideSourceBuilder, + MockDataSourceBuilder, + MockOverrideSource, + StaticInitializer +) +from ldclient.versioned_data_kind import FEATURES + +user = Context.create('user-key') + +# A context whose key is empty is invalid. +invalid_context = Context.create('') + + +class AsyncHangingSynchronizer: + """An async synchronizer that connects but never yields data.""" + + def __init__(self): + self._stop = AsyncEvent() + + @property + def name(self) -> str: + return "AsyncHangingSynchronizer" + + async def sync(self, ss): + await self._stop.wait() + return + yield + + async def stop(self) -> None: + self._stop.set() + + +class AsyncStaticInitializer(StaticInitializer): + async def fetch(self, ss): # type: ignore[override] + return super().fetch(ss) + + +class UnavailableAsyncStore(AsyncInMemoryFeatureStore): + """A persistent store whose every read fails, as during a store outage.""" + + async def is_initialized(self) -> bool: + raise RuntimeError("store unreachable") + + async def get(self, kind, key): + raise RuntimeError("store unreachable") + + async def all(self, kind): + raise RuntimeError("store unreachable") + + +class UninitializedAsyncStore(AsyncInMemoryFeatureStore): + """A persistent store that holds data but was never initialized by an SDK.""" + + @property + def initialized(self) -> bool: + return False + + async def is_initialized(self) -> bool: + return False + + +class AsyncStoreThatCannotListItems(AsyncInMemoryFeatureStore): + """An initialized persistent store whose enumeration fails.""" + + @property + def initialized(self) -> bool: + return True + + async def is_initialized(self) -> bool: + return True + + async def all(self, kind): + raise RuntimeError("store cannot list items") + + +def records_containing(caplog, level: int, text: str): + return [r for r in caplog.records if r.levelno == level and text in r.getMessage()] + + +async def uninitialized_store_holding(key: str) -> UninitializedAsyncStore: + store = UninitializedAsyncStore() + await store.upsert(FEATURES, FlagBuilder(key).version(1).on(False).off_variation(0).variations('ld-value').build().to_json_dict()) + return store + + +def single_value_flag(key: str, value: Any) -> dict: + return make_flag_with_value(key, value).to_json_dict() + + +def recorded_keys(client: AsyncLDClient) -> List[str]: + """The flag keys of the evaluations the client handed to the event processor.""" + processor: Any = client._event_processor + return [event.key for event in processor.events] + + +async def make_uninitialized_client(source: MockOverrideSource, store: Optional[AsyncFeatureStore] = None) -> AsyncLDClient: + """ + A client whose data system can never obtain LaunchDarkly data, with the given override + source and, when given, a read-only persistent store. + """ + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder, data_store=store, data_store_mode=DataStoreMode.READ_ONLY) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=0) + return client + + +async def make_initialized_client(flags: Dict[str, dict], source: MockOverrideSource, segments: Optional[Dict[str, dict]] = None) -> AsyncLDClient: + initializer = AsyncStaticInitializer(flags, segments or {}) + datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=5) + assert await client.is_initialized() is True + return client + + +@pytest.mark.asyncio +async def test_override_is_served_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + assert await client.is_initialized() is False + detail = await client.variation_detail('overridden-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + finally: + await client.close() + assert source.close_count == 1 + + +@pytest.mark.asyncio +async def test_non_overridden_flag_still_short_circuits_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + detail = await client.variation_detail('other-flag', user, False) + assert detail.value is False + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_removal_restores_short_circuit(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + assert await client.variation('overridden-flag', user, False) is True + source.set_overrides({}, {}) + detail = await client.variation_detail('overridden-flag', user, False) + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + finally: + await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('overrides', [{}, {'other-flag': single_value_flag('other-flag', True)}], ids=['empty-layer', 'layer-without-the-key']) +async def test_not_initialized_client_returns_not_ready_for_an_invalid_context_when_the_layer_lacks_the_key(overrides): + client = await make_uninitialized_client(MockOverrideSource(flags=overrides)) + try: + detail = await client.variation_detail('requested-flag', invalid_context, False) + # The not-ready handling runs before the context check, as it does without overrides, + # and records the evaluation of the unknown flag. + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert recorded_keys(client) == ['requested-flag'] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_not_initialized_client_checks_the_context_before_serving_an_override(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + detail = await client.variation_detail('overridden-flag', invalid_context, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'USER_NOT_SPECIFIED'}) + assert recorded_keys(client) == [] + finally: + await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('store_outage', [True, False], ids=['store-outage', 'uninitialized-store']) +async def test_not_initialized_client_returns_not_ready_when_the_store_has_no_launchdarkly_data_and_the_layer_lacks_the_key(store_outage): + store: AsyncFeatureStore = UnavailableAsyncStore() if store_outage else await uninitialized_store_holding('requested-flag') + source = MockOverrideSource(flags={'other-flag': single_value_flag('other-flag', True)}) + client = await make_uninitialized_client(source, store=store) + try: + detail = await client.variation_detail('requested-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert recorded_keys(client) == ['requested-flag'] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_is_served_during_a_persistent_store_outage(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source, store=UnavailableAsyncStore()) + try: + detail = await client.variation_detail('overridden-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_is_invalid_when_the_store_is_uninitialized_and_the_layer_is_empty(): + client = await make_uninitialized_client(MockOverrideSource(), store=await uninitialized_store_holding('ld-flag')) + try: + state = await client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_before_initialization_excludes_launchdarkly_data_from_an_uninitialized_store(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source, store=await uninitialized_store_holding('ld-flag')) + try: + state = await client.all_flags_state(user) + assert state.valid is True + assert state.to_values_map() == {'overridden-flag': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_holds_the_overrides_and_is_invalid_when_the_store_fails(caplog): + # A daemon-mode client whose store cannot list its flags logs the failure, as it does without + # a source. Overrides only add to the state, so it holds the override flags, and it stays + # invalid because the store's flags are missing. + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + datasystem = AsyncDataSystemConfig(override_source=source.builder, data_store=AsyncStoreThatCannotListItems(), data_store_mode=DataStoreMode.READ_ONLY) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=0) + try: + state = await client.all_flags_state(user, with_reasons=True) + assert state.valid is False + assert state.to_values_map() == {'overridden-flag': True} + assert state.get_flag_reason('overridden-flag') == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert records_containing(caplog, logging.ERROR, 'Unable to read flags for all_flag_state') != [] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_is_invalid_and_empty_when_the_store_fails_and_the_layer_holds_no_flags(caplog): + source = MockOverrideSource() + datasystem = AsyncDataSystemConfig(override_source=source.builder, data_store=AsyncStoreThatCannotListItems(), data_store_mode=DataStoreMode.READ_ONLY) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=0) + try: + state = await client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + assert records_containing(caplog, logging.ERROR, 'Unable to read flags for all_flag_state') != [] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_source_start_failure_is_logged_and_the_client_runs_without_overrides(caplog): + # The SDK's own source never raises from start. A custom source could, and that must not + # keep the client from starting: the failure is logged, the source is closed, the snapshot + # it supplied before failing is removed, and a later snapshot is ignored. + flags = {'overridden-flag': single_value_flag('overridden-flag', True)} + source = MockOverrideSource(flags=flags, start_error=RuntimeError("cannot start"), supply_before_start_error=True) + client = await make_uninitialized_client(source) + try: + assert source.close_count == 1 + assert records_containing(caplog, logging.ERROR, 'Unable to start the override source; the client runs without overrides: cannot start') != [] + detail = await client.variation_detail('overridden-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + + source.set_overrides(flags) + assert await client.variation('overridden-flag', user, False) is False + finally: + await client.close() + assert source.close_count == 1 + + +@pytest.mark.asyncio +async def test_snapshot_supplied_after_the_client_is_closed_is_ignored(): + source = MockOverrideSource() + client = await make_uninitialized_client(source) + await client.close() + + source.set_overrides({'late-flag': single_value_flag('late-flag', True)}) + + assert client._data_system.override_layer.get(FEATURES, 'late-flag') is None + + +@pytest.mark.asyncio +async def test_override_source_is_not_started_when_offline(caplog): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + datasystem = AsyncDataSystemConfig(override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, offline=True) + client = AsyncLDClient(config) + await client.start(start_wait=0) + try: + assert source.start_count == 0 + assert await client.variation('overridden-flag', user, False) is False + assert records_containing(caplog, logging.WARNING, 'Offline mode is enabled; the override source is not started and no overrides are served') != [] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_invalid_override_source_configuration_fails_construction(): + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=FailingOverrideSourceBuilder()) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + with pytest.raises(ValueError): + AsyncLDClient(config) + + +@pytest.mark.asyncio +async def test_override_takes_precedence_over_launchdarkly_data_and_all_flags_reflects_it(): + ld_flag = FlagBuilder('flag-precedence').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').build().to_json_dict() + source = MockOverrideSource(flags={'flag-precedence': single_value_flag('flag-precedence', 'override-value')}) + client = await make_initialized_client({'flag-precedence': ld_flag, 'flag-normal': normal}, source) + try: + detail = await client.variation_detail('flag-precedence', user, 'default') + assert detail.value == 'override-value' + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + detail = await client.variation_detail('flag-normal', user, 'default') + assert detail.reason == {'kind': 'OFF'} + + state = await client.all_flags_state(user, with_reasons=True) + assert state.to_values_map() == {'flag-precedence': 'override-value', 'flag-normal': 'normal-value'} + assert state.get_flag_reason('flag-precedence') == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_contains_only_overrides_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + state = await client.all_flags_state(user) + assert state.valid is True + assert state.to_values_map() == {'overridden-flag': True} + source.set_overrides({}, {}) + assert (await client.all_flags_state(user)).valid is False + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_flag_tracker_is_notified_of_override_changes_on_the_event_loop(): + source = MockOverrideSource() + client = await make_uninitialized_client(source) + try: + changes: asyncio.Queue = asyncio.Queue() + loop_thread = threading.current_thread() + listener_threads = [] + + def listener(change): + listener_threads.append(threading.current_thread()) + changes.put_nowait(change) + + client.flag_tracker.add_listener(listener) + + # The source pushes from a worker thread, as the file source does. The notification + # is delivered on the event loop's thread. + await asyncio.to_thread(source.set_overrides, {'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + change = await asyncio.wait_for(changes.get(), 5) + assert change.key == 'overridden-flag' + assert listener_threads[0] is loop_thread + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_flag_value_change_listener_sees_override_value_changes(): + ld_flag = FlagBuilder('flag').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + source = MockOverrideSource() + client = await make_initialized_client({'flag': ld_flag}, source) + try: + changes: asyncio.Queue = asyncio.Queue() + await client.flag_tracker.add_flag_value_change_listener('flag', user, changes.put_nowait) + await asyncio.to_thread(source.set_overrides, {'flag': single_value_flag('flag', 'override-value')}, {}) + change = await asyncio.wait_for(changes.get(), 5) + assert change.old_value == 'ld-value' + assert change.new_value == 'override-value' + finally: + await client.close() + + +async def raise_evaluation_failure(*args): + raise RuntimeError("evaluation failure") + + +@pytest.mark.asyncio +async def test_failed_evaluation_reason_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + # The evaluation read the flag definition before it failed, so the failure of the override + # flag is marked and the failure of the ordinary flag is not, in a single evaluation and in + # the all-flags state. + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_initialized_client({'plain-flag': single_value_flag('plain-flag', True)}, source) + try: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + assert await client.variation_detail('overridden-flag', user, 'default') == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + assert await client.variation_detail('plain-flag', user, 'default') == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + state = await client.all_flags_state(user, with_reasons=True) + assert state.get_flag_reason('overridden-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert state.get_flag_reason('plain-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_evaluation_events_carry_override_affected_marking(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + assert await client.variation('overridden-flag', user, False) is True + records = [e for e in client._event_processor.events if isinstance(e, EventInputEvaluation)] + assert [e.key for e in records] == ['overridden-flag'] + assert records[0].override_affected is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_turns_off_event_tracking_for_override_affected_flags(): + overridden = FlagBuilder('overridden-flag').version(7).on(False).off_variation(0).variations(True).track_events(True).debug_events_until_date(int(time.time() * 1000) + 100000).build().to_json_dict() + plain = FlagBuilder('plain-tracked').version(1).on(False).off_variation(0).variations(True).track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + client = await make_initialized_client({'plain-tracked': plain}, source) + try: + state = await client.all_flags_state(user, with_reasons=True) + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['plain-tracked']['trackEvents'] is True + assert 'trackEvents' not in flags_state['overridden-flag'] + assert 'debugEventsUntilDate' not in flags_state['overridden-flag'] + assert flags_state['overridden-flag']['reason']['overrideAffected'] is True + assert state.get_flag_value('overridden-flag') is True + finally: + await client.close() + + +def test_override_update_after_the_client_and_its_loop_are_gone_is_dropped(): + source = MockOverrideSource() + + async def run_client() -> None: + client = await make_uninitialized_client(source) + client.flag_tracker.add_listener(lambda change: None) + await client.close() + + asyncio.run(run_client()) + # A reload that finishes after the client closed finds the event loop closed. The + # notification has nobody left to reach and is dropped. + source.set_overrides({'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +def tracked_bool_flag(key: str) -> FlagBuilder: + return FlagBuilder(key).version(100).variations(False, True).off_variation(0).fallthrough_variation(1).track_events(True) + + +def evaluation_events_by_key(client: AsyncLDClient) -> Dict[str, EventInputEvaluation]: + """The evaluation records the client handed to the event processor, keyed by flag key.""" + processor: Any = client._event_processor + return {event.key: event for event in processor.events if isinstance(event, EventInputEvaluation)} + + +@pytest.mark.asyncio +async def test_overridden_prerequisite_marks_the_dependent_evaluation_records(): + # top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden) + # --> plain-flag (LaunchDarkly) + # The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override. + ld_data = { + 'top-flag': tracked_bool_flag('top-flag').on(True).prerequisite('mid-flag', 1).prerequisite('plain-flag', 1).build().to_json_dict(), + 'mid-flag': tracked_bool_flag('mid-flag').on(True).prerequisite('leaf-flag', 1).build().to_json_dict(), + 'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict(), + 'leaf-flag': tracked_bool_flag('leaf-flag').on(False).build().to_json_dict(), + } + source = MockOverrideSource(flags={'leaf-flag': tracked_bool_flag('leaf-flag').on(True).build().to_json_dict()}) + client = await make_initialized_client(ld_data, source) + try: + detail = await client.variation_detail('top-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + records = evaluation_events_by_key(client) + assert sorted(records.keys()) == ['leaf-flag', 'mid-flag', 'plain-flag', 'top-flag'] + assert records['top-flag'].override_affected is True + assert records['mid-flag'].override_affected is True + assert records['leaf-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['leaf-flag'].reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert records['plain-flag'].reason == {'kind': 'FALLTHROUGH'} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_wrong_type_result_of_overridden_flag_stays_marked(): + details = [] + + class CapturingHook(AsyncHook): + @property + def metadata(self) -> Metadata: + return Metadata(name='capturing-hook') + + async def before_evaluation(self, series_context: EvaluationSeriesContext, data: dict) -> dict: + return data + + async def after_evaluation(self, series_context: EvaluationSeriesContext, data: dict, detail: EvaluationDetail) -> dict: + details.append(detail) + return data + + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', 'not-a-stage')}) + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor(), hooks=[CapturingHook()]) + client = AsyncLDClient(config) + await client.start(start_wait=0) + try: + stage, _ = await client.migration_variation('overridden-flag', user, Stage.OFF) + assert stage == Stage.OFF + assert len(details) == 1 + assert details[0].value == 'off' + assert details[0].reason == {'kind': 'ERROR', 'errorKind': 'WRONG_TYPE', 'overrideAffected': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_failed_evaluation_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + client = await make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) + try: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + # The failure of the override flag is marked. The failure of the ordinary flag is not. + detail = await client.variation_detail('overridden-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + detail = await client.variation_detail('plain-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + # The marked record produces no individual event. The ordinary record keeps its tracking. + records = evaluation_events_by_key(client) + assert records['overridden-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['plain-flag'].track_events is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_turns_off_event_tracking_for_an_override_flag_whose_evaluation_fails(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + client = await make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) + try: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + state = await client.all_flags_state(user, with_reasons=True) + + # The failed override flag stays in the state with a marked reason and no tracking + # fields. The failed ordinary flag keeps its tracking fields. + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['overridden-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert 'trackEvents' not in flags_state['overridden-flag'] + assert flags_state['plain-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + assert flags_state['plain-flag']['trackEvents'] is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_affected_evaluations_appear_only_in_summary_output(): + # End to end through the real event processor: the overridden flag requests individual + # feature events and debug events, and an ordinary flag requests feature events. + debug_until = int(time.time() * 1000) + 100000 + overridden = FlagBuilder('flag-tracked-override').version(300).on(False).off_variation(0).variations('override-value').track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'flag-tracked-override': overridden}) + mock_http = MockAioHttp() + + initializer = AsyncStaticInitializer({'flag-normal': normal}, {}) + datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, diagnostic_opt_out=True, event_processor_class=lambda config: DefaultAsyncEventProcessor(config, mock_http)) + client = AsyncLDClient(config) + await client.start(start_wait=5) + try: + assert await client.is_initialized() is True + for _ in range(2): + assert await client.variation('flag-tracked-override', user, 'default1') == 'override-value' + assert await client.variation('flag-normal', user, 'default2') == 'normal-value' + assert await client._event_processor.flush_and_wait(5) is True + finally: + await client.close() + + assert mock_http.request_data is not None + output = json.loads(mock_http.request_data) + kinds = sorted(e['kind'] for e in output) + assert kinds == ['feature', 'index', 'summary'] + feature = [e for e in output if e['kind'] == 'feature'][0] + assert feature['key'] == 'flag-normal' + summary = [e for e in output if e['kind'] == 'summary'][0] + assert summary['features']['flag-tracked-override']['default'] == 'default1' + assert summary['features']['flag-tracked-override']['counters'] == [ + {'count': 2, 'value': 'override-value', 'variation': 0, 'version': 300, 'overrideAffected': True} + ] + assert summary['features']['flag-normal']['counters'] == [{'count': 1, 'value': 'normal-value', 'variation': 0, 'version': 100}] diff --git a/ldclient/testing/test_ldclient_override_vectors.py b/ldclient/testing/test_ldclient_override_vectors.py new file mode 100644 index 00000000..a55ea87d --- /dev/null +++ b/ldclient/testing/test_ldclient_override_vectors.py @@ -0,0 +1,95 @@ +""" +Runs the OVERRIDE specification test vectors. Each vector sets up LaunchDarkly data, an +override layer, and an initialization state. The test evaluates one flag through the full +client stack and checks the value, the variation index, the reason, and the marking handed to the +event processor. +""" +import json +import os +from typing import Any, Dict + +import pytest + +from ldclient.client import Config, Context, LDClient +from ldclient.datasystem import custom +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.impl.integrations.files.filedata import make_flag_with_value +from ldclient.testing.mock_components import ( + HangingSynchronizer, + MockOverrideSource, + StaticInitializer +) +from ldclient.testing.stub_util import MockEventProcessor + +VECTORS_PATH = os.path.join(os.path.dirname(__file__), 'testdata', 'override-vectors', 'vectors.json') + +# The vectors' semantics are versioned. A schema change means this runner needs review. +SUPPORTED_SCHEMA_VERSION = '0.4.0' + + +def load_vectors(): + with open(VECTORS_PATH, 'r') as f: + document = json.load(f) + assert document['schemaVersion'] == SUPPORTED_SCHEMA_VERSION, "the vectors changed schema; review this runner against the new schema before updating" + assert len(document['vectors']) > 0 + return document['vectors'] + + +def vector_id(vector: Dict[str, Any]) -> str: + return "%s: %s" % (vector['group'], vector['description']) + + +def override_flags(overrides: Dict[str, Any]) -> Dict[str, Any]: + flags = dict(overrides.get('flags', {})) + for key, value in overrides.get('flagValues', {}).items(): + flags[key] = make_flag_with_value(key, value) + return flags + + +def make_client(vector: Dict[str, Any]) -> LDClient: + source = MockOverrideSource(flags=override_flags(vector['overrides']), segments=dict(vector['overrides'].get('segments', {}))) + ld_data = vector['launchDarklyData'] + datasystem = custom().overrides(source.builder) + if ld_data['initialized']: + datasystem.initializers([StaticInitializer(ld_data.get('flags', {}), ld_data.get('segments', {})).builder]) + start_wait = 5 + else: + # With no sources at all, the client would consider cached data available rather than + # applying its not-initialized handling. A synchronizer that never delivers avoids that. + datasystem.synchronizers(HangingSynchronizer().builder) + start_wait = 0 + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem.build(), event_processor_class=MockEventProcessor) + return LDClient(config, start_wait=start_wait) + + +def assert_reason(expected: Dict[str, Any], actual: Dict[str, Any]) -> None: + """ + The reason is compared only on the fields present in the expected reason. The + override-affected indicator is a tri-state collapse: an expected reason that omits it + requires the actual reason to omit it or report it as false. + """ + for name, value in expected.items(): + assert actual.get(name) == value, "reason property %s: expected %r in %r" % (name, value, actual) + if 'overrideAffected' not in expected: + assert actual.get('overrideAffected', False) is False, "reason must not be override-affected: %r" % actual + + +@pytest.mark.parametrize("vector", load_vectors(), ids=vector_id) +def test_override_spec_vector(vector: Dict[str, Any]): + with make_client(vector) as client: + assert client.is_initialized() is vector['launchDarklyData']['initialized'] + evaluate = vector['evaluate'] + detail = client.variation_detail(evaluate['flagKey'], Context.from_dict(evaluate['context']), evaluate['defaultValue']) + + expect = vector['expect'] + assert detail.value == expect['value'], "value" + assert detail.variation_index == expect['variationIndex'], "variationIndex" + assert_reason(expect['reason'], detail.reason) + + # summaryOverrideAffected is the marking the client hands to the event processor for this + # evaluation. The event processor keys individual-event suppression and the summary + # counter marker on that scalar, not on the reason. + if 'summaryOverrideAffected' in expect: + records = [e for e in client._event_processor._events if isinstance(e, EventInputEvaluation) and e.key == evaluate['flagKey']] + assert len(records) == 1, "expected exactly one evaluation record for the flag" + assert records[0].override_affected is expect['summaryOverrideAffected'], "summaryOverrideAffected" diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py new file mode 100644 index 00000000..78d0a1f9 --- /dev/null +++ b/ldclient/testing/test_ldclient_overrides.py @@ -0,0 +1,692 @@ +""" +Tests for flag overrides through the client: the override source lifecycle, the overlay at the +store read boundary, the not-initialized gate, the all-flags state, and flag change notifications. +""" +import json +import logging +import time +from queue import Empty, Queue +from typing import Any, Dict, List, Optional + +import pytest + +from ldclient.client import Config, Context, LDClient +from ldclient.datasystem import custom, daemon +from ldclient.evaluation import EvaluationDetail +from ldclient.feature_store import InMemoryFeatureStore +from ldclient.hook import EvaluationSeriesContext, Hook, Metadata +from ldclient.impl.datasystem.fdv1 import FDv1 +from ldclient.impl.events.event_processor import DefaultEventProcessor +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.impl.integrations.files.filedata import make_flag_with_value +from ldclient.integrations.test_datav2 import TestDataV2 +from ldclient.interfaces import ( + DataSourceState, + DataStoreMode, + FeatureStore, + FlagChange +) +from ldclient.migrations import Stage +from ldclient.testing.builders import ( + FlagBuilder, + FlagRuleBuilder, + SegmentBuilder, + make_clause_matching_segment_key +) +from ldclient.testing.mock_components import ( + FailingOverrideSourceBuilder, + HangingSynchronizer, + MockOverrideSource, + StaticInitializer +) +from ldclient.testing.stub_util import MockEventProcessor, MockHttp +from ldclient.versioned_data_kind import FEATURES + +user = Context.create('user-key') + +# A context whose key is empty is invalid. +invalid_context = Context.create('') + + +def single_value_flag(key: str, value: Any) -> dict: + return make_flag_with_value(key, value).to_json_dict() + + +def make_uninitialized_client(source: MockOverrideSource, store: Optional[FeatureStore] = None) -> LDClient: + """ + A client whose data system can never obtain LaunchDarkly data, with the given override + source and, when given, a read-only persistent store. + """ + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source.builder) + if store is not None: + datasystem.data_store(store, DataStoreMode.READ_ONLY) + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem.build(), event_processor_class=MockEventProcessor) + return LDClient(config, start_wait=0) + + +def make_initialized_client(flags: Dict[str, dict], source: MockOverrideSource, segments: Optional[Dict[str, dict]] = None) -> LDClient: + """A client that has initialized with the given LaunchDarkly data, with the given override source.""" + initializer = StaticInitializer(flags, segments or {}) + datasystem = custom().initializers([initializer.builder]).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + client = LDClient(config, start_wait=5) + assert client.is_initialized() is True + return client + + +def warnings_containing(caplog, text: str): + return [r for r in caplog.records if r.levelno == logging.WARNING and text in r.getMessage()] + + +def recorded_keys(client: LDClient) -> List[str]: + """The flag keys of the evaluations the client handed to the event processor.""" + processor: Any = client._event_processor + return [event.key for event in processor._events] + + +class UnavailableStore(InMemoryFeatureStore): + """A persistent store whose every read fails, as during a store outage.""" + + @property + def initialized(self) -> bool: + raise RuntimeError("store unreachable") + + def get(self, kind, key, callback=lambda x: x): + raise RuntimeError("store unreachable") + + def all(self, kind, callback=lambda x: x): + raise RuntimeError("store unreachable") + + +class UninitializedStore(InMemoryFeatureStore): + """A persistent store that holds data but was never initialized by an SDK.""" + + @property + def initialized(self) -> bool: + return False + + +def uninitialized_store_holding(key: str) -> UninitializedStore: + store = UninitializedStore() + store.upsert(FEATURES, FlagBuilder(key).version(1).on(False).off_variation(0).variations('ld-value').build().to_json_dict()) + return store + + +class StoreThatCannotListItems(InMemoryFeatureStore): + """An initialized persistent store whose enumeration fails.""" + + @property + def initialized(self) -> bool: + return True + + def all(self, kind, callback=lambda x: x): + raise RuntimeError("store cannot list items") + + +def errors_containing(caplog, text: str): + return [r for r in caplog.records if r.levelno == logging.ERROR and text in r.getMessage()] + + +def test_override_is_served_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.is_initialized() is False + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is True + assert detail.variation_index == 0 + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + +def test_non_overridden_flag_still_short_circuits_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + detail = client.variation_detail('other-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + + +def test_override_removal_restores_short_circuit(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.variation('overridden-flag', user, False) is True + source.set_overrides({}, {}) + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is False + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + + +@pytest.mark.parametrize('overrides', [{}, {'other-flag': single_value_flag('other-flag', True)}], ids=['empty-layer', 'layer-without-the-key']) +def test_not_initialized_client_returns_not_ready_for_an_invalid_context_when_the_layer_lacks_the_key(overrides): + with make_uninitialized_client(MockOverrideSource(flags=overrides)) as client: + detail = client.variation_detail('requested-flag', invalid_context, False) + # The not-ready handling runs before the context check, as it does without overrides, + # and records the evaluation of the unknown flag. + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert recorded_keys(client) == ['requested-flag'] + + +def test_not_initialized_client_checks_the_context_before_serving_an_override(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + detail = client.variation_detail('overridden-flag', invalid_context, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'USER_NOT_SPECIFIED'}) + assert recorded_keys(client) == [] + + +@pytest.mark.parametrize('make_store', [UnavailableStore, lambda: uninitialized_store_holding('requested-flag')], ids=['store-outage', 'uninitialized-store']) +def test_not_initialized_client_returns_not_ready_when_the_store_has_no_launchdarkly_data_and_the_layer_lacks_the_key(make_store): + source = MockOverrideSource(flags={'other-flag': single_value_flag('other-flag', True)}) + with make_uninitialized_client(source, store=make_store()) as client: + detail = client.variation_detail('requested-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert recorded_keys(client) == ['requested-flag'] + + +def test_override_is_served_during_a_persistent_store_outage(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source, store=UnavailableStore()) as client: + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + +def test_all_flags_state_is_invalid_when_the_store_is_uninitialized_and_the_layer_is_empty(): + with make_uninitialized_client(MockOverrideSource(), store=uninitialized_store_holding('ld-flag')) as client: + state = client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + + +def test_all_flags_state_is_invalid_when_the_store_is_uninitialized_and_the_layer_holds_no_flags(): + # A layer that holds only segments has no flags to report, so the LaunchDarkly data in the + # store stays out, as it does without an override source. + source = MockOverrideSource(segments={'segment1': SegmentBuilder('segment1').version(1).build().to_json_dict()}) + with make_uninitialized_client(source, store=uninitialized_store_holding('ld-flag')) as client: + state = client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + + +def test_all_flags_state_before_initialization_excludes_launchdarkly_data_from_an_uninitialized_store(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source, store=uninitialized_store_holding('ld-flag')) as client: + state = client.all_flags_state(user) + assert state.valid is True + assert state.to_values_map() == {'overridden-flag': True} + + +def test_all_flags_state_holds_the_overrides_and_is_invalid_when_the_store_fails(caplog): + # A daemon-mode client whose store cannot list its flags logs the failure, as it does without + # a source. Overrides only add to the state, so it holds the override flags, and it stays + # invalid because the store's flags are missing. + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + datasystem = daemon(StoreThatCannotListItems()).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with LDClient(config, start_wait=0) as client: + state = client.all_flags_state(user, with_reasons=True) + assert state.valid is False + assert state.to_values_map() == {'overridden-flag': True} + assert state.get_flag_reason('overridden-flag') == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert errors_containing(caplog, 'Unable to read flags for all_flag_state') != [] + + +def test_all_flags_state_is_invalid_and_empty_when_the_store_fails_and_the_layer_holds_no_flags(caplog): + source = MockOverrideSource() + datasystem = daemon(StoreThatCannotListItems()).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with LDClient(config, start_wait=0) as client: + state = client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + assert errors_containing(caplog, 'Unable to read flags for all_flag_state') != [] + + +def test_override_source_is_started_before_the_constructor_returns(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = make_uninitialized_client(source) + try: + assert source.start_count == 1 + assert client.variation('overridden-flag', user, False) is True + finally: + client.close() + + +def test_override_source_is_closed_when_the_client_is_closed(): + source = MockOverrideSource() + client = make_uninitialized_client(source) + assert source.close_count == 0 + client.close() + assert source.close_count == 1 + + +def test_invalid_override_source_configuration_fails_construction(): + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(FailingOverrideSourceBuilder()).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with pytest.raises(ValueError): + LDClient(config, start_wait=0) + + +def test_override_source_start_failure_is_logged_and_the_client_runs_without_overrides(caplog): + # The SDK's own source never raises from start. A custom source could, and that must not + # keep the client from starting: the failure is logged, the source is closed, and the + # client behaves as it does without overrides. + source = MockOverrideSource(start_error=RuntimeError("cannot start")) + with make_uninitialized_client(source) as client: + assert source.close_count == 1 + assert errors_containing(caplog, 'Unable to start the override source; the client runs without overrides: cannot start') != [] + detail = client.variation_detail('any-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + # Closing the client does not close the failed source again. + assert source.close_count == 1 + + +def test_override_source_that_supplies_a_snapshot_and_then_fails_to_start_leaves_no_overrides(): + # The snapshot the source supplied before it failed is removed, so the client runs without + # overrides, as the log says, and a later snapshot from the closed source is ignored. + flags = {'overridden-flag': single_value_flag('overridden-flag', True)} + source = MockOverrideSource(flags=flags, start_error=RuntimeError("cannot start"), supply_before_start_error=True) + with make_uninitialized_client(source) as client: + detail = client.variation_detail('overridden-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert client.all_flags_state(user).valid is False + + source.set_overrides(flags) + assert client.variation('overridden-flag', user, False) is False + + +def test_snapshot_supplied_after_the_client_is_closed_is_ignored(): + source = MockOverrideSource() + client = make_uninitialized_client(source) + client.close() + + source.set_overrides({'late-flag': single_value_flag('late-flag', True)}) + + assert client._data_system.override_layer.get(FEATURES, 'late-flag') is None + + +def test_override_source_is_not_started_when_offline(caplog): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + datasystem = custom().overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, offline=True) + with LDClient(config, start_wait=0) as client: + assert source.start_count == 0 + assert client.variation('overridden-flag', user, False) is False + assert warnings_containing(caplog, 'Offline mode is enabled; the override source is not started and no overrides are served') != [] + + +def test_data_system_without_override_source_has_no_override_layer(): + datasystem = custom().synchronizers(HangingSynchronizer().builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with LDClient(config, start_wait=0) as client: + assert client._data_system.override_layer is None + detail = client.variation_detail('any-flag', user, False) + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + + +def test_legacy_data_system_has_no_override_layer(): + config = Config(sdk_key='SDK_KEY', offline=True) + with LDClient(config) as client: + assert isinstance(client._data_system, FDv1) + assert client._data_system.override_layer is None + + +def test_override_takes_precedence_over_launchdarkly_data(): + ld_flag = FlagBuilder('flag-precedence').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').build().to_json_dict() + source = MockOverrideSource(flags={'flag-precedence': single_value_flag('flag-precedence', 'override-value')}) + with make_initialized_client({'flag-precedence': ld_flag, 'flag-normal': normal}, source) as client: + detail = client.variation_detail('flag-precedence', user, 'default') + assert detail.value == 'override-value' + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + detail = client.variation_detail('flag-normal', user, 'default') + assert detail.value == 'normal-value' + assert detail.reason == {'kind': 'OFF'} + + +def test_full_flag_override_evaluates_targeting_rules_and_referenced_override_segment(): + segment_flag = FlagBuilder('flag-segment').version(1).on(True).off_variation(0).fallthrough_variation(0).variations('not-included', 'included').rules( + FlagRuleBuilder().id('segment-rule').variation(1).clauses(make_clause_matching_segment_key('overridden-segment')).build() + ).build().to_json_dict() + ld_segment = SegmentBuilder('overridden-segment').version(100).build().to_json_dict() + override_segment = SegmentBuilder('overridden-segment').version(101).included(user.key).build().to_json_dict() + source = MockOverrideSource(flags={'flag-segment': segment_flag}, segments={'overridden-segment': override_segment}) + with make_initialized_client({}, source, segments={'overridden-segment': ld_segment}) as client: + detail = client.variation_detail('flag-segment', user, 'default') + assert detail.value == 'included' + assert detail.reason == {'kind': 'RULE_MATCH', 'ruleIndex': 0, 'ruleId': 'segment-rule', 'overrideAffected': True} + + +def test_overridden_prerequisite_marks_the_dependent_flag(): + dependent = FlagBuilder('dependent').version(1).on(True).off_variation(0).fallthrough_variation(1).variations('prereq-failed', 'affected-value').prerequisite('prereq', 1).build().to_json_dict() + ld_prereq = FlagBuilder('prereq').version(100).on(False).off_variation(0).variations('a', 'b').build().to_json_dict() + override_prereq = FlagBuilder('prereq').version(200).on(True).off_variation(0).fallthrough_variation(1).variations('a', 'b').build().to_json_dict() + source = MockOverrideSource(flags={'prereq': override_prereq}) + with make_initialized_client({'dependent': dependent, 'prereq': ld_prereq}, source) as client: + detail = client.variation_detail('dependent', user, 'default') + assert detail.value == 'affected-value' + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + +def test_all_flags_state_contains_only_overrides_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + state = client.all_flags_state(user) + assert state.valid is True + assert state.to_values_map() == {'overridden-flag': True} + + +def test_all_flags_state_overrides_only_warning_is_logged_once(caplog): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + with caplog.at_level(logging.WARNING): + assert client.all_flags_state(user).valid is True + assert client.all_flags_state(user).valid is True + assert len(warnings_containing(caplog, 'Returning only flags from the override layer')) == 1 + + +def test_all_flags_state_is_invalid_when_not_initialized_and_override_layer_is_empty(): + source = MockOverrideSource() + with make_uninitialized_client(source) as client: + state = client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + + +def test_all_flags_state_reflects_overrides_when_initialized(): + ld_flag = FlagBuilder('flag-precedence').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').build().to_json_dict() + source = MockOverrideSource(flags={ + 'flag-precedence': single_value_flag('flag-precedence', 'override-value'), + 'override-only': single_value_flag('override-only', 'only-value'), + }) + with make_initialized_client({'flag-precedence': ld_flag, 'flag-normal': normal}, source) as client: + state = client.all_flags_state(user, with_reasons=True) + assert state.valid is True + assert state.to_values_map() == {'flag-precedence': 'override-value', 'flag-normal': 'normal-value', 'override-only': 'only-value'} + assert state.get_flag_reason('flag-precedence') == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert state.get_flag_reason('flag-normal') == {'kind': 'OFF'} + + +def test_flag_tracker_is_notified_of_override_changes(): + source = MockOverrideSource() + with make_uninitialized_client(source) as client: + changes: Queue = Queue() + client.flag_tracker.add_listener(lambda change: changes.put(change)) + + source.set_overrides({'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + change = changes.get(timeout=5) + assert isinstance(change, FlagChange) + assert change.key == 'overridden-flag' + + # An identical snapshot notifies nothing. + source.set_overrides({'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + with pytest.raises(Empty): + changes.get(timeout=0.3) + + source.set_overrides({}, {}) + assert changes.get(timeout=5).key == 'overridden-flag' + + +def test_launchdarkly_change_to_a_prerequisite_notifies_the_override_flag_that_depends_on_it(): + td = TestDataV2.data_source() + td.update(td.flag('ld-flag').on(True)) + override_flag = FlagBuilder('override-flag').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('ld-flag', 0).build().to_json_dict() + source = MockOverrideSource({'override-flag': override_flag}, {}) + datasystem = custom().synchronizers(td.builder).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with LDClient(config, start_wait=5) as client: + assert client.is_initialized() is True + changes: Queue = Queue() + client.flag_tracker.add_listener(lambda change: changes.put(change)) + + td.update(td.flag('ld-flag').on(False)) + keys = {changes.get(timeout=5).key, changes.get(timeout=5).key} + assert keys == {'ld-flag', 'override-flag'} + with pytest.raises(Empty): + changes.get(timeout=0.3) + + +def test_flag_value_change_listener_sees_override_value_changes(): + ld_flag = FlagBuilder('flag').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + source = MockOverrideSource() + with make_initialized_client({'flag': ld_flag}, source) as client: + changes: Queue = Queue() + client.flag_tracker.add_flag_value_change_listener('flag', user, lambda change: changes.put(change)) + + source.set_overrides({'flag': single_value_flag('flag', 'override-value')}, {}) + change = changes.get(timeout=5) + assert change.old_value == 'ld-value' + assert change.new_value == 'override-value' + + source.set_overrides({}, {}) + change = changes.get(timeout=5) + assert change.old_value == 'override-value' + assert change.new_value == 'ld-value' + + +def test_data_source_status_is_unaffected_by_overrides(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.is_initialized() is False + assert client.data_source_status_provider.status.state == DataSourceState.INITIALIZING + + +def raise_evaluation_failure(*args): + raise RuntimeError("evaluation failure") + + +def test_failed_evaluation_reason_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + # The evaluation read the flag definition before it failed, so the failure of the override + # flag is marked and the failure of the ordinary flag is not, in a single evaluation and in + # the all-flags state. + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_initialized_client({'plain-flag': single_value_flag('plain-flag', True)}, source) as client: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + assert client.variation_detail('overridden-flag', user, 'default') == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + assert client.variation_detail('plain-flag', user, 'default') == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + state = client.all_flags_state(user, with_reasons=True) + assert state.get_flag_reason('overridden-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert state.get_flag_reason('plain-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +def evaluation_events_by_key(client: LDClient) -> Dict[str, EventInputEvaluation]: + """The evaluation records the client handed to the event processor, keyed by flag key.""" + records = {} + processor: Any = client._event_processor + for event in processor._events: + if isinstance(event, EventInputEvaluation): + records[event.key] = event + return records + + +def test_override_evaluation_events_carry_override_affected_marking(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.variation('overridden-flag', user, False) is True + records = evaluation_events_by_key(client) + assert list(records.keys()) == ['overridden-flag'] + assert records['overridden-flag'].override_affected is True + + +def test_ordinary_evaluation_events_are_not_marked(): + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'other': single_value_flag('other', True)}) + with make_initialized_client({'flag-normal': normal}, source) as client: + client.variation('flag-normal', user, 'default') + records = evaluation_events_by_key(client) + assert records['flag-normal'].override_affected is False + assert records['flag-normal'].track_events is True + + +def tracked_bool_flag(key: str) -> FlagBuilder: + return FlagBuilder(key).version(100).variations(False, True).off_variation(0).fallthrough_variation(1).track_events(True) + + +def test_overridden_prerequisite_marks_the_dependent_evaluation_records(): + # top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden) + # --> plain-flag (LaunchDarkly) + # The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override. + ld_data = { + 'top-flag': tracked_bool_flag('top-flag').on(True).prerequisite('mid-flag', 1).prerequisite('plain-flag', 1).build().to_json_dict(), + 'mid-flag': tracked_bool_flag('mid-flag').on(True).prerequisite('leaf-flag', 1).build().to_json_dict(), + 'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict(), + 'leaf-flag': tracked_bool_flag('leaf-flag').on(False).build().to_json_dict(), + } + source = MockOverrideSource(flags={'leaf-flag': tracked_bool_flag('leaf-flag').on(True).build().to_json_dict()}) + with make_initialized_client(ld_data, source) as client: + detail = client.variation_detail('top-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + records = evaluation_events_by_key(client) + assert sorted(records.keys()) == ['leaf-flag', 'mid-flag', 'plain-flag', 'top-flag'] + assert records['top-flag'].override_affected is True + assert records['mid-flag'].override_affected is True + assert records['leaf-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['mid-flag'].prereq_of is not None and records['mid-flag'].prereq_of.key == 'top-flag' + assert records['leaf-flag'].prereq_of is not None and records['leaf-flag'].prereq_of.key == 'mid-flag' + assert records['leaf-flag'].reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert records['plain-flag'].reason == {'kind': 'FALLTHROUGH'} + + +def test_all_flags_state_turns_off_event_tracking_for_override_affected_flags(): + debug_until = int(time.time() * 1000) + 100000 + ld_data = { + 'plain-tracked': FlagBuilder('plain-tracked').version(1).on(False).off_variation(0).variations(True).track_events(True).debug_events_until_date(debug_until).build().to_json_dict(), + 'dependent-tracked': FlagBuilder('dependent-tracked').version(1).on(True).variations(False, True).fallthrough_variation(1).prerequisite('overridden-flag', 0).track_events(True).debug_events_until_date(debug_until).build().to_json_dict(), + } + # The overridden flag is on and serves variation 0, so the dependent flag's prerequisite passes. + overridden = FlagBuilder('overridden-flag').version(7).on(True).fallthrough_variation(0).variations(True).track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + with make_initialized_client(ld_data, source) as client: + state = client.all_flags_state(user, with_reasons=True) + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + + # A flag with no override keeps its tracking fields. + assert flags_state['plain-tracked']['trackEvents'] is True + assert flags_state['plain-tracked']['debugEventsUntilDate'] == debug_until + + # The overridden flag and the flag that depends on it stay in the state with their values + # and marked reasons, but with no tracking fields. + for key in ('overridden-flag', 'dependent-tracked'): + assert flags_state[key]['reason']['overrideAffected'] is True, key + assert 'trackEvents' not in flags_state[key], key + assert 'trackReason' not in flags_state[key], key + assert 'debugEventsUntilDate' not in flags_state[key], key + assert state.get_flag_value('overridden-flag') is True + assert state.get_flag_value('dependent-tracked') is True + assert flags_state['overridden-flag']['version'] == 7 + + +def test_all_flags_state_omits_details_of_override_affected_flags_when_details_only_for_tracked_flags(): + # With details only for tracked flags, an override-affected flag counts as untracked, so its + # version and reason are omitted like any other untracked flag, and its value stays. + overridden = FlagBuilder('overridden-flag').version(7).on(False).off_variation(0).variations(True).track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + with make_initialized_client({}, source) as client: + state = client.all_flags_state(user, with_reasons=True, details_only_for_tracked_flags=True) + flags_state = state.to_json_dict()['$flagsState'] + assert state.get_flag_value('overridden-flag') is True + assert 'version' not in flags_state['overridden-flag'] + assert 'reason' not in flags_state['overridden-flag'] + + +def test_wrong_type_result_of_overridden_flag_stays_marked(): + details = [] + + class CapturingHook(Hook): + @property + def metadata(self) -> Metadata: + return Metadata(name='capturing-hook') + + def before_evaluation(self, series_context: EvaluationSeriesContext, data: dict) -> dict: + return data + + def after_evaluation(self, series_context: EvaluationSeriesContext, data: dict, detail: EvaluationDetail) -> dict: + details.append(detail) + return data + + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', 'not-a-stage')}) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor, hooks=[CapturingHook()]) + with LDClient(config, start_wait=0) as client: + stage, _ = client.migration_variation('overridden-flag', user, Stage.OFF) + assert stage == Stage.OFF + assert len(details) == 1 + assert details[0].value == 'off' + assert details[0].reason == {'kind': 'ERROR', 'errorKind': 'WRONG_TYPE', 'overrideAffected': True} + + +def test_failed_evaluation_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + with make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) as client: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + # The failure of the override flag is marked. The failure of the ordinary flag is not. + detail = client.variation_detail('overridden-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + detail = client.variation_detail('plain-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + # The marked record produces no individual event. The ordinary record keeps its tracking. + records = evaluation_events_by_key(client) + assert records['overridden-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['plain-flag'].track_events is True + + +def test_all_flags_state_turns_off_event_tracking_for_an_override_flag_whose_evaluation_fails(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + with make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) as client: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + state = client.all_flags_state(user, with_reasons=True) + + # The failed override flag stays in the state with a marked reason and no tracking + # fields. The failed ordinary flag keeps its tracking fields. + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['overridden-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert 'trackEvents' not in flags_state['overridden-flag'] + assert flags_state['plain-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + assert flags_state['plain-flag']['trackEvents'] is True + + +def test_override_affected_evaluations_appear_only_in_summary_output(): + # End to end through the real event processor: the overridden flag requests individual + # feature events and debug events, and an ordinary flag requests feature events. + debug_until = int(time.time() * 1000) + 100000 + overridden = FlagBuilder('flag-tracked-override').version(300).on(False).off_variation(0).variations('override-value').track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'flag-tracked-override': overridden}) + mock_http = MockHttp() + + initializer = StaticInitializer({'flag-normal': normal}, {}) + datasystem = custom().initializers([initializer.builder]).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, diagnostic_opt_out=True, event_processor_class=lambda config: DefaultEventProcessor(config, mock_http)) + with LDClient(config, start_wait=5) as client: + assert client.is_initialized() is True + for _ in range(2): + assert client.variation('flag-tracked-override', user, 'default1') == 'override-value' + assert client.variation('flag-normal', user, 'default2') == 'normal-value' + client.flush() + client._event_processor._wait_until_inactive() + + assert mock_http.request_data is not None + output = json.loads(mock_http.request_data) + kinds = sorted(e['kind'] for e in output) + assert kinds == ['feature', 'index', 'summary'] + feature = [e for e in output if e['kind'] == 'feature'][0] + assert feature['key'] == 'flag-normal' + summary = [e for e in output if e['kind'] == 'summary'][0] + assert summary['features']['flag-tracked-override']['default'] == 'default1' + assert summary['features']['flag-tracked-override']['counters'] == [ + {'count': 2, 'value': 'override-value', 'variation': 0, 'version': 300, 'overrideAffected': True} + ] + assert summary['features']['flag-normal']['counters'] == [{'count': 1, 'value': 'normal-value', 'variation': 0, 'version': 100}] diff --git a/ldclient/testing/testdata/override-vectors/vectors.json b/ldclient/testing/testdata/override-vectors/vectors.json new file mode 100644 index 00000000..82dc62da --- /dev/null +++ b/ldclient/testing/testdata/override-vectors/vectors.json @@ -0,0 +1,316 @@ +{ + "schemaVersion": "0.4.0", + "vectors": [ + { + "description": "An override takes precedence over LaunchDarkly data for the same flag key", + "group": "precedence", + "launchDarklyData": { + "initialized": true, + "flags": { + "flag-a": { + "key": "flag-a", + "version": 7, + "on": true, + "variations": ["ld-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "salt": "salt-a" + } + }, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-a", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "override-value", + "variationIndex": 0, + "reason": { "kind": "FALLTHROUGH", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "An overridden flag is served when the SDK has not initialized from LaunchDarkly", + "group": "uninitialized", + "launchDarklyData": { + "initialized": false, + "flags": {}, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-a", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "override-value", + "variationIndex": 0, + "reason": { "kind": "FALLTHROUGH", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "A flag absent from the override layer is evaluated from LaunchDarkly data and is not marked", + "group": "passthrough", + "launchDarklyData": { + "initialized": true, + "flags": { + "flag-b": { + "key": "flag-b", + "version": 3, + "on": true, + "variations": ["off-value", "ld-value"], + "fallthrough": { "variation": 1 }, + "offVariation": 0, + "salt": "salt-b" + } + }, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-b", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "ld-value", + "variationIndex": 1, + "reason": { "kind": "FALLTHROUGH" }, + "summaryOverrideAffected": false + } + }, + { + "description": "A flag absent from the override layer still short-circuits when the SDK has not initialized", + "group": "passthrough", + "launchDarklyData": { + "initialized": false, + "flags": {}, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-b", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "fallback", + "variationIndex": null, + "reason": { "kind": "ERROR", "errorKind": "CLIENT_NOT_READY" } + } + }, + { + "description": "A full flag override with a matching targeting rule serves the rule's variation and is marked", + "group": "full-evaluation", + "launchDarklyData": { + "initialized": true, + "flags": {}, + "segments": {} + }, + "overrides": { + "flags": { + "flag-c": { + "key": "flag-c", + "version": 1, + "on": true, + "variations": ["default-value", "eu-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "rules": [ + { + "id": "rule-eu", + "variation": 1, + "clauses": [ + { + "contextKind": "user", + "attribute": "region", + "op": "in", + "values": ["eu"], + "negate": false + } + ] + } + ], + "salt": "salt-c" + } + } + }, + "evaluate": { + "flagKey": "flag-c", + "context": { "kind": "user", "key": "user-1", "region": "eu" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "eu-value", + "variationIndex": 1, + "reason": { "kind": "RULE_MATCH", "ruleIndex": 0, "ruleId": "rule-eu", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "An override flag resolves a referenced segment through the merged view while the SDK is uninitialized", + "group": "merged-view", + "launchDarklyData": { + "initialized": false, + "flags": {}, + "segments": {} + }, + "overrides": { + "flags": { + "flag-d": { + "key": "flag-d", + "version": 1, + "on": true, + "variations": ["default-value", "member-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "rules": [ + { + "id": "rule-seg", + "variation": 1, + "clauses": [ + { + "attribute": "", + "op": "segmentMatch", + "values": ["seg-1"], + "negate": false + } + ] + } + ], + "salt": "salt-d" + } + }, + "segments": { + "seg-1": { + "key": "seg-1", + "version": 1, + "included": ["user-1"], + "excluded": [], + "salt": "seg-salt-1" + } + } + }, + "evaluate": { + "flagKey": "flag-d", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "member-value", + "variationIndex": 1, + "reason": { "kind": "RULE_MATCH", "ruleIndex": 0, "ruleId": "rule-seg", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "A flag is marked overrideAffected when it matches via an overridden segment", + "group": "marking", + "launchDarklyData": { + "initialized": true, + "flags": { + "flag-f": { + "key": "flag-f", + "version": 4, + "on": true, + "variations": ["default-value", "member-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "rules": [ + { + "id": "rule-seg", + "variation": 1, + "clauses": [ + { + "attribute": "", + "op": "segmentMatch", + "values": ["seg-2"], + "negate": false + } + ] + } + ], + "salt": "salt-f" + } + }, + "segments": { + "seg-2": { + "key": "seg-2", + "version": 1, + "included": [], + "excluded": [], + "salt": "seg-salt-2" + } + } + }, + "overrides": { + "segments": { + "seg-2": { + "key": "seg-2", + "version": 2, + "included": ["user-1"], + "excluded": [], + "salt": "seg-salt-2" + } + } + }, + "evaluate": { + "flagKey": "flag-f", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "member-value", + "variationIndex": 1, + "reason": { "kind": "RULE_MATCH", "ruleIndex": 0, "ruleId": "rule-seg", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "A malformed override flag returns the caller default with an error reason and is marked overrideAffected", + "group": "error-handling", + "launchDarklyData": { + "initialized": true, + "flags": {}, + "segments": {} + }, + "overrides": { + "flags": { + "flag-e": { + "key": "flag-e", + "version": 1, + "on": true, + "variations": ["only-value"], + "fallthrough": { "variation": 5 }, + "offVariation": 0, + "salt": "salt-e" + } + } + }, + "evaluate": { + "flagKey": "flag-e", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "fallback", + "variationIndex": null, + "reason": { "kind": "ERROR", "errorKind": "MALFORMED_FLAG", "overrideAffected": true }, + "summaryOverrideAffected": true + } + } + ] +}