diff --git a/manifests/cpp.yml b/manifests/cpp.yml index fddd915329b..c4f65b2c0dc 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -189,6 +189,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: missing_feature (Automatic baggage propagation is not implemented) tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (cpp is not in scope for v1 agentless server SDK configuration-source coverage) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: missing_feature + tests/parametric/test_ffe/test_experimentation_opt_out.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=2.1.1' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index b1cc5602bd1..4fa82656e46 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1183,6 +1183,7 @@ manifest: component_version: <3.55.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=3.36.0' # Modified by easy win activation script tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature # Created by easy win activation script + tests/parametric/test_ffe/test_experimentation_opt_out.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=3.54.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/manifests/golang.yml b/manifests/golang.yml index 5ea6d7bde10..8f9d9423299 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -2170,6 +2170,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: missing_feature (DD_TRACE_PROPAGATION_EXTRACT_FIRST=true is unsupported; dd-trace-go panics when extraction fails) tests/parametric/test_ffe/test_configuration_sources.py: v2.11.0-dev # Easy win for all weblogs and version 2.10.1 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.5.0-dev + tests/parametric/test_ffe/test_experimentation_opt_out.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=2.10.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/manifests/java.yml b/manifests/java.yml index 7465820bd0a..9818dff0422 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4242,6 +4242,7 @@ manifest: : v1.65.0 tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: v1.65.0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.56.0 + tests/parametric/test_ffe/test_experimentation_opt_out.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=1.64.2' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index b9c3ae8a2bf..5531ddf12e6 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2658,6 +2658,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First::test_baggage: *ref_5_56_0 tests/parametric/test_ffe/test_configuration_sources.py: *ref_6_8_0 tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: *ref_5_75_0 + tests/parametric/test_ffe/test_experimentation_opt_out.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: # TODO: a lower version might be supported - declaration: missing_feature (dd-trace-js#8343) diff --git a/manifests/php.yml b/manifests/php.yml index 4d4f1a2f563..eb9a1d62eb1 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1598,6 +1598,7 @@ manifest: : missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_configuration_sources.py::Test_Feature_Flag_Configuration_Source_Warm_State_Preservation: missing_feature (FFL-2705 tracks PHP agentless configuration-source implementation; FFL-2731 tracks system-tests configuration-source contract) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v1.21.0-dev + tests/parametric/test_ffe/test_experimentation_opt_out.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=1.23.3' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/manifests/python.yml b/manifests/python.yml index 206a6a0988f..0a271cede8f 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2257,6 +2257,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: v2.20.0 tests/parametric/test_ffe/test_configuration_sources.py: v4.14.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v4.0.0 + tests/parametric/test_ffe/test_experimentation_opt_out.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=4.9.0' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 20f6ac2dc5d..1b881d2eb18 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2679,6 +2679,7 @@ manifest: tests/parametric/test_extract_behavior.py::Test_ExtractBehavior_Restart_With_Extract_First: incomplete_test_app (The parametric test app does not emit restart span links or preserve baggage) tests/parametric/test_ffe/test_configuration_sources.py: v2.46.0-dev tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: v2.23.0 + tests/parametric/test_ffe/test_experimentation_opt_out.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: missing_feature tests/parametric/test_headers_b3.py::Test_Headers_B3::test_headers_b3_extract_invalid: # Easy win for all weblogs and version 2.27.0 diff --git a/manifests/rust.yml b/manifests/rust.yml index e62fd119f0e..588de4ef484 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -258,6 +258,7 @@ manifest: tests/parametric/test_ffe/test_configuration_sources.py: irrelevant (rust is not in scope for v1 agentless server SDK configuration-source coverage) tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported tests/parametric/test_ffe/test_dynamic_evaluation.py::Test_Feature_Flag_Dynamic_Evaluation::test_ffe_flag_evaluation: missing_feature + tests/parametric/test_ffe/test_experimentation_opt_out.py: missing_feature tests/parametric/test_ffe/test_serial_id_metadata.py: missing_feature tests/parametric/test_ffe/test_span_enrichment.py: '>=0.5.3-dev+fd1307f' # TODO: a lower version might be supported tests/parametric/test_ffe/test_span_enrichment.py::Test_Span_Enrichment_Child_Span_Propagation: missing_feature diff --git a/tests/parametric/test_ffe/EXPERIMENTATION_OPT_OUT.md b/tests/parametric/test_ffe/EXPERIMENTATION_OPT_OUT.md new file mode 100644 index 00000000000..f8511fe3300 --- /dev/null +++ b/tests/parametric/test_ffe/EXPERIMENTATION_OPT_OUT.md @@ -0,0 +1,57 @@ +# Experimentation opt-out runtime contract + +This companion suite tests the real Java provider through its public evaluation API. +It checks values, allocation keys, variant keys, serial IDs, exposure classification, and hook metadata. +It does not infer success from an application default with an equal value. + +## Shared inputs + +The nested fixture snapshot comes from [shared fixture PR #48](https://github.com/DataDog/ffe-system-test-data/pull/48). +The exact commit is `5e2430d9ad36644e2cecb6a32ddb8d05d7c98ac6`. +`fixtures/experimentation-opt-out/SOURCE.json` records every original Git blob ID. +The loader verifies every blob and all 136 case IDs before using the snapshot. +Do not edit copied configurations or expectations. Update the shared repository and refresh the snapshot together. + +The runtime suite executes 116 cases in 11 configuration/provider-option groups. +It excludes 20 cases with fixed allocation time boundaries. +The Java SDK fixture adapter covers those cases with its fixed-clock test seam. +The dogfooding companion also runs them while their original time window is valid. +Changing generated dates would invalidate that proof. + +## Local candidate run + +Copy the agent, provider, and API JARs from the same candidate build into `binaries/`. +Do not combine a released agent with a candidate provider. +The Java image installer already accepts all three local JARs. + +```sh +PYTEST_XDIST_AUTO_NUM_WORKERS=1 TEST_LIBRARY=java ./run.sh PARAMETRIC \ + tests/parametric/test_ffe/test_experimentation_opt_out.py --runxfail +``` + +`run.sh` appends `-n auto`. The environment variable bounds that setting to one worker. +Use `--skip-parametric-build` only when the image and candidate artifacts did not change. +The manifest marks this unreleased behavior as `missing_feature` without inventing a release version. +The local command uses `--runxfail` so a candidate failure fails the run. +The suite enables the existing span-enrichment setting so ordinary allocation serial metadata is observable. +This does not enable experimentation opt-out. The public provider option controls that behavior separately. + +`ffe_start(provider_options=...)` activates the public provider options before evaluation. +A missing option method is a setup failure, not a behavioral negative control. +The baseline negative control uses the existing evaluation path without the new method. +Contexts remain per evaluation. The test does not replace server-global subject state. + +The Java controllers retain JSON integers that Jackson parses as `Long`. +They convert these values through OpenFeature's supported `Double` context type. +The bounded regression in `test_dynamic_evaluation.py` checks an exactly representable integer and both adjacent values. +This does not add exact 64-bit integer support to the SDK. + +## Evidence boundary + +These groups check assignment behavior only. The SDK shares exposure deduplication across providers. +Repeated fixture subjects can therefore suppress duplicate exposures correctly. +The dogfooding companion restarts its own app between the four independent telemetry cases. +It retains exact per-subject exposure and evaluation receipts, metrics, and correlated assignment traces. +It also checks overlapping evaluations and confirms that a skipped assignment does not reserve a deduplication key. + +The 20 Precompute protocol cases and nine client state sequences do not apply to this server runtime suite. diff --git a/tests/parametric/test_ffe/experimentation_contract.py b/tests/parametric/test_ffe/experimentation_contract.py new file mode 100644 index 00000000000..a7349574aa6 --- /dev/null +++ b/tests/parametric/test_ffe/experimentation_contract.py @@ -0,0 +1,63 @@ +"""Read an immutable shared fixture snapshot and compare actual provider details.""" + +import hashlib +import json +from pathlib import Path +from typing import Any + +FIXTURES = Path(__file__).parent / "fixtures" / "experimentation-opt-out" + + +def load_contract() -> tuple[dict[str, Any], dict[str, Any], list[dict[str, Any]]]: + """Verify every source blob before loading the shared contract.""" + source = json.loads((FIXTURES / "SOURCE.json").read_text()) + assert source["commit"] == "5e2430d9ad36644e2cecb6a32ddb8d05d7c98ac6" + for name, expected in source["blobs"].items(): + contents = (FIXTURES / name).read_bytes() + actual = hashlib.sha1(b"blob " + str(len(contents)).encode() + b"\0" + contents).hexdigest() + assert actual == expected, f"Modified shared fixture: {name}" + manifest = json.loads((FIXTURES / "manifest.json").read_text()) + configurations = { + item["id"]: json.loads((FIXTURES / item["path"]).read_text()) for item in manifest["configurations"] + } + cases: list[dict[str, Any]] = [] + for group in manifest["evaluationCases"]: + group_cases = json.loads((FIXTURES / group["path"]).read_text()) + assert [case["id"] for case in group_cases] == group["ids"] + cases.extend(group_cases) + assert len(cases) == manifest["expectedCounts"]["evaluations"] == 136 + return manifest, configurations, cases + + +def has_time_boundary(configuration: dict[str, Any], case: dict[str, Any]) -> bool: + """Time-gated cases require fixed-clock SDK tests or a guarded live runtime.""" + flag = configuration["flags"][case["evaluation"]["flag"]] + return any(allocation.get("startAt") or allocation.get("endAt") for allocation in flag["allocations"]) + + +def assert_details(result: dict[str, Any], case: dict[str, Any]) -> None: + """Require assignment identity and metadata, not only an equal value.""" + expected = case["expected"] + assert result["value"] == expected["value"], (case["id"], result) + assignment = expected["assignment"] + if assignment is None: + assert not result.get("allocationKey"), (case["id"], result) + assert not result.get("variant"), (case["id"], result) + assert result.get("serialId") is None, (case["id"], result) + if expected["outcome"] != "no-assignment": + assert result.get("errorCode"), (case["id"], result) + return + assert not result.get("errorCode"), (case["id"], result) + assert result.get("reason"), (case["id"], result) + assert result.get("allocationKey") == assignment["allocationKey"], (case["id"], result) + assert result.get("variant") == assignment["variationKey"], (case["id"], result) + assert result.get("serialId") == assignment["serialId"], (case["id"], result) + metadata = result.get("flagMetadata", {}) + assert metadata.get("__dd_do_log") is assignment["doLog"], (case["id"], result) + expected_features = { + "__dd_feature." + feature["key"]: feature["value"] + for feature in assignment.get("features", []) + if "HOOK" in feature["destinations"] + } + actual_features = {key: value for key, value in metadata.items() if key.startswith("__dd_feature.")} + assert actual_features == expected_features, (case["id"], actual_features, expected_features) diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/README.md b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/README.md new file mode 100644 index 00000000000..3930cbe6747 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/README.md @@ -0,0 +1,288 @@ +# Experimentation opt-out contract, version 1 + +This suite defines subject experimentation opt-out across providers, evaluators, +the Precompute API, and precompute clients. It includes **reviewed contract +inputs** and separately identified **compiler-generated artifacts**. Neither +fixture validation nor compiler generation proves that an SDK implements opt-out. + +The contract uses the proposed winner-logging change: ordinary winner rollouts +have doLog=false. Active experiments, generated holdouts, and experiment-backed +canaries and bandits have doLog=true. The compiler must preserve ordinary +fallbacks when an experimental allocation can be skipped. + +## Customer contract + +The customer enables enableExperimentationOptOutParameter once on the provider +or native client. The customer supplies dd_experimentation_opt_out in the context. + +| Provider activation | Attribute | Effective opt-out | +| --- | --- | --- | +| Absent or false | Any value | False | +| True | Boolean true | True | +| True | Missing, false, null, string, number, array, or object | False | + +Read the typed boolean before converting context attributes to strings. +Do not coerce the string "true" or the number 1. Do not change the targeting key. + +When effective opt-out is true, skip each allocation with doLog=true before +enrollment or arm selection. Continue ordinary allocations in their original +order. Do not force experimental control, false, or the application default. + +Exclusion covers both experimental control and treatment. It also covers both +holdout groups and experiments that enroll 100% of subjects. Ordinary targeting, +percentage rollouts, and completed winners remain available. + +Keep salts, shards, and weights unchanged. The eligible population retains +configured probabilities, not exact finite-population headcounts. Customers +must not derive eligibility from treatment assignment or treatment effects. + +## Contents and exact coverage + +The manifest lists every configuration, case ID, and sequence. Consumers must +read it. Unlisted files, missing cases, duplicate IDs, and wrong counts fail validation. + +| Artifact | Purpose | +| --- | --- | +| configurations/supported.json | Ordinary rules, experiments, fallbacks, winner states, and holdout states. | +| configurations/legacy-without-capability.json | Old winner logging and a pruned fallback. Exclusion cannot safely use this artifact. | +| configurations/missing-dolog.json | Deliberately malformed marked JSON. Exclusion cannot treat a missing classification as ordinary. | +| configurations/compiler-*.json | Seven verbatim lifecycle outputs from the actual compiler with the proposed changes. | +| cases/activation.json | Explicit activation and strict boolean handling. | +| cases/evaluation.json | Allocation order, every randomized arm, identity, ordinary rollouts, winners, canaries, and bandits. | +| cases/holdouts.json | Both groups before shipment, after a non-control winner, and after a control winner. | +| cases/compatibility.json | Old or malformed configurations do not silently admit excluded subjects. | +| cases/telemetry.json | All four combinations of experimentation and evaluation-telemetry opt-out. | +| cases/compiler-generated.json | Thirty independently authored evaluations against generated lifecycle outputs. | +| compiler-generated/ | Paired protobuf bytes, generation provenance, observations, and independent shard vectors. | +| protocol/request-and-response.json | Actual request and response envelopes, malformed options, and missing acknowledgements. | +| sequences/ | Late responses, failed refresh, deduplication, logout, and cache identity. | + +There are 136 evaluation cases, 20 protocol cases, and 9 client sequences. +The manifest defines 32 reviewed-input eligible-parity cases, 15 generated-input +eligible-parity cases, and a concurrent-server +check. Those checks reuse existing case IDs; they do not add new cases. + +### Configuration provenance + +The original three configurations specify expected behavior. Their provenance +is reviewed-contract-input. They must not be described as compiler-generated. + +Seven additional configurations have compiler-generated provenance. The manifest +pairs each JSON file with its deterministic protobuf bytes and checks SHA256 for +both. It also checks the generation record and exact artifact enumeration. + +The generation receipt is in compiler-generated/PROVENANCE.md. It records fixed +inputs, the command, exact source commit, generator hash, and successful invocation. +Generation ran from that clean commit. All generated artifacts match its output +byte for byte. + +The generated scenarios cover active 50/50, active 100% single-arm, both completed +winners, and active/completed experiments combined with holdouts. Their JSON and +protobuf files were copied verbatim. Do not edit them to satisfy a consumer test; +regenerate them with the actual compiler and update the recorded provenance. + +The generator's eligible examples in compiler-scenarios.json are observations +from its Go evaluator. They are **not** the shared expected-result oracle. +The thirty shared cases use different subject IDs and explicit expected identities. +Independent MD5 vectors establish holdout membership, group, and experimental arm. +The validator checks the vectors, not an alternate targeting evaluator. + +The validator verifies protobuf bytes and pairing, but does not decode protobuf. +Consumers with protobuf support must decode the paired file and run the same +generated-input evaluation cases. Other adapters run the paired JSON. + +Compiler tests must construct real lifecycle models and run the actual compiler. +Compare the relevant output with this contract. Establish these properties: + +- Winner selection preserves the experiment link but clears winner doLog. +- Both generated holdout groups retain logging and analysis metadata. +- A non-control winner changes holdout exposure contexts as specified upstream. +- A control winner does not incorrectly start primary holdout analysis. +- Full-population experiments retain ordinary fallbacks and required variants. +- JSON, protobuf, and archived representations preserve the selected semantics. + +UFC does not contain the retained experiment link. Fixtures cannot prove that +the database or compiler preserved it. Compiler lifecycle tests must supply +source provenance and a generator revision. + +Canary and bandit cases cover their allocation classification after compilation. +They do not model bandit learning, action selection, or canary monitoring. + +## Evaluation case format + +Each case supplies providerOptions, typed context, an evaluation, and explicit +expected results. The expected.assignment identifies the allocation and split. +Never infer identity from value equality. + +The expected.outcome is a test-harness category, **not a new SDK error code**: + +| Outcome | Required behavior | +| --- | --- | +| assignment | Return the selected assignment and its exact identity. | +| no-assignment | Preserve the existing no-assignment result; the provider returns the caller default. | +| incompatible-configuration | Do not apply opt-out to old compiler semantics. Use the existing error/default path. | +| invalid-configuration | Do not classify malformed allocations as ordinary. Use the existing error/default path. | + +Fixtures distinguish experimental control, ordinary fallback, and caller default. +One case gives control and fallback equal values but different allocation identities. +A boolean case requires an ordinary true, so forcing false cannot pass. + +The wireReason is the selected split's reason in UFC or the precomputed payload. +Compiler and wire decoders must preserve it. Existing evaluator APIs can expose +different reason mappings. Compare each adapter's observable reason with its +unchanged eligible baseline. Do not invent an OPTED_OUT reason or silently +normalize existing reason behavior. + +### Holdout identity + +Route and search flags share the holdout key and cohort hash. They have distinct +generated allocation keys and serial IDs. Before shipment, both groups can +return the same value. Their group features and serial IDs still differ. + +After a non-control winner ships, eligible subjects retain holdout assignments. +Excluded subjects skip the holdout and receive the ordinary winner. The +primary-analysis marker remains false for a control winner. + +Assert assignment identity, serial ID, and the complete feature list. +Do not use holdout.should_include_in_holdout_analysis to decide eligibility. + +## Compatibility and Precompute + +Supported configurations declare: + + "evaluationCapabilities": ["experimentation-opt-out-v1"] + +This marker attests to compiler winner classification, fallback preservation, +and holdout semantics. It is not an SDK version declaration. + +With opt-out requested, a missing marker cannot produce a successful excluded +assignment. Ordinary requests retain existing behavior. Marked JSON also requires +explicit boolean doLog values for the evaluated flag's allocations. For protobuf, +the marker attests to compiler semantics; scalar presence is not required. +Archives must preserve the marker. + +Ordinary missing-doLog behavior is not a universal assignment case. Current +Go and Rust JSON evaluators default it to true. Java and JavaScript default it +to false. Each adapter must run an ordinary missing-doLog regression against +its unchanged baseline and preserve that assignment and logging behavior. +Do not normalize these defaults as part of opt-out. The shared +missing-dolog-excluded case requires rejection when exclusion is requested. + +The SDK sends the effective decision separately from targeting attributes: + + data.attributes.evaluation_options.experimentation_opt_out = true + +False requests omit evaluation_options. The edge also accepts an explicit +boolean false or an empty options object. A targeting attribute alone cannot +activate exclusion. Non-boolean wire values are rejected. + +A successful excluded response contains the same path with boolean true. +It means the evaluator applied exclusion using compatible configuration, not an +unconditional echo. Old edge code can ignore unknown fields, so HTTP success +alone does not prove support. + +An excluded client rejects missing, false, malformed, or contradictory +acknowledgements. Ordinary clients accept legacy responses without this field. +A present acknowledgement must agree with the requested option. + +Protocol cases test decoding and acceptance. Their fixed bodies do not prove +that the edge evaluated their corresponding request. + +The option does not change the shared UFC cache key. Assignment caches and +request generations include subject, targeting context, environment, and +effective opt-out. A mismatch cannot publish or restore an old assignment. + +## Telemetry and exposures + +Cases reuse the existing precomputed EvaluationContext and Expectations schema +definitions and their ordered event matchers. Each case starts with clean state +and performs one evaluation: + +- A selected doLog=true assignment produces one exposure. +- A skipped allocation produces no exposure and no deduplication entry. +- Evaluation telemetry stays enabled unless its independent control applies. +- Evaluation-telemetry opt-out does **not** suppress exposures or change assignment. +- noUnmatchedEvents=true requires exact observation, with a positive emission control. + +Core evaluators return exposure eligibility; they do not necessarily emit events. +Core tests assert classification and identity. Runtime tests inspect actual +exposure events, evaluation events, and supported trace, metric, or RUM +annotations. A silent collector is not proof of suppression. + +## Consumer responsibilities + +| Consumer | Required checks | +| --- | --- | +| Compiler | Real lifecycle generation, compatibility, wire fields, preserved fallback variants, and format parity. | +| Core evaluator | Every evaluation case's effective option, outcome, identity, value, logging, serial ID, and features. | +| Provider | Strict activation through the public API; never inject the expected effective option into the provider. | +| Edge | Request parsing, both evaluator paths, acknowledgement correctness, and buffered/streamed output parity. | +| Precompute SDK | Public activation, outgoing options, response acceptance, typed getters, events, and all state sequences. | +| Server provider | Concurrent subject isolation on one provider using manifest case IDs and repetition count. | + +Core adapters can calculate the effective option from fixture inputs because +activation is outside the core. Report that limited scope. A core pass is not +a provider activation pass. + +Precompute adapters can encode the explicitly expected assignment as a mock +response. This is a mechanical field mapping, not another evaluator. Separately, +run the real edge against the same configuration and case. Compare its response +with the expected assignment. + +Run every case in manifest.invariants.eligibleParityCaseIds and +manifest.compilerGeneration.eligibleParityCaseIds with activation enabled and disabled. +Use the same configuration and context. Require identical identity, value, serial ID, +features, exposure eligibility, and existing reason. The fixed MD5 vectors make +known arm selection independently inspectable. MD5 is the existing bucketing +contract, not a security mechanism. + +For concurrent server cases, share one provider and run both contexts concurrently +for concurrentServerRepetitions iterations. Each result must match its own case. +Do not mutate global server context. Single-subject clients use transition sequences. + +Test all supported representations. Backend Rust tests JSON and archived rkyv. +JavaScript tests JSON and protobuf. Report unsupported formats explicitly. + +### Sequence actions + +Start with empty assignments, cache, and exposure deduplication state. Use the +sequence flag and defaultValue. The default SDK environment is production; +configure and restart can specify another environment. + +| Action | Test operation | +| --- | --- | +| configure | Initialize the real provider or native client with the specified options. | +| set-context | Change subject or attributes; assert immediately published state. | +| start-request | Start or observe refresh; keep its response pending and inspect outgoing options. | +| deliver-response | Encode the referenced assignment, set the requested acknowledgement, and deliver that pending response. | +| fail-request | Fail the pending request; assert which compatible state remains. | +| evaluate | Call the typed getter and inspect incremental exposure and evaluation-telemetry effects. | +| persist-cache | Wait for the real persistent write and retain its bytes under the test cache ID. | +| restart | Recreate the client with that saved cache and supplied identity. Inspect state before any new network response. | + +An applied acknowledgement matches the pending request's effective option. +Missing removes the field. False supplies literal false. A null published case +means no assignment is available, not an assignment with a null value. + +expectedExposureCount is incremental, not cumulative. Evaluation telemetry can +aggregate. Flush and compare logical evaluation counts with the existing platform adapter. + +## Validation and reports + +Install jsonschema, then run: + + python3 ci/validate-experimentation-opt-out.py + python3 ci/test-validate-experimentation-opt-out.py + +Validation checks schemas, duplicate JSON keys, exact manifest coverage, +references, strict activation, assignment consistency, metadata, protocol +expectations, and independent bucketing vectors. It does not evaluate targeting +rules or claim SDK conformance. + +Consumer reports must include fixture Git SHA, compiler/evaluator revision, +manifest counts, executed IDs, failed IDs, unsupported IDs, representation, and +raw request/event receipts where relevant. Unsupported or skipped cases do not +count as passes. Keep runtime evidence separate from fixture validation. + +Legacy suites remain unchanged. Old readers must not discover this directory +and silently ignore its provider, protocol, or sequence fields. diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/SOURCE.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/SOURCE.json new file mode 100644 index 00000000000..5ae439ccd73 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/SOURCE.json @@ -0,0 +1,45 @@ +{ + "repository": "https://github.com/DataDog/ffe-system-test-data", + "commit": "5e2430d9ad36644e2cecb6a32ddb8d05d7c98ac6", + "suite": "experimentation-opt-out/v1", + "blobs": { + "README.md": "3930cbe674786d719478a225cf09fe7317565efe", + "cases/activation.json": "8dcc68dc8f68affbc7d9b9618fb7864f42fcac18", + "cases/compatibility.json": "f798e603c10ca62dba39619145af3b8b1f1e387d", + "cases/compiler-generated.json": "f1fa893a48c4849abcc7b69b4c6f4370a345cf90", + "cases/evaluation.json": "d39a73f3467cfc29ad39f07706104967060848b3", + "cases/holdouts.json": "e68ed824e86e89ff46be92671a632cafdb3631a2", + "cases/telemetry.json": "76707f0dd9b2e1421dd8c46e3c5858beeede44ec", + "compiler-generated/PROVENANCE.md": "2b2b64724739d7357571d9ae91f7edffb1d56ace", + "compiler-generated/compiler-active-100-single-arm.pb": "205844eed3169a205cfd39100f635ff20b99ba9c", + "compiler-generated/compiler-active-50-50.pb": "36e59b85e36f5475dc82c62adc86dc507724cbc3", + "compiler-generated/compiler-completed-control.pb": "7a16a11331fc74faa0a8bf34efe44c4f3b9409f1", + "compiler-generated/compiler-completed-treatment.pb": "e870641b5efb2bd7ffe4f5cae145c1ec2ddb830c", + "compiler-generated/compiler-holdout-active.pb": "5762bd280d6eac15e3e94b89a2377f2bcca0fbba", + "compiler-generated/compiler-holdout-completed-control.pb": "1933c34a7fc9e33dc132e76868a461e7e765214a", + "compiler-generated/compiler-holdout-completed-treatment.pb": "d9206f2d2fb4dde5a1f0b4eaf6c8be44cdc756cd", + "compiler-generated/compiler-scenarios.json": "86a3873894237f3115e7c4256f8cc2d1f88e57ac", + "compiler-generated/hash-vectors.json": "7d3e6d680ea8e34a51294e98a55ef442d7b2fdf4", + "configurations/compiler-active-100-single-arm.json": "c25e4dc9443f32d6de874ab06de6aafbebc73ca9", + "configurations/compiler-active-50-50.json": "db28d65eb79cd3b3ec648a163abb1cd16031ad39", + "configurations/compiler-completed-control.json": "72d71e2685108acb2a4d56ab52bfa293c39a2c70", + "configurations/compiler-completed-treatment.json": "9964a7ed70aa803c550a52a94d674b2c05cf56d9", + "configurations/compiler-holdout-active.json": "5257eb0f2764cefb0f6cadea459cb7ed5420693c", + "configurations/compiler-holdout-completed-control.json": "0a62aadfc870c3c4f7717c92d026967a2eb29a3b", + "configurations/compiler-holdout-completed-treatment.json": "f3618fbe8717d6f2450805c1ed5c9ba45e6eb464", + "configurations/legacy-without-capability.json": "b00537add2492c6e7e92f552867d8ebf5df22bb7", + "configurations/missing-dolog.json": "f918a8735f368f17a7c8b59d8e9d50451f6387ee", + "configurations/supported.json": "94548b07be62bdb42c0e429a43412bf6f4f4bd52", + "manifest.json": "d232ece46e3aef4b540b8407f7e33a9617229c18", + "protocol/request-and-response.json": "3b5013d87e2232ece5e4ec430b4e291415eda883", + "sequences/cache-environment-identity.json": "97ec40a29a9218e0dde1f60dd5643ad693929947", + "sequences/cache-participation-and-subject-identity.json": "73d87d3716f65b3bf267c061d570a0b17a45a43c", + "sequences/cache-targeting-context-identity.json": "40f1c1ba2f06d40e3d71cde31d6e9516d74e6f26", + "sequences/failed-refresh-after-opt-out.json": "9dccd25e15c6525d6e816eecdd5fb44a030ed23d", + "sequences/late-response-after-opt-out.json": "54e5814f99fe7a432b443f2b534b03305730ca41", + "sequences/logout-rejects-authenticated-response.json": "de07b2eec74d53369d61b3e06e8c550db88cf044", + "sequences/missing-ack-never-publishes.json": "49074fecc1bdd1f353e18015f965458cfbd87967", + "sequences/opt-out-does-not-create-exposure-dedup.json": "5f8c95286c9c3f402fb1da3c882c40231526d810", + "sequences/same-state-compatible-cache-on-failure.json": "8181019674f07352e918ad20c7e4ba9646bf48de" + } +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/activation.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/activation.json new file mode 100644 index 00000000000..8dcc68dc8f6 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/activation.json @@ -0,0 +1,1247 @@ +[ + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-disabled-missing", + "description": "activation disabled missing", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": false + }, + "context": { + "targetingKey": "subject-2", + "attributes": {} + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-disabled-false", + "description": "activation disabled false", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": false + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-disabled-true", + "description": "activation disabled true", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": false + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-disabled-null", + "description": "activation disabled null", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": false + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": null + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-disabled-string-true", + "description": "activation disabled string true", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": false + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": "true" + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-disabled-string-false", + "description": "activation disabled string false", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": false + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": "false" + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-disabled-number-one", + "description": "activation disabled number one", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": false + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": 1 + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-disabled-number-zero", + "description": "activation disabled number zero", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": false + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": 0 + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-disabled-array", + "description": "activation disabled array", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": false + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": [ + true + ] + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-disabled-object", + "description": "activation disabled object", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": false + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": { + "value": true + } + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-disabled-nested", + "description": "activation disabled nested", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": false + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "preferences": { + "dd_experimentation_opt_out": true + } + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-enabled-missing", + "description": "activation enabled missing", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": {} + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-enabled-false", + "description": "activation enabled false", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-enabled-true", + "description": "activation enabled true", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-enabled-null", + "description": "activation enabled null", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": null + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-enabled-string-true", + "description": "activation enabled string true", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": "true" + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-enabled-string-false", + "description": "activation enabled string false", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": "false" + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-enabled-number-one", + "description": "activation enabled number one", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": 1 + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-enabled-number-zero", + "description": "activation enabled number zero", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": 0 + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-enabled-array", + "description": "activation enabled array", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": [ + true + ] + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-enabled-object", + "description": "activation enabled object", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": { + "value": true + } + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-enabled-nested", + "description": "activation enabled nested", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "preferences": { + "dd_experimentation_opt_out": true + } + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation", + "strict-boolean" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "activation-unconfigured-true", + "description": "activation unconfigured true", + "configuration": "supported", + "providerOptions": {}, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "activation" + ] + } +] diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/compatibility.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/compatibility.json new file mode 100644 index 00000000000..f798e603c10 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/compatibility.json @@ -0,0 +1,234 @@ +[ + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "legacy-ordinary", + "description": "legacy ordinary", + "configuration": "legacy-without-capability", + "providerOptions": { + "enableExperimentationOptOutParameter": false + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compatibility" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "legacy-excluded", + "description": "legacy excluded", + "configuration": "legacy-without-capability", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "incompatible-configuration", + "assignment": null, + "value": "application-default", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compatibility" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "missing-dolog-excluded", + "description": "missing dolog excluded", + "configuration": "missing-dolog", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "invalid-configuration", + "assignment": null, + "value": "application-default", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compatibility", + "missing-dolog" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "legacy-winner-not-misclassified", + "description": "legacy winner not misclassified", + "configuration": "legacy-without-capability", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "winner-treatment", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "incompatible-configuration", + "assignment": null, + "value": "application-default", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compatibility", + "winner" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "legacy-pruned-fallback-not-used", + "description": "legacy pruned fallback not used", + "configuration": "legacy-without-capability", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "full-enrollment-experiment", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "incompatible-configuration", + "assignment": null, + "value": "application-default", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compatibility", + "optimizer" + ] + } +] diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/compiler-generated.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/compiler-generated.json new file mode 100644 index 00000000000..f1fa893a48c --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/compiler-generated.json @@ -0,0 +1,1749 @@ +[ + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-active-50-50-0001-eligible", + "description": "Independent contract expectation for active-50-50: preserve the selected assignment and exposure context.", + "configuration": "compiler-active-50-50", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-active-50-50-0001-excluded", + "description": "Independent contract expectation for active-50-50: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-active-50-50", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "allocation-default-7feefdf76b9e", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-active-50-50-0005-eligible", + "description": "Independent contract expectation for active-50-50: preserve the selected assignment and exposure context.", + "configuration": "compiler-active-50-50", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0005", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 102, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-active-50-50-0005-excluded", + "description": "Independent contract expectation for active-50-50: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-active-50-50", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0005", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "allocation-default-7feefdf76b9e", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-active-100-single-arm-0001-eligible", + "description": "Independent contract expectation for active-100-single-arm: preserve the selected assignment and exposure context.", + "configuration": "compiler-active-100-single-arm", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 102, + "wireReason": "STATIC" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-active-100-single-arm-0001-excluded", + "description": "Independent contract expectation for active-100-single-arm: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-active-100-single-arm", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "allocation-default-7feefdf76b9e", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-completed-control-0001-eligible", + "description": "Independent contract expectation for completed-control: preserve the selected assignment and exposure context.", + "configuration": "compiler-completed-control", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": false, + "serialId": 101, + "wireReason": "STATIC" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-completed-control-0001-excluded", + "description": "Independent contract expectation for completed-control: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-completed-control", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": false, + "serialId": 101, + "wireReason": "STATIC" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-completed-treatment-0001-eligible", + "description": "Independent contract expectation for completed-treatment: preserve the selected assignment and exposure context.", + "configuration": "compiler-completed-treatment", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": false, + "serialId": 102, + "wireReason": "STATIC" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-completed-treatment-0001-excluded", + "description": "Independent contract expectation for completed-treatment: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-completed-treatment", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": false, + "serialId": 102, + "wireReason": "STATIC" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-active-0006-eligible", + "description": "Independent contract expectation for holdout-active: preserve the selected assignment and exposure context.", + "configuration": "compiler-holdout-active", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0006", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment-holdout-quarterly", + "variationKey": "control", + "doLog": true, + "serialId": 201, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-active-0006-excluded", + "description": "Independent contract expectation for holdout-active: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-holdout-active", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0006", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "allocation-default-7feefdf76b9e", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-active-0000-eligible", + "description": "Independent contract expectation for holdout-active: preserve the selected assignment and exposure context.", + "configuration": "compiler-holdout-active", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0000", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment-holdout-quarterly", + "variationKey": "control", + "doLog": true, + "serialId": 202, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-active-0000-excluded", + "description": "Independent contract expectation for holdout-active: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-holdout-active", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0000", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "allocation-default-7feefdf76b9e", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-active-0001-eligible", + "description": "Independent contract expectation for holdout-active: preserve the selected assignment and exposure context.", + "configuration": "compiler-holdout-active", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-active-0001-excluded", + "description": "Independent contract expectation for holdout-active: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-holdout-active", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "allocation-default-7feefdf76b9e", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-active-0005-eligible", + "description": "Independent contract expectation for holdout-active: preserve the selected assignment and exposure context.", + "configuration": "compiler-holdout-active", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0005", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 102, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-active-0005-excluded", + "description": "Independent contract expectation for holdout-active: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-holdout-active", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0005", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "allocation-default-7feefdf76b9e", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-completed-control-0006-eligible", + "description": "Independent contract expectation for holdout-completed-control: preserve the selected assignment and exposure context.", + "configuration": "compiler-holdout-completed-control", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0006", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment-holdout-quarterly", + "variationKey": "control", + "doLog": true, + "serialId": 201, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-completed-control-0006-excluded", + "description": "Independent contract expectation for holdout-completed-control: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-holdout-completed-control", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0006", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": false, + "serialId": 101, + "wireReason": "DEFAULT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-completed-control-0000-eligible", + "description": "Independent contract expectation for holdout-completed-control: preserve the selected assignment and exposure context.", + "configuration": "compiler-holdout-completed-control", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0000", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment-holdout-quarterly", + "variationKey": "control", + "doLog": true, + "serialId": 202, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-completed-control-0000-excluded", + "description": "Independent contract expectation for holdout-completed-control: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-holdout-completed-control", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0000", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": false, + "serialId": 101, + "wireReason": "DEFAULT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-completed-control-0001-eligible", + "description": "Independent contract expectation for holdout-completed-control: preserve the selected assignment and exposure context.", + "configuration": "compiler-holdout-completed-control", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": false, + "serialId": 101, + "wireReason": "DEFAULT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-completed-control-0001-excluded", + "description": "Independent contract expectation for holdout-completed-control: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-holdout-completed-control", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": false, + "serialId": 101, + "wireReason": "DEFAULT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-completed-treatment-0006-eligible", + "description": "Independent contract expectation for holdout-completed-treatment: preserve the selected assignment and exposure context.", + "configuration": "compiler-holdout-completed-treatment", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0006", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment-holdout-quarterly", + "variationKey": "control", + "doLog": true, + "serialId": 301, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-completed-treatment-0006-excluded", + "description": "Independent contract expectation for holdout-completed-treatment: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-holdout-completed-treatment", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0006", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": false, + "serialId": 102, + "wireReason": "DEFAULT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-completed-treatment-0000-eligible", + "description": "Independent contract expectation for holdout-completed-treatment: preserve the selected assignment and exposure context.", + "configuration": "compiler-holdout-completed-treatment", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0000", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment-holdout-quarterly", + "variationKey": "treatment", + "doLog": true, + "serialId": 302, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-completed-treatment-0000-excluded", + "description": "Independent contract expectation for holdout-completed-treatment: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-holdout-completed-treatment", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0000", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": false, + "serialId": 102, + "wireReason": "DEFAULT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-completed-treatment-0001-eligible", + "description": "Independent contract expectation for holdout-completed-treatment: preserve the selected assignment and exposure context.", + "configuration": "compiler-holdout-completed-treatment", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": false, + "serialId": 102, + "wireReason": "DEFAULT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout", + "generated-eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "generated-holdout-completed-treatment-0001-excluded", + "description": "Independent contract expectation for holdout-completed-treatment: skip all experimental allocations and select ordinary delivery.", + "configuration": "compiler-holdout-completed-treatment", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "contract-subject-0001", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "planner", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": false, + "serialId": 102, + "wireReason": "DEFAULT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "compiler-generated", + "lifecycle", + "holdout" + ] + } +] diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/evaluation.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/evaluation.json new file mode 100644 index 00000000000..d39a73f3467 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/evaluation.json @@ -0,0 +1,2825 @@ +[ + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "excluded-control", + "description": "excluded control", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "two-arm", + "control" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "excluded-treatment", + "description": "excluded treatment", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "two-arm", + "treatment" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "excluded-third-arm", + "description": "excluded third arm", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-1", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "three-arm" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "excluded-full-enrollment", + "description": "excluded full enrollment", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "full-enrollment-experiment", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "full-enrollment", + "optimizer" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "excluded-partial-enrollment", + "description": "excluded partial enrollment", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "partial-experiment", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "partial-enrollment" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-outside-enrollment", + "description": "eligible outside enrollment", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "partial-experiment", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "partial-enrollment" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "ordinary-beta-before-experiment", + "description": "ordinary beta before experiment", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true, + "beta": "yes" + } + }, + "evaluation": { + "flag": "ordered-ordinary", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-beta", + "allocationKey": "beta", + "variationKey": "beta", + "doLog": false, + "serialId": 901, + "wireReason": "TARGETING_MATCH" + }, + "value": "ordinary-beta", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "allocation-order", + "ordinary" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "ordinary-targeted-after-experiment", + "description": "ordinary targeted after experiment", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true, + "tier": "gold" + } + }, + "evaluation": { + "flag": "ordered-ordinary", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-gold", + "allocationKey": "gold", + "variationKey": "gold", + "doLog": false, + "serialId": 902, + "wireReason": "TARGETING_MATCH" + }, + "value": "ordinary-gold", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "allocation-order", + "ordinary" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "ordinary-default-after-experiment", + "description": "ordinary default after experiment", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "ordered-ordinary", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "allocation-order", + "ordinary" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "skip-multiple-experiments", + "description": "skip multiple experiments", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "multiple-experiments", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "allocation-order" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "equal-value-ordinary-fallback", + "description": "equal value ordinary fallback", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "equal-value-fallback", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "same-value", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "same-value", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "allocation-identity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "equal-value-experimental-control", + "description": "equal value experimental control", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "equal-value-fallback", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "same-value", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "same-value", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "allocation-identity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "no-ordinary-assignment", + "description": "no ordinary assignment", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "no-fallback", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "no-assignment", + "assignment": null, + "value": "application-default", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "no-assignment" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "ordinary-percentage-included", + "description": "ordinary percentage included", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "ordinary-percentage", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "ordinary-rollout", + "variationKey": "treatment", + "doLog": false, + "serialId": 701, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "ordinary", + "partial-enrollment" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "ordinary-percentage-not-included", + "description": "ordinary percentage not included", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "ordinary-percentage", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "ordinary", + "partial-enrollment" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "boolean-fallback-is-true", + "description": "boolean fallback is true", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "boolean-fallback", + "variationType": "BOOLEAN", + "defaultValue": false, + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": true, + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": true, + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "no-forced-false" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "winner-control-eligible", + "description": "winner control eligible", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "winner-control", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "winner-rollout", + "variationKey": "control", + "doLog": false, + "serialId": 801, + "wireReason": "STATIC" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "winner", + "lifecycle" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "winner-control-excluded", + "description": "winner control excluded", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "winner-control", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "winner-rollout", + "variationKey": "control", + "doLog": false, + "serialId": 801, + "wireReason": "STATIC" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "winner", + "lifecycle" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "winner-treatment-eligible", + "description": "winner treatment eligible", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "winner-treatment", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "winner-rollout", + "variationKey": "treatment", + "doLog": false, + "serialId": 802, + "wireReason": "STATIC" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "winner", + "lifecycle" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "winner-treatment-excluded", + "description": "winner treatment excluded", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "winner-treatment", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "winner-rollout", + "variationKey": "treatment", + "doLog": false, + "serialId": 802, + "wireReason": "STATIC" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "winner", + "lifecycle" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "excluded-canary", + "description": "excluded canary", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "canary", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "canary", + "classification" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "excluded-bandit", + "description": "excluded bandit", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "bandit", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "bandit", + "classification" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-0", + "description": "eligible two arm subject 0", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 102, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-0", + "description": "eligible three arm subject 0", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 202, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-1", + "description": "eligible two arm subject 1", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-1", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 102, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-1", + "description": "eligible three arm subject 1", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-1", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-third", + "allocationKey": "experiment", + "variationKey": "third", + "doLog": true, + "serialId": 203, + "wireReason": "SPLIT" + }, + "value": "experimental-third", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-2", + "description": "eligible two arm subject 2", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-2", + "description": "eligible three arm subject 2", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 201, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-3", + "description": "eligible two arm subject 3", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-3", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-3", + "description": "eligible three arm subject 3", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-3", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 201, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-4", + "description": "eligible two arm subject 4", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-4", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-4", + "description": "eligible three arm subject 4", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-4", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 201, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-5", + "description": "eligible two arm subject 5", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-5", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 102, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-5", + "description": "eligible three arm subject 5", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-5", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 202, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-6", + "description": "eligible two arm subject 6", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-6", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-6", + "description": "eligible three arm subject 6", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-6", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 201, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-7", + "description": "eligible two arm subject 7", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-7", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-7", + "description": "eligible three arm subject 7", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-7", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 201, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-8", + "description": "eligible two arm subject 8", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-8", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 102, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-8", + "description": "eligible three arm subject 8", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-8", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-third", + "allocationKey": "experiment", + "variationKey": "third", + "doLog": true, + "serialId": 203, + "wireReason": "SPLIT" + }, + "value": "experimental-third", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-9", + "description": "eligible two arm subject 9", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-9", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 102, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-9", + "description": "eligible three arm subject 9", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-9", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 202, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-10", + "description": "eligible two arm subject 10", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-10", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 102, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-10", + "description": "eligible three arm subject 10", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-10", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-third", + "allocationKey": "experiment", + "variationKey": "third", + "doLog": true, + "serialId": 203, + "wireReason": "SPLIT" + }, + "value": "experimental-third", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-11", + "description": "eligible two arm subject 11", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-11", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-11", + "description": "eligible three arm subject 11", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-11", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 202, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-12", + "description": "eligible two arm subject 12", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-12", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-12", + "description": "eligible three arm subject 12", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-12", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 202, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-13", + "description": "eligible two arm subject 13", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-13", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-13", + "description": "eligible three arm subject 13", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-13", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 202, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-14", + "description": "eligible two arm subject 14", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-14", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-14", + "description": "eligible three arm subject 14", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-14", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "experiment", + "variationKey": "treatment", + "doLog": true, + "serialId": 202, + "wireReason": "SPLIT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-two-arm-subject-15", + "description": "eligible two arm subject 15", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-15", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "eligible-three-arm-subject-15", + "description": "eligible three arm subject 15", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-15", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "three-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 201, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "eligible-parity" + ] + } +] diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/holdouts.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/holdouts.json new file mode 100644 index 00000000000..e68ed824e86 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/holdouts.json @@ -0,0 +1,1340 @@ +[ + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-before-route-status-quo-eligible", + "description": "holdout before route status quo eligible", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "holdout-before-route", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "generated-holdout-route", + "variationKey": "control", + "doLog": true, + "serialId": 401, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "before", + "status-quo", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-before-route-status-quo-excluded", + "description": "holdout before route status quo excluded", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "holdout-before-route", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "before", + "status-quo", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-before-route-all-shipped-eligible", + "description": "holdout before route all shipped eligible", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "holdout-before-route", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "generated-holdout-route", + "variationKey": "control", + "doLog": true, + "serialId": 402, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "before", + "all-shipped", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-before-route-all-shipped-excluded", + "description": "holdout before route all shipped excluded", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "holdout-before-route", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "before", + "all-shipped", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-before-search-status-quo-eligible", + "description": "holdout before search status quo eligible", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "holdout-before-search", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "generated-holdout-search", + "variationKey": "control", + "doLog": true, + "serialId": 501, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "before", + "status-quo", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-before-search-status-quo-excluded", + "description": "holdout before search status quo excluded", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "holdout-before-search", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "before", + "status-quo", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-before-search-all-shipped-eligible", + "description": "holdout before search all shipped eligible", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "holdout-before-search", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "generated-holdout-search", + "variationKey": "control", + "doLog": true, + "serialId": 502, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "before", + "all-shipped", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-before-search-all-shipped-excluded", + "description": "holdout before search all shipped excluded", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "holdout-before-search", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "before", + "all-shipped", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-after-route-status-quo-eligible", + "description": "holdout after route status quo eligible", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "holdout-after-route", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "generated-holdout-route", + "variationKey": "control", + "doLog": true, + "serialId": 411, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "after", + "status-quo", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-after-route-status-quo-excluded", + "description": "holdout after route status quo excluded", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "holdout-after-route", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "winner-rollout", + "variationKey": "treatment", + "doLog": false, + "serialId": 802, + "wireReason": "DEFAULT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "after", + "status-quo", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-after-route-all-shipped-eligible", + "description": "holdout after route all shipped eligible", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "holdout-after-route", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "generated-holdout-route", + "variationKey": "treatment", + "doLog": true, + "serialId": 412, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "after", + "all-shipped", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-after-route-all-shipped-excluded", + "description": "holdout after route all shipped excluded", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "holdout-after-route", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "winner-rollout", + "variationKey": "treatment", + "doLog": false, + "serialId": 802, + "wireReason": "DEFAULT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "after", + "all-shipped", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-after-search-status-quo-eligible", + "description": "holdout after search status quo eligible", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "holdout-after-search", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "generated-holdout-search", + "variationKey": "control", + "doLog": true, + "serialId": 511, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "after", + "status-quo", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-after-search-status-quo-excluded", + "description": "holdout after search status quo excluded", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "holdout-after-search", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "winner-rollout", + "variationKey": "treatment", + "doLog": false, + "serialId": 852, + "wireReason": "DEFAULT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "after", + "status-quo", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-after-search-all-shipped-eligible", + "description": "holdout after search all shipped eligible", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "holdout-after-search", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "generated-holdout-search", + "variationKey": "treatment", + "doLog": true, + "serialId": 512, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "after", + "all-shipped", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-after-search-all-shipped-excluded", + "description": "holdout after search all shipped excluded", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "holdout-after-search", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-treatment", + "allocationKey": "winner-rollout", + "variationKey": "treatment", + "doLog": false, + "serialId": 852, + "wireReason": "DEFAULT" + }, + "value": "experimental-treatment", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "after", + "all-shipped", + "shared-holdout" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-control-winner-route-status-quo-eligible", + "description": "A control winner does not begin primary holdout analysis. Opt-out skips the holdout and reaches the ordinary control winner.", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "holdout-control-winner-route", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "generated-holdout-control-winner", + "variationKey": "control", + "doLog": true, + "serialId": 601, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "before", + "status-quo", + "shared-holdout", + "control-winner", + "lifecycle" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-control-winner-route-status-quo-excluded", + "description": "A control winner does not begin primary holdout analysis. Opt-out skips the holdout and reaches the ordinary control winner.", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "holdout-control-winner-route", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "winner-rollout", + "variationKey": "control", + "doLog": false, + "serialId": 803, + "wireReason": "DEFAULT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "before", + "status-quo", + "shared-holdout", + "control-winner", + "lifecycle" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-control-winner-route-all-shipped-eligible", + "description": "A control winner does not begin primary holdout analysis. Opt-out skips the holdout and reaches the ordinary control winner.", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "evaluation": { + "flag": "holdout-control-winner-route", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "generated-holdout-control-winner", + "variationKey": "control", + "doLog": true, + "serialId": 602, + "wireReason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "before", + "all-shipped", + "shared-holdout", + "control-winner", + "lifecycle" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "holdout-control-winner-route-all-shipped-excluded", + "description": "A control winner does not begin primary holdout analysis. Opt-out skips the holdout and reaches the ordinary control winner.", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation": { + "flag": "holdout-control-winner-route", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "winner-rollout", + "variationKey": "control", + "doLog": false, + "serialId": 803, + "wireReason": "DEFAULT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "holdout", + "before", + "all-shipped", + "shared-holdout", + "control-winner", + "lifecycle" + ] + } +] diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/telemetry.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/telemetry.json new file mode 100644 index 00000000000..76707f0dd9b --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/cases/telemetry.json @@ -0,0 +1,210 @@ +[ + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "independent-experiment-false-telemetry-false", + "description": "independent experiment false telemetry false", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true, + "enableEvaluationTelemetryOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false, + "dd_evaluation_telemetry_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "independent-telemetry" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "independent-experiment-false-telemetry-true", + "description": "independent experiment false telemetry true", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true, + "enableEvaluationTelemetryOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false, + "dd_evaluation_telemetry_opt_out": true + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": false, + "outcome": "assignment", + "assignment": { + "value": "experimental-control", + "allocationKey": "experiment", + "variationKey": "control", + "doLog": true, + "serialId": 101, + "wireReason": "SPLIT" + }, + "value": "experimental-control", + "evaluationTelemetryEnabled": false, + "requestEvaluationOptions": null + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [ + { + "_count": 1 + } + ], + "evaluationEvents": [] + }, + "tags": [ + "independent-telemetry" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "independent-experiment-true-telemetry-false", + "description": "independent experiment true telemetry false", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true, + "enableEvaluationTelemetryOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true, + "dd_evaluation_telemetry_opt_out": false + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": true, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [ + { + "_count": 1 + } + ] + }, + "tags": [ + "independent-telemetry" + ] + }, + { + "schemaVersion": 1, + "kind": "evaluation", + "id": "independent-experiment-true-telemetry-true", + "description": "independent experiment true telemetry true", + "configuration": "supported", + "providerOptions": { + "enableExperimentationOptOutParameter": true, + "enableEvaluationTelemetryOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true, + "dd_evaluation_telemetry_opt_out": true + } + }, + "evaluation": { + "flag": "two-arm", + "variationType": "STRING", + "defaultValue": "application-default", + "now": "2026-10-16T12:00:00Z" + }, + "expected": { + "effectiveExperimentationOptOut": true, + "outcome": "assignment", + "assignment": { + "value": "ordinary-baseline", + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "doLog": false, + "serialId": 900, + "wireReason": "DEFAULT" + }, + "value": "ordinary-baseline", + "evaluationTelemetryEnabled": false, + "requestEvaluationOptions": { + "experimentation_opt_out": true + } + }, + "expectations": { + "noUnmatchedEvents": true, + "exposures": [], + "evaluationEvents": [] + }, + "tags": [ + "independent-telemetry" + ] + } +] diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/PROVENANCE.md b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/PROVENANCE.md new file mode 100644 index 00000000000..2b2b6472473 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/PROVENANCE.md @@ -0,0 +1,52 @@ +# Compiler-generated experimentation opt-out fixtures + +These files come from the production UFC compiler with fixed test inputs. +They are separate from the database lifecycle receipts, which use runtime identifiers and timestamps. + +- Source repository: `ddoghq/dd-source`. +- Source commit: `654b5a5a34c9a74f223a8215b1202f47b9d3e0d8`. +- Generator: `domains/ffe/libs/flagging/ufc/experimentation_opt_out_test.go`, `TestCompilerExperimentationOptOutGeneratedFixtures`. +- Generator source SHA256: `4438ef0520626e75184690ffbcb0d37fcb55697b7ada220219befd51aeb2554d`. +- Successful invocation: `f1f65df6-3d14-4286-8d5a-6ce12973d504`. +- Fixed compilation time: `2026-10-16T12:00:00Z`. + +Standalone generation command: + +```sh +bzl test //domains/ffe/libs/flagging/ufc:go_default_test --test_filter=TestCompilerExperimentationOptOutGeneratedFixtures --test_output=errors +``` + +The recorded invocation ran from the clean source commit and passed in 4.090 seconds. +All seven JSON/protobuf pairs and compiler-scenarios.json match the checked-in artifacts byte for byte. + +## Fixed inputs + +| Input | Value | +| --- | --- | +| Organization | `123` | +| Flag ID / key / type | `opt-out-flag` / `planner` / string | +| Environment ID / name | `opt-out-env` / `experimentation-opt-out` | +| Allocation ID / key | `experiment-allocation` / `experiment` | +| Retained experiment ID | `experiment` | +| Variants | `control=experimental-control`, `treatment=experimental-treatment`, `baseline=ordinary-baseline` | +| Active two-arm weights | `50/50` | +| Active single-arm weights | `0/100`; experiment remains active | +| Completed weights | `100/0` or `0/100`; `ExperimentRolloutComplete=true` | +| Ordinary default | `baseline` | +| Standard serials | `101=control`, `102=treatment`, `900=default` | +| Holdout | `quarterly`, traffic `0.2`, status quo `control` | +| Holdout inclusion | `2026-10-16T09:00:00Z` through `2026-10-16T10:00:00Z` | +| Holdout analysis | `2026-10-16T11:00:00Z` through `2026-10-17T12:00:00Z` | +| Holdout contexts | `pre-ship`, `post-ship` | +| Holdout serials | pre: `201/202`; post: `301/302`, status-quo/all-shipped order | + +The generator uses the existing compiler shard sizes and salts. It does not change them. +The control winner does not represent a shipped change. Its holdout keeps pre-ship serials and a false analysis flag. +The treatment winner uses post-ship holdout serials and a true analysis flag. + +`compiler-scenarios.json` records the intended ordinary fallback and illustrative eligible assignment identities. +Its eligible examples come from the existing Go evaluator. Shared fixture checks must independently verify the hash vectors. +The generator does not claim that the new opt-out evaluator is implemented. That work belongs to stage 3. + +The separate lifecycle integration test starts and concludes an experiment, then reloads through `GetFlagsWithRelations`. +It proves that production loads the completion state without injecting it into the compiler. diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-active-100-single-arm.pb b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-active-100-single-arm.pb new file mode 100644 index 00000000000..205844eed31 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-active-100-single-arm.pb @@ -0,0 +1,6 @@ + +À£ÈÖexperimentation-opt-outY +plannerN" + +experiment"f ("( +allocation-default-7feefdf76b9e"„ *baseline*ordinary-baseline* treatment*experimental-treatmentZexperimentation-opt-out-v1 \ No newline at end of file diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-active-50-50.pb b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-active-50-50.pb new file mode 100644 index 00000000000..36e59b85e36 Binary files /dev/null and b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-active-50-50.pb differ diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-completed-control.pb b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-completed-control.pb new file mode 100644 index 00000000000..7a16a11331f --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-completed-control.pb @@ -0,0 +1,5 @@ + +À£ÈÖexperimentation-opt-out% +planner" + +experiment"e *control*experimental-controlZexperimentation-opt-out-v1 \ No newline at end of file diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-completed-treatment.pb b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-completed-treatment.pb new file mode 100644 index 00000000000..e870641b5ef --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-completed-treatment.pb @@ -0,0 +1,5 @@ + +À£ÈÖexperimentation-opt-out% +planner" + +experiment"f * treatment*experimental-treatmentZexperimentation-opt-out-v1 \ No newline at end of file diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-holdout-active.pb b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-holdout-active.pb new file mode 100644 index 00000000000..5762bd280d6 Binary files /dev/null and b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-holdout-active.pb differ diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-holdout-completed-control.pb b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-holdout-completed-control.pb new file mode 100644 index 00000000000..1933c34a7fc Binary files /dev/null and b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-holdout-completed-control.pb differ diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-holdout-completed-treatment.pb b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-holdout-completed-treatment.pb new file mode 100644 index 00000000000..d9206f2d2fb Binary files /dev/null and b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-holdout-completed-treatment.pb differ diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-scenarios.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-scenarios.json new file mode 100644 index 00000000000..86a38738942 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/compiler-scenarios.json @@ -0,0 +1,426 @@ +[ + { + "name": "active-50-50", + "singleArm": false, + "holdout": false, + "configuration": "compiler-active-50-50.json", + "eligible": [ + { + "targetingKey": "athlete-0000", + "allocationKey": "experiment", + "variationKey": "control", + "value": "experimental-control", + "serialId": 101, + "doLog": true, + "features": null + }, + { + "targetingKey": "athlete-0002", + "allocationKey": "experiment", + "variationKey": "treatment", + "value": "experimental-treatment", + "serialId": 102, + "doLog": true, + "features": null + } + ], + "optedOut": { + "targetingKey": "", + "allocationKey": "allocation-default-7feefdf76b9e", + "variationKey": "baseline", + "value": "ordinary-baseline", + "serialId": 900, + "doLog": false, + "features": null + }, + "allocationOrder": [ + "experiment", + "allocation-default-7feefdf76b9e" + ], + "explanation": "Skip the active experiment before randomization; deliver the ordinary baseline." + }, + { + "name": "active-100-single-arm", + "singleArm": true, + "holdout": false, + "configuration": "compiler-active-100-single-arm.json", + "eligible": [ + { + "targetingKey": "athlete-0000", + "allocationKey": "experiment", + "variationKey": "treatment", + "value": "experimental-treatment", + "serialId": 102, + "doLog": true, + "features": null + } + ], + "optedOut": { + "targetingKey": "", + "allocationKey": "allocation-default-7feefdf76b9e", + "variationKey": "baseline", + "value": "ordinary-baseline", + "serialId": 900, + "doLog": false, + "features": null + }, + "allocationOrder": [ + "experiment", + "allocation-default-7feefdf76b9e" + ], + "explanation": "Full enrollment and a single arm do not make an active experiment ordinary delivery." + }, + { + "name": "completed-control", + "winner": "control", + "singleArm": false, + "holdout": false, + "configuration": "compiler-completed-control.json", + "eligible": [ + { + "targetingKey": "athlete-0000", + "allocationKey": "experiment", + "variationKey": "control", + "value": "experimental-control", + "serialId": 101, + "doLog": false, + "features": null + } + ], + "optedOut": { + "targetingKey": "", + "allocationKey": "experiment", + "variationKey": "control", + "value": "experimental-control", + "serialId": 101, + "doLog": false, + "features": null + }, + "allocationOrder": [ + "experiment" + ], + "explanation": "The completed control winner remains available without experiment exposures." + }, + { + "name": "completed-treatment", + "winner": "treatment", + "singleArm": false, + "holdout": false, + "configuration": "compiler-completed-treatment.json", + "eligible": [ + { + "targetingKey": "athlete-0000", + "allocationKey": "experiment", + "variationKey": "treatment", + "value": "experimental-treatment", + "serialId": 102, + "doLog": false, + "features": null + } + ], + "optedOut": { + "targetingKey": "", + "allocationKey": "experiment", + "variationKey": "treatment", + "value": "experimental-treatment", + "serialId": 102, + "doLog": false, + "features": null + }, + "allocationOrder": [ + "experiment" + ], + "explanation": "The completed treatment winner remains available without experiment exposures." + }, + { + "name": "holdout-active", + "singleArm": false, + "holdout": true, + "configuration": "compiler-holdout-active.json", + "eligible": [ + { + "targetingKey": "athlete-0000", + "allocationKey": "experiment", + "variationKey": "control", + "value": "experimental-control", + "serialId": 101, + "doLog": true, + "features": null + }, + { + "targetingKey": "athlete-0001", + "allocationKey": "experiment-holdout-quarterly", + "variationKey": "control", + "value": "experimental-control", + "serialId": 201, + "doLog": true, + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + { + "targetingKey": "athlete-0003", + "allocationKey": "experiment", + "variationKey": "treatment", + "value": "experimental-treatment", + "serialId": 102, + "doLog": true, + "features": null + }, + { + "targetingKey": "athlete-0006", + "allocationKey": "experiment-holdout-quarterly", + "variationKey": "control", + "value": "experimental-control", + "serialId": 202, + "doLog": true, + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + } + ], + "optedOut": { + "targetingKey": "", + "allocationKey": "allocation-default-7feefdf76b9e", + "variationKey": "baseline", + "value": "ordinary-baseline", + "serialId": 900, + "doLog": false, + "features": null + }, + "allocationOrder": [ + "experiment-holdout-quarterly", + "experiment", + "allocation-default-7feefdf76b9e" + ], + "explanation": "Skip both holdout groups and the active experiment; deliver the ordinary baseline." + }, + { + "name": "holdout-completed-control", + "winner": "control", + "singleArm": false, + "holdout": true, + "configuration": "compiler-holdout-completed-control.json", + "eligible": [ + { + "targetingKey": "athlete-0000", + "allocationKey": "experiment", + "variationKey": "control", + "value": "experimental-control", + "serialId": 101, + "doLog": false, + "features": null + }, + { + "targetingKey": "athlete-0001", + "allocationKey": "experiment-holdout-quarterly", + "variationKey": "control", + "value": "experimental-control", + "serialId": 201, + "doLog": true, + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + { + "targetingKey": "athlete-0006", + "allocationKey": "experiment-holdout-quarterly", + "variationKey": "control", + "value": "experimental-control", + "serialId": 202, + "doLog": true, + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + } + ], + "optedOut": { + "targetingKey": "", + "allocationKey": "experiment", + "variationKey": "control", + "value": "experimental-control", + "serialId": 101, + "doLog": false, + "features": null + }, + "allocationOrder": [ + "experiment-holdout-quarterly", + "experiment" + ], + "explanation": "Skip the holdout and deliver the ordinary control winner; unchanged holdout groups keep pre-ship serials." + }, + { + "name": "holdout-completed-treatment", + "winner": "treatment", + "singleArm": false, + "holdout": true, + "configuration": "compiler-holdout-completed-treatment.json", + "eligible": [ + { + "targetingKey": "athlete-0000", + "allocationKey": "experiment", + "variationKey": "treatment", + "value": "experimental-treatment", + "serialId": 102, + "doLog": false, + "features": null + }, + { + "targetingKey": "athlete-0001", + "allocationKey": "experiment-holdout-quarterly", + "variationKey": "control", + "value": "experimental-control", + "serialId": 301, + "doLog": true, + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + }, + { + "targetingKey": "athlete-0006", + "allocationKey": "experiment-holdout-quarterly", + "variationKey": "treatment", + "value": "experimental-treatment", + "serialId": 302, + "doLog": true, + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + } + ], + "optedOut": { + "targetingKey": "", + "allocationKey": "experiment", + "variationKey": "treatment", + "value": "experimental-treatment", + "serialId": 102, + "doLog": false, + "features": null + }, + "allocationOrder": [ + "experiment-holdout-quarterly", + "experiment" + ], + "explanation": "Skip the holdout and deliver the ordinary treatment winner; eligible holdouts keep post-ship serials." + } +] diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/hash-vectors.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/hash-vectors.json new file mode 100644 index 00000000000..7d3e6d680ea --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/compiler-generated/hash-vectors.json @@ -0,0 +1,109 @@ +{ + "schemaVersion": 1, + "kind": "independent-shard-vectors", + "algorithm": "MD5 UTF-8 salt-targetingKey; first four bytes as unsigned big-endian integer modulo totalShards. Computed independently with Python hashlib, not a Datadog evaluator.", + "vectors": [ + { + "targetingKey": "contract-subject-0000", + "salt": "planner-experiment-variant", + "totalShards": 8192, + "md5": "e39414496e299eb70ee759f2a7207070", + "shard": 5193 + }, + { + "targetingKey": "contract-subject-0000", + "salt": "quarterly-holdout-traffic", + "totalShards": 1048576, + "md5": "593136accf9296a58bfc750b73f86b3a", + "shard": 79532 + }, + { + "targetingKey": "contract-subject-0000", + "salt": "quarterly-holdout-split", + "totalShards": 8192, + "md5": "24bc109583e09b9b3e1d6f47531ee4fd", + "shard": 4245 + }, + { + "targetingKey": "contract-subject-0001", + "salt": "planner-experiment-variant", + "totalShards": 8192, + "md5": "4e8f826f26fb3334ed4bd10e4ddb5f81", + "shard": 623 + }, + { + "targetingKey": "contract-subject-0001", + "salt": "quarterly-holdout-traffic", + "totalShards": 1048576, + "md5": "0e446b84c1679347890594b4bd1d64f1", + "shard": 289668 + }, + { + "targetingKey": "contract-subject-0001", + "salt": "quarterly-holdout-split", + "totalShards": 8192, + "md5": "690a4d2a935cb0ce5026878a0f364c88", + "shard": 3370 + }, + { + "targetingKey": "contract-subject-0005", + "salt": "planner-experiment-variant", + "totalShards": 8192, + "md5": "1180b0af3f603af395b85978075c62f3", + "shard": 4271 + }, + { + "targetingKey": "contract-subject-0005", + "salt": "quarterly-holdout-traffic", + "totalShards": 1048576, + "md5": "821c18a08ed0d52a418318d0c6c013c8", + "shard": 792736 + }, + { + "targetingKey": "contract-subject-0005", + "salt": "quarterly-holdout-split", + "totalShards": 8192, + "md5": "c26725b7d31575c43bbfe66f53902139", + "shard": 1463 + }, + { + "targetingKey": "contract-subject-0006", + "salt": "planner-experiment-variant", + "totalShards": 8192, + "md5": "26040e3bef8aeb6e639639c56451d8a8", + "shard": 3643 + }, + { + "targetingKey": "contract-subject-0006", + "salt": "quarterly-holdout-traffic", + "totalShards": 1048576, + "md5": "c061339433789c86027ffa700d193546", + "shard": 78740 + }, + { + "targetingKey": "contract-subject-0006", + "salt": "quarterly-holdout-split", + "totalShards": 8192, + "md5": "4b1926ba846ffeb8ee19020dc8feca8e", + "shard": 1722 + } + ], + "subjectRoles": [ + { + "targetingKey": "contract-subject-0000", + "role": "Holdout all_shipped: traffic 79532 < 209715; split 4245 >= 4096. Outside holdout, the experiment selects treatment." + }, + { + "targetingKey": "contract-subject-0001", + "role": "Outside holdout: traffic 289668 >= 209715. Experiment control: variant shard 623 < 4096." + }, + { + "targetingKey": "contract-subject-0005", + "role": "Outside holdout: traffic 792736 >= 209715. Experiment treatment: variant shard 4271 >= 4096." + }, + { + "targetingKey": "contract-subject-0006", + "role": "Holdout status_quo: traffic 78740 < 209715; split 1722 < 4096." + } + ] +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-active-100-single-arm.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-active-100-single-arm.json new file mode 100644 index 00000000000..c25e4dc9443 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-active-100-single-arm.json @@ -0,0 +1,56 @@ +{ + "createdAt": "2026-10-16T12:00:00Z", + "format": "SERVER", + "observeFullEvaluationData": false, + "evaluationCapabilities": [ + "experimentation-opt-out-v1" + ], + "environment": { + "name": "experimentation-opt-out" + }, + "flags": { + "planner": { + "key": "planner", + "enabled": true, + "variationType": "STRING", + "variations": { + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "shards": [], + "variationKey": "treatment", + "serialId": 102, + "reason": "STATIC" + } + ], + "doLog": true + }, + { + "key": "allocation-default-7feefdf76b9e", + "rules": [], + "splits": [ + { + "shards": [], + "variationKey": "baseline", + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + } + } +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-active-50-50.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-active-50-50.json new file mode 100644 index 00000000000..db28d65eb79 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-active-50-50.json @@ -0,0 +1,88 @@ +{ + "createdAt": "2026-10-16T12:00:00Z", + "format": "SERVER", + "observeFullEvaluationData": false, + "evaluationCapabilities": [ + "experimentation-opt-out-v1" + ], + "environment": { + "name": "experimentation-opt-out" + }, + "flags": { + "planner": { + "key": "planner", + "enabled": true, + "variationType": "STRING", + "variations": { + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + }, + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "shards": [ + { + "salt": "planner-experiment-variant", + "ranges": [ + { + "start": 0, + "end": 4096 + } + ], + "totalShards": 8192 + } + ], + "variationKey": "control", + "serialId": 101, + "reason": "SPLIT" + }, + { + "shards": [ + { + "salt": "planner-experiment-variant", + "ranges": [ + { + "start": 4096, + "end": 8192 + } + ], + "totalShards": 8192 + } + ], + "variationKey": "treatment", + "serialId": 102, + "reason": "SPLIT" + } + ], + "doLog": true + }, + { + "key": "allocation-default-7feefdf76b9e", + "rules": [], + "splits": [ + { + "shards": [], + "variationKey": "baseline", + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + } + } +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-completed-control.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-completed-control.json new file mode 100644 index 00000000000..72d71e26851 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-completed-control.json @@ -0,0 +1,39 @@ +{ + "createdAt": "2026-10-16T12:00:00Z", + "format": "SERVER", + "observeFullEvaluationData": false, + "evaluationCapabilities": [ + "experimentation-opt-out-v1" + ], + "environment": { + "name": "experimentation-opt-out" + }, + "flags": { + "planner": { + "key": "planner", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "shards": [], + "variationKey": "control", + "serialId": 101, + "reason": "STATIC" + } + ], + "doLog": false + } + ] + } + } +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-completed-treatment.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-completed-treatment.json new file mode 100644 index 00000000000..9964a7ed70a --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-completed-treatment.json @@ -0,0 +1,39 @@ +{ + "createdAt": "2026-10-16T12:00:00Z", + "format": "SERVER", + "observeFullEvaluationData": false, + "evaluationCapabilities": [ + "experimentation-opt-out-v1" + ], + "environment": { + "name": "experimentation-opt-out" + }, + "flags": { + "planner": { + "key": "planner", + "enabled": true, + "variationType": "STRING", + "variations": { + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "shards": [], + "variationKey": "treatment", + "serialId": 102, + "reason": "STATIC" + } + ], + "doLog": false + } + ] + } + } +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-holdout-active.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-holdout-active.json new file mode 100644 index 00000000000..5257eb0f276 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-holdout-active.json @@ -0,0 +1,196 @@ +{ + "createdAt": "2026-10-16T12:00:00Z", + "format": "SERVER", + "observeFullEvaluationData": false, + "evaluationCapabilities": [ + "experimentation-opt-out-v1" + ], + "environment": { + "name": "experimentation-opt-out" + }, + "flags": { + "planner": { + "key": "planner", + "enabled": true, + "variationType": "STRING", + "variations": { + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + }, + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + } + }, + "allocations": [ + { + "key": "experiment-holdout-quarterly", + "rules": [], + "endAt": "2026-10-17T12:00:00Z", + "splits": [ + { + "shards": [ + { + "salt": "quarterly-holdout-traffic", + "ranges": [ + { + "start": 0, + "end": 209715 + } + ], + "totalShards": 1048576 + }, + { + "salt": "quarterly-holdout-split", + "ranges": [ + { + "start": 0, + "end": 4096 + } + ], + "totalShards": 8192 + } + ], + "variationKey": "control", + "serialId": 201, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + { + "shards": [ + { + "salt": "quarterly-holdout-traffic", + "ranges": [ + { + "start": 0, + "end": 209715 + } + ], + "totalShards": 1048576 + }, + { + "salt": "quarterly-holdout-split", + "ranges": [ + { + "start": 4096, + "end": 8192 + } + ], + "totalShards": 8192 + } + ], + "variationKey": "control", + "serialId": 202, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "experiment", + "rules": [], + "splits": [ + { + "shards": [ + { + "salt": "planner-experiment-variant", + "ranges": [ + { + "start": 0, + "end": 4096 + } + ], + "totalShards": 8192 + } + ], + "variationKey": "control", + "serialId": 101, + "reason": "SPLIT" + }, + { + "shards": [ + { + "salt": "planner-experiment-variant", + "ranges": [ + { + "start": 4096, + "end": 8192 + } + ], + "totalShards": 8192 + } + ], + "variationKey": "treatment", + "serialId": 102, + "reason": "SPLIT" + } + ], + "doLog": true + }, + { + "key": "allocation-default-7feefdf76b9e", + "rules": [], + "splits": [ + { + "shards": [], + "variationKey": "baseline", + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + } + } +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-holdout-completed-control.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-holdout-completed-control.json new file mode 100644 index 00000000000..0a62aadfc87 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-holdout-completed-control.json @@ -0,0 +1,147 @@ +{ + "createdAt": "2026-10-16T12:00:00Z", + "format": "SERVER", + "observeFullEvaluationData": false, + "evaluationCapabilities": [ + "experimentation-opt-out-v1" + ], + "environment": { + "name": "experimentation-opt-out" + }, + "flags": { + "planner": { + "key": "planner", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + } + }, + "allocations": [ + { + "key": "experiment-holdout-quarterly", + "rules": [], + "endAt": "2026-10-17T12:00:00Z", + "splits": [ + { + "shards": [ + { + "salt": "quarterly-holdout-traffic", + "ranges": [ + { + "start": 0, + "end": 209715 + } + ], + "totalShards": 1048576 + }, + { + "salt": "quarterly-holdout-split", + "ranges": [ + { + "start": 0, + "end": 4096 + } + ], + "totalShards": 8192 + } + ], + "variationKey": "control", + "serialId": 201, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + { + "shards": [ + { + "salt": "quarterly-holdout-traffic", + "ranges": [ + { + "start": 0, + "end": 209715 + } + ], + "totalShards": 1048576 + }, + { + "salt": "quarterly-holdout-split", + "ranges": [ + { + "start": 4096, + "end": 8192 + } + ], + "totalShards": 8192 + } + ], + "variationKey": "control", + "serialId": 202, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "experiment", + "rules": [], + "splits": [ + { + "shards": [], + "variationKey": "control", + "serialId": 101, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + } + } +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-holdout-completed-treatment.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-holdout-completed-treatment.json new file mode 100644 index 00000000000..f3618fbe871 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/compiler-holdout-completed-treatment.json @@ -0,0 +1,151 @@ +{ + "createdAt": "2026-10-16T12:00:00Z", + "format": "SERVER", + "observeFullEvaluationData": false, + "evaluationCapabilities": [ + "experimentation-opt-out-v1" + ], + "environment": { + "name": "experimentation-opt-out" + }, + "flags": { + "planner": { + "key": "planner", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + } + }, + "allocations": [ + { + "key": "experiment-holdout-quarterly", + "rules": [], + "endAt": "2026-10-17T12:00:00Z", + "splits": [ + { + "shards": [ + { + "salt": "quarterly-holdout-traffic", + "ranges": [ + { + "start": 0, + "end": 209715 + } + ], + "totalShards": 1048576 + }, + { + "salt": "quarterly-holdout-split", + "ranges": [ + { + "start": 0, + "end": 4096 + } + ], + "totalShards": 8192 + } + ], + "variationKey": "control", + "serialId": 301, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + }, + { + "shards": [ + { + "salt": "quarterly-holdout-traffic", + "ranges": [ + { + "start": 0, + "end": 209715 + } + ], + "totalShards": 1048576 + }, + { + "salt": "quarterly-holdout-split", + "ranges": [ + { + "start": 4096, + "end": 8192 + } + ], + "totalShards": 8192 + } + ], + "variationKey": "treatment", + "serialId": 302, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "quarterly", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "experiment", + "rules": [], + "splits": [ + { + "shards": [], + "variationKey": "treatment", + "serialId": 102, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + } + } +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/legacy-without-capability.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/legacy-without-capability.json new file mode 100644 index 00000000000..b00537add24 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/legacy-without-capability.json @@ -0,0 +1,152 @@ +{ + "createdAt": "2026-10-16T12:00:00Z", + "format": "SERVER", + "environment": { + "name": "experimentation-opt-out-fixtures" + }, + "flags": { + "two-arm": { + "key": "two-arm", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 5000 + } + ] + } + ], + "serialId": 101, + "reason": "SPLIT" + }, + { + "variationKey": "treatment", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 5000, + "end": 10000 + } + ] + } + ], + "serialId": 102, + "reason": "SPLIT" + } + ], + "doLog": true + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "winner-treatment": { + "key": "winner-treatment", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "winner-rollout", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 802, + "reason": "STATIC" + } + ], + "doLog": true + } + ] + }, + "full-enrollment-experiment": { + "key": "full-enrollment-experiment", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 103, + "reason": "STATIC" + } + ], + "doLog": true + } + ] + } + } +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/missing-dolog.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/missing-dolog.json new file mode 100644 index 00000000000..f918a8735f3 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/missing-dolog.json @@ -0,0 +1,86 @@ +{ + "createdAt": "2026-10-16T12:00:00Z", + "format": "SERVER", + "environment": { + "name": "experimentation-opt-out-fixtures" + }, + "evaluationCapabilities": [ + "experimentation-opt-out-v1" + ], + "flags": { + "two-arm": { + "key": "two-arm", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 5000 + } + ] + } + ], + "serialId": 101, + "reason": "SPLIT" + }, + { + "variationKey": "treatment", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 5000, + "end": 10000 + } + ] + } + ], + "serialId": 102, + "reason": "SPLIT" + } + ] + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + } + } +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/supported.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/supported.json new file mode 100644 index 00000000000..94548b07be6 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/configurations/supported.json @@ -0,0 +1,1507 @@ +{ + "createdAt": "2026-10-16T12:00:00Z", + "format": "SERVER", + "environment": { + "name": "experimentation-opt-out-fixtures" + }, + "evaluationCapabilities": [ + "experimentation-opt-out-v1" + ], + "flags": { + "two-arm": { + "key": "two-arm", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 5000 + } + ] + } + ], + "serialId": 101, + "reason": "SPLIT" + }, + { + "variationKey": "treatment", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 5000, + "end": 10000 + } + ] + } + ], + "serialId": 102, + "reason": "SPLIT" + } + ], + "doLog": true + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "three-arm": { + "key": "three-arm", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + }, + "third": { + "key": "third", + "value": "experimental-third" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 3333 + } + ] + } + ], + "serialId": 201, + "reason": "SPLIT" + }, + { + "variationKey": "treatment", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 3333, + "end": 6666 + } + ] + } + ], + "serialId": 202, + "reason": "SPLIT" + }, + { + "variationKey": "third", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 6666, + "end": 10000 + } + ] + } + ], + "serialId": 203, + "reason": "SPLIT" + } + ], + "doLog": true + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "partial-experiment": { + "key": "partial-experiment", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 2500 + } + ] + } + ], + "serialId": 301, + "reason": "SPLIT" + }, + { + "variationKey": "treatment", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 2500, + "end": 5000 + } + ] + } + ], + "serialId": 302, + "reason": "SPLIT" + } + ], + "doLog": true + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "full-enrollment-experiment": { + "key": "full-enrollment-experiment", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 103, + "reason": "STATIC" + } + ], + "doLog": true + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "ordered-ordinary": { + "key": "ordered-ordinary", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + }, + "beta": { + "key": "beta", + "value": "ordinary-beta" + }, + "gold": { + "key": "gold", + "value": "ordinary-gold" + } + }, + "allocations": [ + { + "key": "beta", + "rules": [ + { + "conditions": [ + { + "attribute": "beta", + "operator": "ONE_OF", + "value": [ + "yes" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "beta", + "shards": [], + "serialId": 901, + "reason": "TARGETING_MATCH" + } + ], + "doLog": false + }, + { + "key": "experiment", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 5000 + } + ] + } + ], + "serialId": 101, + "reason": "SPLIT" + }, + { + "variationKey": "treatment", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 5000, + "end": 10000 + } + ] + } + ], + "serialId": 102, + "reason": "SPLIT" + } + ], + "doLog": true + }, + { + "key": "gold", + "rules": [ + { + "conditions": [ + { + "attribute": "tier", + "operator": "ONE_OF", + "value": [ + "gold" + ] + } + ] + } + ], + "splits": [ + { + "variationKey": "gold", + "shards": [], + "serialId": 902, + "reason": "TARGETING_MATCH" + } + ], + "doLog": false + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "multiple-experiments": { + "key": "multiple-experiments", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "experiment-first", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 111, + "reason": "STATIC" + } + ], + "doLog": true + }, + { + "key": "experiment-second", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [], + "serialId": 112, + "reason": "DEFAULT" + } + ], + "doLog": true + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "equal-value-fallback": { + "key": "equal-value-fallback", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "same-value" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "same-value" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 5000 + } + ] + } + ], + "serialId": 101, + "reason": "SPLIT" + }, + { + "variationKey": "treatment", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 5000, + "end": 10000 + } + ] + } + ], + "serialId": 102, + "reason": "SPLIT" + } + ], + "doLog": true + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "no-fallback": { + "key": "no-fallback", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 5000 + } + ] + } + ], + "serialId": 101, + "reason": "SPLIT" + }, + { + "variationKey": "treatment", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 5000, + "end": 10000 + } + ] + } + ], + "serialId": 102, + "reason": "SPLIT" + } + ], + "doLog": true + } + ] + }, + "ordinary-percentage": { + "key": "ordinary-percentage", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "ordinary-rollout", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 2500 + } + ] + } + ], + "serialId": 701, + "reason": "SPLIT" + } + ], + "doLog": false + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "winner-control": { + "key": "winner-control", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "winner-rollout", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [], + "serialId": 801, + "reason": "STATIC" + } + ], + "doLog": false + } + ] + }, + "winner-treatment": { + "key": "winner-treatment", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "winner-rollout", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 802, + "reason": "STATIC" + } + ], + "doLog": false + } + ] + }, + "canary": { + "key": "canary", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "experiment-backed-canary", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 811, + "reason": "STATIC" + } + ], + "doLog": true + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "bandit": { + "key": "bandit", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "experiment-backed-bandit", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 821, + "reason": "STATIC" + } + ], + "doLog": true + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "boolean-fallback": { + "key": "boolean-fallback", + "enabled": true, + "variationType": "BOOLEAN", + "variations": { + "control": { + "key": "control", + "value": false + }, + "treatment": { + "key": "treatment", + "value": false + }, + "baseline": { + "key": "baseline", + "value": true + } + }, + "allocations": [ + { + "key": "experiment", + "rules": [], + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 831, + "reason": "STATIC" + } + ], + "doLog": true + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "holdout-before-route": { + "key": "holdout-before-route", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "generated-holdout-route", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 5000 + } + ] + } + ], + "serialId": 401, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 5000, + "end": 10000 + } + ] + } + ], + "serialId": 402, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "holdout-before-search": { + "key": "holdout-before-search", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "generated-holdout-search", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 5000 + } + ] + } + ], + "serialId": 501, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 5000, + "end": 10000 + } + ] + } + ], + "serialId": 502, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "ordinary-fallback", + "rules": [], + "splits": [ + { + "variationKey": "baseline", + "shards": [], + "serialId": 900, + "reason": "DEFAULT" + } + ], + "doLog": false + } + ] + }, + "holdout-after-route": { + "key": "holdout-after-route", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "generated-holdout-route", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 5000 + } + ] + } + ], + "serialId": 411, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + }, + { + "variationKey": "treatment", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 5000, + "end": 10000 + } + ] + } + ], + "serialId": 412, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "winner-rollout", + "rules": [], + "doLog": false, + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 802, + "reason": "DEFAULT" + } + ] + } + ] + }, + "holdout-after-search": { + "key": "holdout-after-search", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "generated-holdout-search", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 5000 + } + ] + } + ], + "serialId": 511, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + }, + { + "variationKey": "treatment", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 5000, + "end": 10000 + } + ] + } + ], + "serialId": 512, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": true, + "destinations": [ + "HOOK" + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "winner-rollout", + "rules": [], + "doLog": false, + "splits": [ + { + "variationKey": "treatment", + "shards": [], + "serialId": 852, + "reason": "DEFAULT" + } + ] + } + ] + }, + "holdout-control-winner-route": { + "key": "holdout-control-winner-route", + "enabled": true, + "variationType": "STRING", + "variations": { + "control": { + "key": "control", + "value": "experimental-control" + }, + "treatment": { + "key": "treatment", + "value": "experimental-treatment" + }, + "baseline": { + "key": "baseline", + "value": "ordinary-baseline" + } + }, + "allocations": [ + { + "key": "generated-holdout-control-winner", + "rules": [], + "splits": [ + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 0, + "end": 5000 + } + ] + } + ], + "serialId": 601, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "status_quo", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + }, + { + "variationKey": "control", + "shards": [ + { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "ranges": [ + { + "start": 5000, + "end": 10000 + } + ] + } + ], + "serialId": 602, + "reason": "SPLIT", + "features": [ + { + "key": "holdout.key", + "value": "shared-holdout", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.assignment_group", + "value": "all_shipped", + "destinations": [ + "HOOK" + ] + }, + { + "key": "holdout.should_include_in_holdout_analysis", + "value": false, + "destinations": [ + "HOOK" + ] + } + ] + } + ], + "doLog": true + }, + { + "key": "winner-rollout", + "rules": [], + "doLog": false, + "splits": [ + { + "variationKey": "control", + "shards": [], + "serialId": 803, + "reason": "DEFAULT" + } + ] + } + ] + } + } +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/manifest.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/manifest.json new file mode 100644 index 00000000000..d232ece46e3 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/manifest.json @@ -0,0 +1,560 @@ +{ + "schemaVersion": 1, + "kind": "manifest", + "capability": "experimentation-opt-out-v1", + "configurations": [ + { + "id": "supported", + "path": "configurations/supported.json", + "provenance": { + "kind": "reviewed-contract-input", + "description": "Hand-authored expected UFC semantics. Not production compiler output. Winner lifecycle/link preservation and generated holdout correctness require compiler tests." + } + }, + { + "id": "legacy-without-capability", + "path": "configurations/legacy-without-capability.json", + "provenance": { + "kind": "reviewed-contract-input", + "description": "Deliberately old artifact: no capability, winner still logs, and a full-enrollment experiment lacks its fallback." + } + }, + { + "id": "missing-dolog", + "path": "configurations/missing-dolog.json", + "provenance": { + "kind": "reviewed-contract-input", + "description": "Deliberately malformed marked JSON: the first experimental allocation omits doLog. Ordinary legacy defaults differ by adapter and require a baseline-preservation test." + } + }, + { + "id": "compiler-active-50-50", + "path": "configurations/compiler-active-50-50.json", + "provenance": { + "kind": "compiler-generated", + "description": "Verbatim output from the production UFC compiler with deterministic lifecycle inputs. Generation details are in compiler-generated/PROVENANCE.md.", + "scenario": "active-50-50" + } + }, + { + "id": "compiler-active-100-single-arm", + "path": "configurations/compiler-active-100-single-arm.json", + "provenance": { + "kind": "compiler-generated", + "description": "Verbatim output from the production UFC compiler with deterministic lifecycle inputs. Generation details are in compiler-generated/PROVENANCE.md.", + "scenario": "active-100-single-arm" + } + }, + { + "id": "compiler-completed-control", + "path": "configurations/compiler-completed-control.json", + "provenance": { + "kind": "compiler-generated", + "description": "Verbatim output from the production UFC compiler with deterministic lifecycle inputs. Generation details are in compiler-generated/PROVENANCE.md.", + "scenario": "completed-control" + } + }, + { + "id": "compiler-completed-treatment", + "path": "configurations/compiler-completed-treatment.json", + "provenance": { + "kind": "compiler-generated", + "description": "Verbatim output from the production UFC compiler with deterministic lifecycle inputs. Generation details are in compiler-generated/PROVENANCE.md.", + "scenario": "completed-treatment" + } + }, + { + "id": "compiler-holdout-active", + "path": "configurations/compiler-holdout-active.json", + "provenance": { + "kind": "compiler-generated", + "description": "Verbatim output from the production UFC compiler with deterministic lifecycle inputs. Generation details are in compiler-generated/PROVENANCE.md.", + "scenario": "holdout-active" + } + }, + { + "id": "compiler-holdout-completed-control", + "path": "configurations/compiler-holdout-completed-control.json", + "provenance": { + "kind": "compiler-generated", + "description": "Verbatim output from the production UFC compiler with deterministic lifecycle inputs. Generation details are in compiler-generated/PROVENANCE.md.", + "scenario": "holdout-completed-control" + } + }, + { + "id": "compiler-holdout-completed-treatment", + "path": "configurations/compiler-holdout-completed-treatment.json", + "provenance": { + "kind": "compiler-generated", + "description": "Verbatim output from the production UFC compiler with deterministic lifecycle inputs. Generation details are in compiler-generated/PROVENANCE.md.", + "scenario": "holdout-completed-treatment" + } + } + ], + "evaluationCases": [ + { + "path": "cases/activation.json", + "ids": [ + "activation-disabled-missing", + "activation-disabled-false", + "activation-disabled-true", + "activation-disabled-null", + "activation-disabled-string-true", + "activation-disabled-string-false", + "activation-disabled-number-one", + "activation-disabled-number-zero", + "activation-disabled-array", + "activation-disabled-object", + "activation-disabled-nested", + "activation-enabled-missing", + "activation-enabled-false", + "activation-enabled-true", + "activation-enabled-null", + "activation-enabled-string-true", + "activation-enabled-string-false", + "activation-enabled-number-one", + "activation-enabled-number-zero", + "activation-enabled-array", + "activation-enabled-object", + "activation-enabled-nested", + "activation-unconfigured-true" + ] + }, + { + "path": "cases/evaluation.json", + "ids": [ + "excluded-control", + "excluded-treatment", + "excluded-third-arm", + "excluded-full-enrollment", + "excluded-partial-enrollment", + "eligible-outside-enrollment", + "ordinary-beta-before-experiment", + "ordinary-targeted-after-experiment", + "ordinary-default-after-experiment", + "skip-multiple-experiments", + "equal-value-ordinary-fallback", + "equal-value-experimental-control", + "no-ordinary-assignment", + "ordinary-percentage-included", + "ordinary-percentage-not-included", + "boolean-fallback-is-true", + "winner-control-eligible", + "winner-control-excluded", + "winner-treatment-eligible", + "winner-treatment-excluded", + "excluded-canary", + "excluded-bandit", + "eligible-two-arm-subject-0", + "eligible-three-arm-subject-0", + "eligible-two-arm-subject-1", + "eligible-three-arm-subject-1", + "eligible-two-arm-subject-2", + "eligible-three-arm-subject-2", + "eligible-two-arm-subject-3", + "eligible-three-arm-subject-3", + "eligible-two-arm-subject-4", + "eligible-three-arm-subject-4", + "eligible-two-arm-subject-5", + "eligible-three-arm-subject-5", + "eligible-two-arm-subject-6", + "eligible-three-arm-subject-6", + "eligible-two-arm-subject-7", + "eligible-three-arm-subject-7", + "eligible-two-arm-subject-8", + "eligible-three-arm-subject-8", + "eligible-two-arm-subject-9", + "eligible-three-arm-subject-9", + "eligible-two-arm-subject-10", + "eligible-three-arm-subject-10", + "eligible-two-arm-subject-11", + "eligible-three-arm-subject-11", + "eligible-two-arm-subject-12", + "eligible-three-arm-subject-12", + "eligible-two-arm-subject-13", + "eligible-three-arm-subject-13", + "eligible-two-arm-subject-14", + "eligible-three-arm-subject-14", + "eligible-two-arm-subject-15", + "eligible-three-arm-subject-15" + ] + }, + { + "path": "cases/holdouts.json", + "ids": [ + "holdout-before-route-status-quo-eligible", + "holdout-before-route-status-quo-excluded", + "holdout-before-route-all-shipped-eligible", + "holdout-before-route-all-shipped-excluded", + "holdout-before-search-status-quo-eligible", + "holdout-before-search-status-quo-excluded", + "holdout-before-search-all-shipped-eligible", + "holdout-before-search-all-shipped-excluded", + "holdout-after-route-status-quo-eligible", + "holdout-after-route-status-quo-excluded", + "holdout-after-route-all-shipped-eligible", + "holdout-after-route-all-shipped-excluded", + "holdout-after-search-status-quo-eligible", + "holdout-after-search-status-quo-excluded", + "holdout-after-search-all-shipped-eligible", + "holdout-after-search-all-shipped-excluded", + "holdout-control-winner-route-status-quo-eligible", + "holdout-control-winner-route-status-quo-excluded", + "holdout-control-winner-route-all-shipped-eligible", + "holdout-control-winner-route-all-shipped-excluded" + ] + }, + { + "path": "cases/compatibility.json", + "ids": [ + "legacy-ordinary", + "legacy-excluded", + "missing-dolog-excluded", + "legacy-winner-not-misclassified", + "legacy-pruned-fallback-not-used" + ] + }, + { + "path": "cases/telemetry.json", + "ids": [ + "independent-experiment-false-telemetry-false", + "independent-experiment-false-telemetry-true", + "independent-experiment-true-telemetry-false", + "independent-experiment-true-telemetry-true" + ] + }, + { + "path": "cases/compiler-generated.json", + "ids": [ + "generated-active-50-50-0001-eligible", + "generated-active-50-50-0001-excluded", + "generated-active-50-50-0005-eligible", + "generated-active-50-50-0005-excluded", + "generated-active-100-single-arm-0001-eligible", + "generated-active-100-single-arm-0001-excluded", + "generated-completed-control-0001-eligible", + "generated-completed-control-0001-excluded", + "generated-completed-treatment-0001-eligible", + "generated-completed-treatment-0001-excluded", + "generated-holdout-active-0006-eligible", + "generated-holdout-active-0006-excluded", + "generated-holdout-active-0000-eligible", + "generated-holdout-active-0000-excluded", + "generated-holdout-active-0001-eligible", + "generated-holdout-active-0001-excluded", + "generated-holdout-active-0005-eligible", + "generated-holdout-active-0005-excluded", + "generated-holdout-completed-control-0006-eligible", + "generated-holdout-completed-control-0006-excluded", + "generated-holdout-completed-control-0000-eligible", + "generated-holdout-completed-control-0000-excluded", + "generated-holdout-completed-control-0001-eligible", + "generated-holdout-completed-control-0001-excluded", + "generated-holdout-completed-treatment-0006-eligible", + "generated-holdout-completed-treatment-0006-excluded", + "generated-holdout-completed-treatment-0000-eligible", + "generated-holdout-completed-treatment-0000-excluded", + "generated-holdout-completed-treatment-0001-eligible", + "generated-holdout-completed-treatment-0001-excluded" + ] + } + ], + "protocolCases": [ + { + "path": "protocol/request-and-response.json", + "ids": [ + "request-omitted", + "request-true", + "request-false", + "request-empty", + "request-string", + "request-number", + "request-null-value", + "request-null-options", + "request-array-options", + "response-applied", + "response-missing-ack", + "response-empty-ack", + "response-false-ack", + "response-string-ack", + "response-number-ack", + "response-null-ack", + "response-array-ack", + "response-legacy-ordinary", + "response-explicit-ordinary", + "response-contradictory-ordinary" + ] + } + ], + "sequences": [ + { + "path": "sequences/late-response-after-opt-out.json", + "ids": [ + "late-response-after-opt-out" + ] + }, + { + "path": "sequences/failed-refresh-after-opt-out.json", + "ids": [ + "failed-refresh-after-opt-out" + ] + }, + { + "path": "sequences/opt-out-does-not-create-exposure-dedup.json", + "ids": [ + "opt-out-does-not-create-exposure-dedup" + ] + }, + { + "path": "sequences/missing-ack-never-publishes.json", + "ids": [ + "missing-ack-never-publishes" + ] + }, + { + "path": "sequences/cache-participation-and-subject-identity.json", + "ids": [ + "cache-participation-and-subject-identity" + ] + }, + { + "path": "sequences/logout-rejects-authenticated-response.json", + "ids": [ + "logout-rejects-authenticated-response" + ] + }, + { + "path": "sequences/same-state-compatible-cache-on-failure.json", + "ids": [ + "same-state-compatible-cache-on-failure" + ] + }, + { + "path": "sequences/cache-targeting-context-identity.json", + "ids": [ + "cache-targeting-context-identity" + ] + }, + { + "path": "sequences/cache-environment-identity.json", + "ids": [ + "cache-environment-identity" + ] + } + ], + "expectedCounts": { + "configurations": 10, + "evaluations": 136, + "protocols": 20, + "sequences": 9 + }, + "bucketing": { + "salt": "opt-out:v1:assignment", + "totalShards": 10000, + "algorithm": "MD5(UTF-8(salt + \"-\" + targetingKey)); first four digest bytes, unsigned big-endian integer, modulo totalShards. MD5 is the existing assignment contract, not security.", + "vectors": [ + { + "targetingKey": "subject-0", + "md5": "b55c51498344de9afe27a0b7af828588", + "shard": 6217 + }, + { + "targetingKey": "subject-1", + "md5": "d9a4857a0e8256823803b6d0390761e4", + "shard": 7946 + }, + { + "targetingKey": "subject-2", + "md5": "f43df548110636124d8adaed4685ea69", + "shard": 1192 + }, + { + "targetingKey": "subject-3", + "md5": "938d4c316f70119fb3a903261b04a263", + "shard": 833 + }, + { + "targetingKey": "subject-4", + "md5": "23aa9288dd462de95808cbc3f6331edb", + "shard": 1192 + }, + { + "targetingKey": "subject-5", + "md5": "516d48d600a20ed90531145eaba1c85f", + "shard": 6566 + }, + { + "targetingKey": "subject-6", + "md5": "e59fe9fe5c16cf70a259e46652b834b6", + "shard": 2590 + }, + { + "targetingKey": "subject-7", + "md5": "50959ccaea8e5cea276edbcf85374fff", + "shard": 2282 + }, + { + "targetingKey": "subject-8", + "md5": "1976130512646bd2d41e0529618b4729", + "shard": 8517 + }, + { + "targetingKey": "subject-9", + "md5": "e645ac19eca2592d6b832fbadffc5e44", + "shard": 5721 + }, + { + "targetingKey": "subject-10", + "md5": "23e3bd30e2d251ca082fe11c11518fa2", + "shard": 7664 + }, + { + "targetingKey": "subject-11", + "md5": "9a769814c5024df96b14f59625e64620", + "shard": 3444 + }, + { + "targetingKey": "subject-12", + "md5": "1ca1c768e15650ff1c75d0914a01d51a", + "shard": 4392 + }, + { + "targetingKey": "subject-13", + "md5": "86a2c5230d4d88e890468446d5b6eaaa", + "shard": 4243 + }, + { + "targetingKey": "subject-14", + "md5": "26ba02b495beabdeb051ad95a3626207", + "shard": 4596 + }, + { + "targetingKey": "subject-15", + "md5": "891195f0cd1c24872cdd5819d48e0bef", + "shard": 1088 + } + ] + }, + "invariants": { + "eligibleParityCaseIds": [ + "eligible-two-arm-subject-0", + "eligible-three-arm-subject-0", + "eligible-two-arm-subject-1", + "eligible-three-arm-subject-1", + "eligible-two-arm-subject-2", + "eligible-three-arm-subject-2", + "eligible-two-arm-subject-3", + "eligible-three-arm-subject-3", + "eligible-two-arm-subject-4", + "eligible-three-arm-subject-4", + "eligible-two-arm-subject-5", + "eligible-three-arm-subject-5", + "eligible-two-arm-subject-6", + "eligible-three-arm-subject-6", + "eligible-two-arm-subject-7", + "eligible-three-arm-subject-7", + "eligible-two-arm-subject-8", + "eligible-three-arm-subject-8", + "eligible-two-arm-subject-9", + "eligible-three-arm-subject-9", + "eligible-two-arm-subject-10", + "eligible-three-arm-subject-10", + "eligible-two-arm-subject-11", + "eligible-three-arm-subject-11", + "eligible-two-arm-subject-12", + "eligible-three-arm-subject-12", + "eligible-two-arm-subject-13", + "eligible-three-arm-subject-13", + "eligible-two-arm-subject-14", + "eligible-three-arm-subject-14", + "eligible-two-arm-subject-15", + "eligible-three-arm-subject-15" + ], + "concurrentServerCaseIds": [ + "activation-enabled-false", + "excluded-control" + ], + "concurrentServerRepetitions": 64, + "localRepresentations": [ + "json", + "protobuf", + "rkyv" + ] + }, + "compilerGeneration": { + "provenancePath": "compiler-generated/PROVENANCE.md", + "provenanceSha256": "3c280c6a20c1178394e51cdd542bbcc9960d39f9eea809be50ee97031eace22a", + "observationsPath": "compiler-generated/compiler-scenarios.json", + "observationsSha256": "cd1e3b7a27cf413a32d9849879721ee86769e36736b64846d9abd6bb1d1c1299", + "hashVectorsPath": "compiler-generated/hash-vectors.json", + "hashVectorsSha256": "f0e655675db16dfa2cb3fdfb13f54207204a5a8fca7718875bcafbecfb55966f", + "pairs": [ + { + "scenario": "active-50-50", + "configurationId": "compiler-active-50-50", + "jsonSha256": "be254dddcbe9fb525de452c52be57b5ae7e501910181cc9cf02fbf7502f0e60d", + "protobufPath": "compiler-generated/compiler-active-50-50.pb", + "protobufSha256": "bbf851e5f3eaf28b6f00f19216bab0c18eda7cc9335e2218823afae85ab7a381" + }, + { + "scenario": "active-100-single-arm", + "configurationId": "compiler-active-100-single-arm", + "jsonSha256": "08cf944c460776dc881c8c557f0cc621e3ef6bbab14a4a4575312480956c8dcc", + "protobufPath": "compiler-generated/compiler-active-100-single-arm.pb", + "protobufSha256": "cd47e5e0771f94cf81a68519d807955fe148facf09166008648bd7b9cac71456" + }, + { + "scenario": "completed-control", + "configurationId": "compiler-completed-control", + "jsonSha256": "46adedaf2831cfff6b5915be609a9f1b535cd1b68ecb591a0bcaacfa2bc05c47", + "protobufPath": "compiler-generated/compiler-completed-control.pb", + "protobufSha256": "c49dcff900f74ce93aebf4a58653890f7d1acc8bd345a06cb7a8d9349d420649" + }, + { + "scenario": "completed-treatment", + "configurationId": "compiler-completed-treatment", + "jsonSha256": "88747d93db3a69ec5614350ee1af5018ce4020162daa0f4fe95c2f9504e90afe", + "protobufPath": "compiler-generated/compiler-completed-treatment.pb", + "protobufSha256": "0b12e14714a72a0b915f715ef21af3de6c5b5c3fbb0c87c874236fc7b02a85a3" + }, + { + "scenario": "holdout-active", + "configurationId": "compiler-holdout-active", + "jsonSha256": "77e74cfb44f94c5fc26a63f6458e1bb32006bd0ab9b3c18968e5c520be53aa52", + "protobufPath": "compiler-generated/compiler-holdout-active.pb", + "protobufSha256": "290aeba7f940c3c97b8d3596c26ad3e829e45b94c47f7ef0837eb28c21c33976" + }, + { + "scenario": "holdout-completed-control", + "configurationId": "compiler-holdout-completed-control", + "jsonSha256": "9524a3b823a64aa4f12f1bfdfe0bf486b49923ab0ecec6ba1e5331c112576ca6", + "protobufPath": "compiler-generated/compiler-holdout-completed-control.pb", + "protobufSha256": "79228842a5881ccf0f7c6ecd2396ab5c3c0d39cbeff8dc65972b0bc5096d5ca3" + }, + { + "scenario": "holdout-completed-treatment", + "configurationId": "compiler-holdout-completed-treatment", + "jsonSha256": "b12df3b3fa551ea56eae01b5ea1676da1f5317e304d4bd111a9dee9fb1f32e99", + "protobufPath": "compiler-generated/compiler-holdout-completed-treatment.pb", + "protobufSha256": "63c9a9495a54da2da565d51d63df177acb31581a058c1665045ec4d9c7f2c8d8" + } + ], + "eligibleParityCaseIds": [ + "generated-active-50-50-0001-eligible", + "generated-active-50-50-0005-eligible", + "generated-active-100-single-arm-0001-eligible", + "generated-completed-control-0001-eligible", + "generated-completed-treatment-0001-eligible", + "generated-holdout-active-0006-eligible", + "generated-holdout-active-0000-eligible", + "generated-holdout-active-0001-eligible", + "generated-holdout-active-0005-eligible", + "generated-holdout-completed-control-0006-eligible", + "generated-holdout-completed-control-0000-eligible", + "generated-holdout-completed-control-0001-eligible", + "generated-holdout-completed-treatment-0006-eligible", + "generated-holdout-completed-treatment-0000-eligible", + "generated-holdout-completed-treatment-0001-eligible" + ] + } +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/protocol/request-and-response.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/protocol/request-and-response.json new file mode 100644 index 00000000000..3b5013d87e2 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/protocol/request-and-response.json @@ -0,0 +1,643 @@ +[ + { + "schemaVersion": 1, + "kind": "protocol", + "id": "request-omitted", + "description": "Validate the explicit evaluation option without inferring activation from targeting attributes.", + "operation": "request-validation", + "requestedOptOut": false, + "message": { + "data": { + "type": "precompute-assignments-request", + "attributes": { + "env": { + "dd_env": "production" + }, + "subject": { + "targeting_key": "subject-2", + "targeting_attributes": { + "dd_experimentation_opt_out": true + } + } + } + } + }, + "expected": { + "accepted": true, + "effectiveExperimentationOptOut": false, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "request-true", + "description": "Validate the explicit evaluation option without inferring activation from targeting attributes.", + "operation": "request-validation", + "requestedOptOut": true, + "message": { + "data": { + "type": "precompute-assignments-request", + "attributes": { + "env": { + "dd_env": "production" + }, + "subject": { + "targeting_key": "subject-2", + "targeting_attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation_options": { + "experimentation_opt_out": true + } + } + } + }, + "expected": { + "accepted": true, + "effectiveExperimentationOptOut": true, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "request-false", + "description": "Validate the explicit evaluation option without inferring activation from targeting attributes.", + "operation": "request-validation", + "requestedOptOut": false, + "message": { + "data": { + "type": "precompute-assignments-request", + "attributes": { + "env": { + "dd_env": "production" + }, + "subject": { + "targeting_key": "subject-2", + "targeting_attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation_options": { + "experimentation_opt_out": false + } + } + } + }, + "expected": { + "accepted": true, + "effectiveExperimentationOptOut": false, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "request-empty", + "description": "Validate the explicit evaluation option without inferring activation from targeting attributes.", + "operation": "request-validation", + "requestedOptOut": false, + "message": { + "data": { + "type": "precompute-assignments-request", + "attributes": { + "env": { + "dd_env": "production" + }, + "subject": { + "targeting_key": "subject-2", + "targeting_attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation_options": {} + } + } + }, + "expected": { + "accepted": true, + "effectiveExperimentationOptOut": false, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "request-string", + "description": "Validate the explicit evaluation option without inferring activation from targeting attributes.", + "operation": "request-validation", + "requestedOptOut": false, + "message": { + "data": { + "type": "precompute-assignments-request", + "attributes": { + "env": { + "dd_env": "production" + }, + "subject": { + "targeting_key": "subject-2", + "targeting_attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation_options": { + "experimentation_opt_out": "true" + } + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "request-number", + "description": "Validate the explicit evaluation option without inferring activation from targeting attributes.", + "operation": "request-validation", + "requestedOptOut": false, + "message": { + "data": { + "type": "precompute-assignments-request", + "attributes": { + "env": { + "dd_env": "production" + }, + "subject": { + "targeting_key": "subject-2", + "targeting_attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation_options": { + "experimentation_opt_out": 1 + } + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "request-null-value", + "description": "Validate the explicit evaluation option without inferring activation from targeting attributes.", + "operation": "request-validation", + "requestedOptOut": false, + "message": { + "data": { + "type": "precompute-assignments-request", + "attributes": { + "env": { + "dd_env": "production" + }, + "subject": { + "targeting_key": "subject-2", + "targeting_attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation_options": { + "experimentation_opt_out": null + } + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "request-null-options", + "description": "Validate the explicit evaluation option without inferring activation from targeting attributes.", + "operation": "request-validation", + "requestedOptOut": false, + "message": { + "data": { + "type": "precompute-assignments-request", + "attributes": { + "env": { + "dd_env": "production" + }, + "subject": { + "targeting_key": "subject-2", + "targeting_attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation_options": null + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "request-array-options", + "description": "Validate the explicit evaluation option without inferring activation from targeting attributes.", + "operation": "request-validation", + "requestedOptOut": false, + "message": { + "data": { + "type": "precompute-assignments-request", + "attributes": { + "env": { + "dd_env": "production" + }, + "subject": { + "targeting_key": "subject-2", + "targeting_attributes": { + "dd_experimentation_opt_out": true + } + }, + "evaluation_options": [] + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "response-applied", + "description": "Publish assignments only when response acknowledgement matches the requested participation state.", + "operation": "response-validation", + "requestedOptOut": true, + "message": { + "data": { + "attributes": { + "createdAt": "2026-10-16T12:00:00Z", + "flags": { + "two-arm": { + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "variationType": "string", + "variationValue": "ordinary-baseline", + "reason": "DEFAULT", + "doLog": false, + "serialId": 900 + } + }, + "evaluation_options": { + "experimentation_opt_out": true + } + } + } + }, + "expected": { + "accepted": true, + "effectiveExperimentationOptOut": true, + "publishAssignments": true + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "response-missing-ack", + "description": "Publish assignments only when response acknowledgement matches the requested participation state.", + "operation": "response-validation", + "requestedOptOut": true, + "message": { + "data": { + "attributes": { + "createdAt": "2026-10-16T12:00:00Z", + "flags": { + "two-arm": { + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "variationType": "string", + "variationValue": "ordinary-baseline", + "reason": "DEFAULT", + "doLog": false, + "serialId": 900 + } + } + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "response-empty-ack", + "description": "Publish assignments only when response acknowledgement matches the requested participation state.", + "operation": "response-validation", + "requestedOptOut": true, + "message": { + "data": { + "attributes": { + "createdAt": "2026-10-16T12:00:00Z", + "flags": { + "two-arm": { + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "variationType": "string", + "variationValue": "ordinary-baseline", + "reason": "DEFAULT", + "doLog": false, + "serialId": 900 + } + }, + "evaluation_options": {} + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "response-false-ack", + "description": "Publish assignments only when response acknowledgement matches the requested participation state.", + "operation": "response-validation", + "requestedOptOut": true, + "message": { + "data": { + "attributes": { + "createdAt": "2026-10-16T12:00:00Z", + "flags": { + "two-arm": { + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "variationType": "string", + "variationValue": "ordinary-baseline", + "reason": "DEFAULT", + "doLog": false, + "serialId": 900 + } + }, + "evaluation_options": { + "experimentation_opt_out": false + } + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "response-string-ack", + "description": "Publish assignments only when response acknowledgement matches the requested participation state.", + "operation": "response-validation", + "requestedOptOut": true, + "message": { + "data": { + "attributes": { + "createdAt": "2026-10-16T12:00:00Z", + "flags": { + "two-arm": { + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "variationType": "string", + "variationValue": "ordinary-baseline", + "reason": "DEFAULT", + "doLog": false, + "serialId": 900 + } + }, + "evaluation_options": { + "experimentation_opt_out": "true" + } + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "response-number-ack", + "description": "Publish assignments only when response acknowledgement matches the requested participation state.", + "operation": "response-validation", + "requestedOptOut": true, + "message": { + "data": { + "attributes": { + "createdAt": "2026-10-16T12:00:00Z", + "flags": { + "two-arm": { + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "variationType": "string", + "variationValue": "ordinary-baseline", + "reason": "DEFAULT", + "doLog": false, + "serialId": 900 + } + }, + "evaluation_options": { + "experimentation_opt_out": 1 + } + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "response-null-ack", + "description": "Publish assignments only when response acknowledgement matches the requested participation state.", + "operation": "response-validation", + "requestedOptOut": true, + "message": { + "data": { + "attributes": { + "createdAt": "2026-10-16T12:00:00Z", + "flags": { + "two-arm": { + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "variationType": "string", + "variationValue": "ordinary-baseline", + "reason": "DEFAULT", + "doLog": false, + "serialId": 900 + } + }, + "evaluation_options": null + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "response-array-ack", + "description": "Publish assignments only when response acknowledgement matches the requested participation state.", + "operation": "response-validation", + "requestedOptOut": true, + "message": { + "data": { + "attributes": { + "createdAt": "2026-10-16T12:00:00Z", + "flags": { + "two-arm": { + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "variationType": "string", + "variationValue": "ordinary-baseline", + "reason": "DEFAULT", + "doLog": false, + "serialId": 900 + } + }, + "evaluation_options": [] + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "response-legacy-ordinary", + "description": "Publish assignments only when response acknowledgement matches the requested participation state.", + "operation": "response-validation", + "requestedOptOut": false, + "message": { + "data": { + "attributes": { + "createdAt": "2026-10-16T12:00:00Z", + "flags": { + "two-arm": { + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "variationType": "string", + "variationValue": "ordinary-baseline", + "reason": "DEFAULT", + "doLog": false, + "serialId": 900 + } + } + } + } + }, + "expected": { + "accepted": true, + "effectiveExperimentationOptOut": false, + "publishAssignments": true + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "response-explicit-ordinary", + "description": "Publish assignments only when response acknowledgement matches the requested participation state.", + "operation": "response-validation", + "requestedOptOut": false, + "message": { + "data": { + "attributes": { + "createdAt": "2026-10-16T12:00:00Z", + "flags": { + "two-arm": { + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "variationType": "string", + "variationValue": "ordinary-baseline", + "reason": "DEFAULT", + "doLog": false, + "serialId": 900 + } + }, + "evaluation_options": { + "experimentation_opt_out": false + } + } + } + }, + "expected": { + "accepted": true, + "effectiveExperimentationOptOut": false, + "publishAssignments": true + } + }, + { + "schemaVersion": 1, + "kind": "protocol", + "id": "response-contradictory-ordinary", + "description": "Publish assignments only when response acknowledgement matches the requested participation state.", + "operation": "response-validation", + "requestedOptOut": false, + "message": { + "data": { + "attributes": { + "createdAt": "2026-10-16T12:00:00Z", + "flags": { + "two-arm": { + "allocationKey": "ordinary-fallback", + "variationKey": "baseline", + "variationType": "string", + "variationValue": "ordinary-baseline", + "reason": "DEFAULT", + "doLog": false, + "serialId": 900 + } + }, + "evaluation_options": { + "experimentation_opt_out": true + } + } + } + }, + "expected": { + "accepted": false, + "effectiveExperimentationOptOut": null, + "publishAssignments": false + } + } +] diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/cache-environment-identity.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/cache-environment-identity.json new file mode 100644 index 00000000000..97ec40a29a9 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/cache-environment-identity.json @@ -0,0 +1,90 @@ +{ + "schemaVersion": 1, + "kind": "sequence", + "id": "cache-environment-identity", + "description": "Assignments stored for one environment cannot be loaded into a different environment.", + "flag": "two-arm", + "defaultValue": "application-default", + "steps": [ + { + "action": "configure", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "environment": "production" + }, + { + "action": "set-context", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "expectedPublishedCase": null + }, + { + "action": "start-request", + "requestId": "first", + "expectedEvaluationOptions": { + "experimentation_opt_out": true + } + }, + { + "action": "deliver-response", + "requestId": "first", + "caseId": "excluded-control", + "acknowledgement": "applied", + "expectedAccepted": true, + "expectedPublishedCase": "excluded-control" + }, + { + "action": "persist-cache", + "cacheId": "excluded-cache" + }, + { + "action": "restart", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "environment": "staging", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "cacheId": "excluded-cache", + "expectedPublishedCase": null + }, + { + "action": "evaluate", + "caseId": null, + "expectedValue": "application-default", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + }, + { + "action": "restart", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "environment": "production", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "cacheId": "excluded-cache", + "expectedPublishedCase": "excluded-control" + }, + { + "action": "evaluate", + "caseId": "excluded-control", + "expectedValue": "ordinary-baseline", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + } + ] +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/cache-participation-and-subject-identity.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/cache-participation-and-subject-identity.json new file mode 100644 index 00000000000..73d87d3716f --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/cache-participation-and-subject-identity.json @@ -0,0 +1,108 @@ +{ + "schemaVersion": 1, + "kind": "sequence", + "id": "cache-participation-and-subject-identity", + "description": "A cache survives only for the same subject, context, and effective participation option.", + "steps": [ + { + "action": "configure", + "providerOptions": { + "enableExperimentationOptOutParameter": true + } + }, + { + "action": "set-context", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "expectedPublishedCase": null + }, + { + "action": "start-request", + "requestId": "excluded", + "expectedEvaluationOptions": { + "experimentation_opt_out": true + } + }, + { + "action": "deliver-response", + "requestId": "excluded", + "caseId": "excluded-control", + "acknowledgement": "applied", + "expectedAccepted": true, + "expectedPublishedCase": "excluded-control" + }, + { + "action": "persist-cache", + "cacheId": "excluded-cache" + }, + { + "action": "restart", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "cacheId": "excluded-cache", + "expectedPublishedCase": "excluded-control" + }, + { + "action": "evaluate", + "caseId": "excluded-control", + "expectedValue": "ordinary-baseline", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + }, + { + "action": "restart", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "cacheId": "excluded-cache", + "expectedPublishedCase": null + }, + { + "action": "evaluate", + "caseId": null, + "expectedValue": "application-default", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + }, + { + "action": "restart", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-0", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "cacheId": "excluded-cache", + "expectedPublishedCase": null + }, + { + "action": "evaluate", + "caseId": null, + "expectedValue": "application-default", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + } + ], + "flag": "two-arm", + "defaultValue": "application-default" +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/cache-targeting-context-identity.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/cache-targeting-context-identity.json new file mode 100644 index 00000000000..40f1c1ba2f0 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/cache-targeting-context-identity.json @@ -0,0 +1,68 @@ +{ + "schemaVersion": 1, + "kind": "sequence", + "id": "cache-targeting-context-identity", + "description": "Changing a targeting attribute invalidates a stored assignment, even when the subject and participation option do not change.", + "flag": "two-arm", + "defaultValue": "application-default", + "steps": [ + { + "action": "configure", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "environment": "production" + }, + { + "action": "set-context", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "expectedPublishedCase": null + }, + { + "action": "start-request", + "requestId": "first", + "expectedEvaluationOptions": { + "experimentation_opt_out": true + } + }, + { + "action": "deliver-response", + "requestId": "first", + "caseId": "excluded-control", + "acknowledgement": "applied", + "expectedAccepted": true, + "expectedPublishedCase": "excluded-control" + }, + { + "action": "persist-cache", + "cacheId": "excluded-cache" + }, + { + "action": "restart", + "providerOptions": { + "enableExperimentationOptOutParameter": true + }, + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true, + "country": "FR" + } + }, + "cacheId": "excluded-cache", + "expectedPublishedCase": null + }, + { + "action": "evaluate", + "caseId": null, + "expectedValue": "application-default", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + } + ] +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/failed-refresh-after-opt-out.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/failed-refresh-after-opt-out.json new file mode 100644 index 00000000000..9dccd25e15c --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/failed-refresh-after-opt-out.json @@ -0,0 +1,75 @@ +{ + "schemaVersion": 1, + "kind": "sequence", + "id": "failed-refresh-after-opt-out", + "description": "A failed excluded refresh cannot restore an eligible assignment from the previous participation state.", + "steps": [ + { + "action": "configure", + "providerOptions": { + "enableExperimentationOptOutParameter": true + } + }, + { + "action": "set-context", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "expectedPublishedCase": null + }, + { + "action": "start-request", + "requestId": "eligible", + "expectedEvaluationOptions": null + }, + { + "action": "deliver-response", + "requestId": "eligible", + "caseId": "activation-enabled-false", + "acknowledgement": "missing", + "expectedAccepted": true, + "expectedPublishedCase": "activation-enabled-false" + }, + { + "action": "evaluate", + "caseId": "activation-enabled-false", + "expectedValue": "experimental-control", + "expectedExposureCount": 1, + "expectedEvaluationTelemetry": true + }, + { + "action": "set-context", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "expectedPublishedCase": null + }, + { + "action": "start-request", + "requestId": "excluded", + "expectedEvaluationOptions": { + "experimentation_opt_out": true + } + }, + { + "action": "fail-request", + "requestId": "excluded", + "expectedPublishedCase": null + }, + { + "action": "evaluate", + "caseId": null, + "expectedValue": "application-default", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + } + ], + "flag": "two-arm", + "defaultValue": "application-default" +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/late-response-after-opt-out.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/late-response-after-opt-out.json new file mode 100644 index 00000000000..54e5814f99f --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/late-response-after-opt-out.json @@ -0,0 +1,78 @@ +{ + "schemaVersion": 1, + "kind": "sequence", + "id": "late-response-after-opt-out", + "description": "A previous eligible response cannot publish after opt-out, including before the replacement request finishes.", + "steps": [ + { + "action": "configure", + "providerOptions": { + "enableExperimentationOptOutParameter": true + } + }, + { + "action": "set-context", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "expectedPublishedCase": null + }, + { + "action": "start-request", + "requestId": "eligible", + "expectedEvaluationOptions": null + }, + { + "action": "set-context", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "expectedPublishedCase": null + }, + { + "action": "start-request", + "requestId": "excluded", + "expectedEvaluationOptions": { + "experimentation_opt_out": true + } + }, + { + "action": "deliver-response", + "requestId": "eligible", + "caseId": "activation-enabled-false", + "acknowledgement": "missing", + "expectedAccepted": false, + "expectedPublishedCase": null + }, + { + "action": "evaluate", + "caseId": null, + "expectedValue": "application-default", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + }, + { + "action": "deliver-response", + "requestId": "excluded", + "caseId": "excluded-control", + "acknowledgement": "applied", + "expectedAccepted": true, + "expectedPublishedCase": "excluded-control" + }, + { + "action": "evaluate", + "caseId": "excluded-control", + "expectedValue": "ordinary-baseline", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + } + ], + "flag": "two-arm", + "defaultValue": "application-default" +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/logout-rejects-authenticated-response.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/logout-rejects-authenticated-response.json new file mode 100644 index 00000000000..de07b2eec74 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/logout-rejects-authenticated-response.json @@ -0,0 +1,56 @@ +{ + "schemaVersion": 1, + "kind": "sequence", + "id": "logout-rejects-authenticated-response", + "description": "A response for the previous subject cannot publish after logout changes the subject.", + "steps": [ + { + "action": "configure", + "providerOptions": { + "enableExperimentationOptOutParameter": true + } + }, + { + "action": "set-context", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "expectedPublishedCase": null + }, + { + "action": "start-request", + "requestId": "athlete", + "expectedEvaluationOptions": null + }, + { + "action": "set-context", + "context": { + "targetingKey": "anonymous", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "expectedPublishedCase": null + }, + { + "action": "deliver-response", + "requestId": "athlete", + "caseId": "activation-enabled-false", + "acknowledgement": "missing", + "expectedAccepted": false, + "expectedPublishedCase": null + }, + { + "action": "evaluate", + "caseId": null, + "expectedValue": "application-default", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + } + ], + "flag": "two-arm", + "defaultValue": "application-default" +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/missing-ack-never-publishes.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/missing-ack-never-publishes.json new file mode 100644 index 00000000000..49074fecc1b --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/missing-ack-never-publishes.json @@ -0,0 +1,48 @@ +{ + "schemaVersion": 1, + "kind": "sequence", + "id": "missing-ack-never-publishes", + "description": "An unsupported edge response never publishes assignments for an excluded request.", + "steps": [ + { + "action": "configure", + "providerOptions": { + "enableExperimentationOptOutParameter": true + } + }, + { + "action": "set-context", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "expectedPublishedCase": null + }, + { + "action": "start-request", + "requestId": "excluded", + "expectedEvaluationOptions": { + "experimentation_opt_out": true + } + }, + { + "action": "deliver-response", + "requestId": "excluded", + "caseId": "excluded-control", + "acknowledgement": "missing", + "expectedAccepted": false, + "expectedPublishedCase": null + }, + { + "action": "evaluate", + "caseId": null, + "expectedValue": "application-default", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + } + ], + "flag": "two-arm", + "defaultValue": "application-default" +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/opt-out-does-not-create-exposure-dedup.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/opt-out-does-not-create-exposure-dedup.json new file mode 100644 index 00000000000..5f8c95286c9 --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/opt-out-does-not-create-exposure-dedup.json @@ -0,0 +1,85 @@ +{ + "schemaVersion": 1, + "kind": "sequence", + "id": "opt-out-does-not-create-exposure-dedup", + "description": "Skipped experiment allocations create no deduplication entry. A later eligible evaluation can record its first exposure.", + "steps": [ + { + "action": "configure", + "providerOptions": { + "enableExperimentationOptOutParameter": true + } + }, + { + "action": "set-context", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "expectedPublishedCase": null + }, + { + "action": "start-request", + "requestId": "excluded", + "expectedEvaluationOptions": { + "experimentation_opt_out": true + } + }, + { + "action": "deliver-response", + "requestId": "excluded", + "caseId": "excluded-control", + "acknowledgement": "applied", + "expectedAccepted": true, + "expectedPublishedCase": "excluded-control" + }, + { + "action": "evaluate", + "caseId": "excluded-control", + "expectedValue": "ordinary-baseline", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + }, + { + "action": "set-context", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": false + } + }, + "expectedPublishedCase": null + }, + { + "action": "start-request", + "requestId": "eligible", + "expectedEvaluationOptions": null + }, + { + "action": "deliver-response", + "requestId": "eligible", + "caseId": "activation-enabled-false", + "acknowledgement": "missing", + "expectedAccepted": true, + "expectedPublishedCase": "activation-enabled-false" + }, + { + "action": "evaluate", + "caseId": "activation-enabled-false", + "expectedValue": "experimental-control", + "expectedExposureCount": 1, + "expectedEvaluationTelemetry": true + }, + { + "action": "evaluate", + "caseId": "activation-enabled-false", + "expectedValue": "experimental-control", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + } + ], + "flag": "two-arm", + "defaultValue": "application-default" +} diff --git a/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/same-state-compatible-cache-on-failure.json b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/same-state-compatible-cache-on-failure.json new file mode 100644 index 00000000000..8181019674f --- /dev/null +++ b/tests/parametric/test_ffe/fixtures/experimentation-opt-out/sequences/same-state-compatible-cache-on-failure.json @@ -0,0 +1,60 @@ +{ + "schemaVersion": 1, + "kind": "sequence", + "id": "same-state-compatible-cache-on-failure", + "description": "A failed refresh may keep a compatible assignment. It must not erase valid same-context state unnecessarily.", + "steps": [ + { + "action": "configure", + "providerOptions": { + "enableExperimentationOptOutParameter": true + } + }, + { + "action": "set-context", + "context": { + "targetingKey": "subject-2", + "attributes": { + "dd_experimentation_opt_out": true + } + }, + "expectedPublishedCase": null + }, + { + "action": "start-request", + "requestId": "first", + "expectedEvaluationOptions": { + "experimentation_opt_out": true + } + }, + { + "action": "deliver-response", + "requestId": "first", + "caseId": "excluded-control", + "acknowledgement": "applied", + "expectedAccepted": true, + "expectedPublishedCase": "excluded-control" + }, + { + "action": "start-request", + "requestId": "refresh", + "expectedEvaluationOptions": { + "experimentation_opt_out": true + } + }, + { + "action": "fail-request", + "requestId": "refresh", + "expectedPublishedCase": "excluded-control" + }, + { + "action": "evaluate", + "caseId": "excluded-control", + "expectedValue": "ordinary-baseline", + "expectedExposureCount": 0, + "expectedEvaluationTelemetry": true + } + ], + "flag": "two-arm", + "defaultValue": "application-default" +} diff --git a/tests/parametric/test_ffe/test_dynamic_evaluation.py b/tests/parametric/test_ffe/test_dynamic_evaluation.py index 35b15215c50..c7fbaae80b3 100644 --- a/tests/parametric/test_ffe/test_dynamic_evaluation.py +++ b/tests/parametric/test_ffe/test_dynamic_evaluation.py @@ -137,6 +137,51 @@ class Test_Feature_Flag_Dynamic_Evaluation: """ + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) + def test_large_integer_context(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: + """Preserve JSON integers above int32 that remain exact IEEE-754 numbers.""" + number = 1_234_567_890_123_456 + fixture: dict[str, Any] = { + "createdAt": "2026-10-16T00:00:00Z", + "format": "SERVER", + "environment": {"name": "Test"}, + "flags": { + "large-integer-context": { + "key": "large-integer-context", + "enabled": True, + "variationType": "BOOLEAN", + "variations": {"matched": {"key": "matched", "value": True}}, + "allocations": [ + { + "key": "exact-number", + "rules": [ + { + "conditions": [ + {"attribute": "number", "operator": "GTE", "value": number}, + {"attribute": "number", "operator": "LTE", "value": number}, + ] + } + ], + "splits": [{"variationKey": "matched", "shards": []}], + "doLog": False, + } + ], + } + }, + } + _set_and_wait_ffe_rc(test_agent, fixture) + assert test_library.ffe_start(fixture), "Failed to start FFE provider" + for actual in (number - 1, number, number + 1): + result = _ffe_evaluate_with_rc_retry( + test_library, + flag="large-integer-context", + variation_type="BOOLEAN", + default_value=False, + targeting_key="large-integer-subject", + attributes={"number": actual}, + ) + assert result["value"] is (actual == number), result + @parametrize("library_env", [{**DEFAULT_ENVVARS}]) def test_ffe_remote_config(self, test_agent: TestAgentAPI, test_library: APMLibrary) -> None: """Test to verify FFE can receive and acknowledge UFC configurations via Remote Config.""" diff --git a/tests/parametric/test_ffe/test_experimentation_opt_out.py b/tests/parametric/test_ffe/test_experimentation_opt_out.py new file mode 100644 index 00000000000..f018496e13e --- /dev/null +++ b/tests/parametric/test_ffe/test_experimentation_opt_out.py @@ -0,0 +1,65 @@ +"""Exercise shared experimentation opt-out inputs through a real Java provider. + +This suite checks assignment behavior, not per-case exposure counts. Exposure +deduplication is process-wide. The isolated dogfooding suite checks emissions. +Time-gated generated cases stay in fixed-clock SDK tests and guarded dogfooding. +The shared configurations and expected results are never rewritten for runtime. +""" + +import hashlib +import json +import time +from collections import defaultdict +from typing import Any + +from utils import features, pytest, scenarios +from utils.dd_constants import RemoteConfigApplyState +from utils.docker_fixtures import ParametricTestClientApi as APMLibrary, TestAgentAPI +from tests.parametric.test_ffe.experimentation_contract import assert_details, has_time_boundary, load_contract + +MANIFEST, CONFIGURATIONS, CASES = load_contract() +GROUPS: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) +for case in CASES: + if not has_time_boundary(CONFIGURATIONS[case["configuration"]], case): + GROUPS[(case["configuration"], json.dumps(case["providerOptions"], sort_keys=True))].append(case) + +DEFAULT_ENVVARS = { + "DD_FEATURE_FLAGS_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", + # Ordinary allocations expose serial metadata when this existing feature is enabled. + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": "true", +} + + +@scenarios.parametric +@features.feature_flags_dynamic_evaluation +class Test_Feature_Flag_Experimentation_Opt_Out: + """Compare exact shared assignments with explicitly activated public options.""" + + @pytest.mark.parametrize("library_env", [DEFAULT_ENVVARS]) + @pytest.mark.parametrize("group", list(GROUPS), ids=[f"{key[0]}-{index}" for index, key in enumerate(GROUPS)]) + def test_shared_contract(self, group: tuple[str, str], test_agent: TestAgentAPI, test_library: APMLibrary) -> None: + configuration_id, options_json = group + configuration = CONFIGURATIONS[configuration_id] + config_id = hashlib.sha256(json.dumps(configuration, sort_keys=True).encode()).hexdigest() + test_agent.set_remote_config(path=f"datadog/2/FFE_FLAGS/{config_id}/config", payload=configuration) + state = test_agent.wait_for_rc_apply_state( + "FFE_FLAGS", state=RemoteConfigApplyState.ACKNOWLEDGED, config_id=config_id, wait_loops=500, clear=True + ) + assert state["apply_state"] == RemoteConfigApplyState.ACKNOWLEDGED.value + assert test_library.ffe_start(configuration, provider_options=json.loads(options_json)) + for case in GROUPS[group]: + evaluation = case["evaluation"] + for attempt in range(10): + result = test_library.ffe_evaluate( + flag=evaluation["flag"], + variation_type=evaluation["variationType"], + default_value=evaluation["defaultValue"], + targeting_key=case["context"]["targetingKey"], + attributes=case["context"].get("attributes", {}), + ) + if result.get("errorCode") != "PROVIDER_NOT_READY" or attempt == 9: + break + time.sleep(0.2) + assert_details(result, case) diff --git a/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java b/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java index 40bd69d869a..3cdb196351f 100644 --- a/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java +++ b/utils/build/docker/java/parametric/src/main/java/com/datadoghq/trace/controller/FeatureFlagEvaluatorController.java @@ -7,6 +7,7 @@ import dev.openfeature.sdk.Client; import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.FlagEvaluationDetails; import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.NoOpProvider; import dev.openfeature.sdk.OpenFeatureAPI; @@ -69,7 +70,32 @@ public ProviderState getState() { private Client client; @PostMapping(value = "/start") - public ResponseEntity start() { + public synchronized ResponseEntity start(@RequestBody(required = false) final Map request) { + if (request != null && request.containsKey("providerOptions")) { + final Object requestedOptions = request.get("providerOptions"); + if (!(requestedOptions instanceof Map optionsMap)) { + return ResponseEntity.badRequest().body(false); + } + final Provider.Options options = new Provider.Options().initTimeout(100, TimeUnit.MILLISECONDS); + try { + for (Map.Entry entry : optionsMap.entrySet()) { + final String name = String.valueOf(entry.getKey()); + if ((!name.equals("enableExperimentationOptOutParameter") + && !name.equals("enableEvaluationTelemetryOptOutParameter")) + || !(entry.getValue() instanceof Boolean)) { + return ResponseEntity.badRequest().body(false); + } + // A missing public API is a start failure, not a behavioral negative control. + Provider.Options.class.getMethod(name, boolean.class).invoke(options, entry.getValue()); + } + // Resolve the lazy default provider before replacing it with explicit options. + client.getProviderState(); + OpenFeatureAPI.getInstance().setProviderAndWait(new Provider(options)); + } catch (ReflectiveOperationException error) { + LOGGER.error("Requested provider option is unavailable", error); + return ResponseEntity.internalServerError().body(false); + } + } final ProviderState state = client.getProviderState(); if (state == ProviderState.READY) { return ResponseEntity.ok(true); @@ -80,38 +106,42 @@ public ResponseEntity start() { @PostMapping(value = "/evaluate", consumes = APPLICATION_JSON_VALUE, produces = APPLICATION_JSON_VALUE) public ResponseEntity> evaluate(@RequestBody final EvaluateRequest request) { - Object value; - String reason; + final Map result = new HashMap<>(); final EvaluationContext context = context(request); try { - value = switch (request.getVariationType()) { + final FlagEvaluationDetails details = switch (request.getVariationType()) { case "BOOLEAN" -> - client.getBooleanValue(request.getFlag(), (Boolean) request.getDefaultValue(), context); - case "STRING" -> client.getStringValue(request.getFlag(), (String) request.getDefaultValue(), context); + client.getBooleanDetails(request.getFlag(), (Boolean) request.getDefaultValue(), context); + case "STRING" -> client.getStringDetails(request.getFlag(), (String) request.getDefaultValue(), context); case "INTEGER" -> { final Number integerEval = (Number) request.getDefaultValue(); - yield client.getIntegerValue(request.getFlag(), integerEval.intValue(), context); + yield client.getIntegerDetails(request.getFlag(), integerEval.intValue(), context); } case "NUMERIC" -> { final Number doubleEval = (Number) request.getDefaultValue(); - yield client.getDoubleValue(request.getFlag(), doubleEval.doubleValue(), context); - } - case "JSON" -> { - final Value objectValue = client.getObjectValue(request.getFlag(), Value.objectToValue(request.getDefaultValue()), context); - yield context.convertValue(objectValue); + yield client.getDoubleDetails(request.getFlag(), doubleEval.doubleValue(), context); } - default -> request.getDefaultValue(); + case "JSON" -> client.getObjectDetails(request.getFlag(), Value.objectToValue(request.getDefaultValue()), context); + default -> throw new IllegalArgumentException("Unsupported variation type: " + request.getVariationType()); }; - - reason = "DEFAULT"; + final Object value = details.getValue(); + result.put("value", value instanceof Value ? context.convertValue((Value) value) : value); + result.put("reason", details.getReason()); + result.put("variant", details.getVariant()); + result.put("errorCode", details.getErrorCode()); + result.put("errorMessage", details.getErrorMessage()); + if (details.getFlagMetadata() != null) { + result.put("allocationKey", details.getFlagMetadata().getString("allocationKey")); + result.put("serialId", details.getFlagMetadata().getInteger("__dd_split_serial_id")); + result.put("flagMetadata", details.getFlagMetadata().asUnmodifiableMap()); + } } catch (Throwable e) { LOGGER.error("Error on resolution", e); - value = request.getDefaultValue(); - reason = "ERROR"; + result.clear(); + result.put("value", request.getDefaultValue()); + result.put("reason", "ERROR"); + result.put("errorMessage", e.getMessage()); } - final Map result = new HashMap<>(); - result.put("reason", reason); - result.put("value", value); return ResponseEntity.ok(result); } @@ -123,6 +153,9 @@ private static EvaluationContext context(final EvaluateRequest request) { context.add(key, (Boolean) value); } else if (value instanceof Integer) { context.add(key, (Integer) value); + } else if (value instanceof Long) { + // OpenFeature represents numbers outside the Integer range as Double. + context.add(key, ((Long) value).doubleValue()); } else if (value instanceof Double) { context.add(key, (Double) value); } else if (value instanceof String) { diff --git a/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/featureflag/FeatureFlagEvaluatorController.java b/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/featureflag/FeatureFlagEvaluatorController.java index 16a2e6b8875..5b2ef06ed5e 100644 --- a/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/featureflag/FeatureFlagEvaluatorController.java +++ b/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/featureflag/FeatureFlagEvaluatorController.java @@ -6,6 +6,7 @@ import dev.openfeature.sdk.Client; import dev.openfeature.sdk.EvaluationContext; import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.FlagEvaluationDetails; import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.NoOpProvider; import dev.openfeature.sdk.OpenFeatureAPI; @@ -66,51 +67,58 @@ public ProviderState getState() { @PostMapping(value = "/ffe", consumes = APPLICATION_JSON_VALUE, produces = APPLICATION_JSON_VALUE) public ResponseEntity> evaluate(@RequestBody final EvaluateRequest request) { - Object value; - String reason; + final Map result = new HashMap<>(); final List targetingKeys = request.getTargetingKeys() == null || request.getTargetingKeys().isEmpty() ? List.of(request.getTargetingKey()) : request.getTargetingKeys(); try { - value = request.getDefaultValue(); for (final String targetingKey : targetingKeys) { + result.clear(); final EvaluationContext context = context(request, targetingKey); + final FlagEvaluationDetails details; switch (request.getVariationType()) { case "BOOLEAN": - value = client.getBooleanValue(request.getFlag(), (Boolean) request.getDefaultValue(), context); + details = client.getBooleanDetails(request.getFlag(), (Boolean) request.getDefaultValue(), context); break; case "STRING": - value = client.getStringValue(request.getFlag(), (String) request.getDefaultValue(), context); + details = client.getStringDetails(request.getFlag(), (String) request.getDefaultValue(), context); break; case "INTEGER": - value = client.getIntegerValue(request.getFlag(), (Integer) request.getDefaultValue(), context); + details = client.getIntegerDetails(request.getFlag(), (Integer) request.getDefaultValue(), context); break; case "NUMERIC": final Number number = (Number) request.getDefaultValue(); if (number instanceof Double) { - value = client.getDoubleValue(request.getFlag(), number.doubleValue(), context); + details = client.getDoubleDetails(request.getFlag(), number.doubleValue(), context); } else { - value = client.getIntegerValue(request.getFlag(), number.intValue(), context); + details = client.getIntegerDetails(request.getFlag(), number.intValue(), context); } break; case "JSON": - final Value objectValue = client.getObjectValue(request.getFlag(), Value.objectToValue(request.getDefaultValue()), context); - value = context.convertValue(objectValue); + details = client.getObjectDetails(request.getFlag(), Value.objectToValue(request.getDefaultValue()), context); break; default: - value = request.getDefaultValue(); + throw new IllegalArgumentException("Unsupported variation type: " + request.getVariationType()); + } + final Object value = details.getValue(); + result.put("value", value instanceof Value ? context.convertValue((Value) value) : value); + result.put("reason", details.getReason()); + result.put("variant", details.getVariant()); + result.put("errorCode", details.getErrorCode()); + result.put("errorMessage", details.getErrorMessage()); + if (details.getFlagMetadata() != null) { + result.put("allocationKey", details.getFlagMetadata().getString("allocationKey")); + result.put("serialId", details.getFlagMetadata().getInteger("__dd_split_serial_id")); + result.put("flagMetadata", details.getFlagMetadata().asUnmodifiableMap()); } } - - reason = "DEFAULT"; } catch (Throwable e) { LOGGER.error("Error on resolution", e); - value = request.getDefaultValue(); - reason = "ERROR"; + result.clear(); + result.put("value", request.getDefaultValue()); + result.put("reason", "ERROR"); + result.put("errorMessage", e.getMessage()); } - final Map result = new HashMap<>(); - result.put("reason", reason); - result.put("value", value); result.put("count", targetingKeys.size()); return ResponseEntity.ok(result); } @@ -123,6 +131,9 @@ private static EvaluationContext context(final EvaluateRequest request, final St context.add(key, (Boolean) value); } else if (value instanceof Integer) { context.add(key, (Integer) value); + } else if (value instanceof Long) { + // OpenFeature represents numbers outside the Integer range as Double. + context.add(key, ((Long) value).doubleValue()); } else if (value instanceof Double) { context.add(key, (Double) value); } else if (value instanceof String) { diff --git a/utils/docker_fixtures/_test_clients/_test_client_parametric.py b/utils/docker_fixtures/_test_clients/_test_client_parametric.py index e0107306095..c92fd86672e 100644 --- a/utils/docker_fixtures/_test_clients/_test_client_parametric.py +++ b/utils/docker_fixtures/_test_clients/_test_client_parametric.py @@ -805,7 +805,7 @@ def otel_current_span(self) -> _TestOtelSpan | None: return _TestOtelSpan(self, span_response["span_id"], span_response["trace_id"]) - def ffe_start(self, configuration: dict | None = None) -> bool: + def ffe_start(self, configuration: dict | None = None, *, provider_options: dict[str, bool] | None = None) -> bool: """Initialize the FFE (Feature Flagging & Experimentation) provider. Returns: @@ -815,6 +815,8 @@ def ffe_start(self, configuration: dict | None = None) -> bool: payload = {} if configuration is not None: payload["configuration"] = configuration + if provider_options is not None: + payload["providerOptions"] = provider_options resp = self._session.post(self._url("/ffe/start"), json=payload) return HTTPStatus(resp.status_code).is_success @@ -1258,9 +1260,9 @@ def write_log( ) -> bool: return self._client.write_log(logger_name, level, message, span_id=span_id) - def ffe_start(self, configuration: dict | None = None) -> bool: + def ffe_start(self, configuration: dict | None = None, *, provider_options: dict[str, bool] | None = None) -> bool: """Initialize the FFE (Feature Flagging & Experimentation) provider.""" - return self._client.ffe_start(configuration) + return self._client.ffe_start(configuration, provider_options=provider_options) def ffe_evaluate( self,