diff --git a/manifests/c.yml b/manifests/c.yml index 416336889bb..23631e4c4ff 100644 --- a/manifests/c.yml +++ b/manifests/c.yml @@ -4,6 +4,7 @@ manifest: tests/ai_guard/: irrelevant (dd-trace-c does not implement AI Guard) tests/appsec/: irrelevant (dd-trace-c does not implement AppSec or IAST) tests/cws/test_thread_context_sharing.py::Test_ThreadContextSharing: missing_feature (missing /security/thread_context_sharing endpoint on weblog) + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/integrations/test_dbm.py: incomplete_test_app (The Perl Mojolicious workload does not provide database endpoints) tests/parametric/otel_env_vars/test_otel_exporter_otlp_compression.py::Test_OTEL_EXPORTER_OTLP_COMPRESSION: missing_feature tests/parametric/otel_env_vars/test_otel_exporter_otlp_headers.py::Test_OTEL_EXPORTER_OTLP_HEADERS: missing_feature diff --git a/manifests/cpp.yml b/manifests/cpp.yml index fddd915329b..113a08ad890 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -22,6 +22,7 @@ manifest: tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_flag_eval_evp.py: irrelevant (only parametric tests are run for cpp) tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_sqs.py::Test_SQS_PROPAGATION_VIA_AWS_XRAY_HEADERS: irrelevant (Localstack SQS does not support AWS Xray Header parsing) diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index 47030f0bcc9..4b240565b9a 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -54,6 +54,7 @@ manifest: tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (C++ does not implement per-subject evaluation telemetry opt-out) tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 30648f6498b..566ddb4e656 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -287,6 +287,7 @@ manifest: tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature tests/ffe/test_agentless_configuration.py: missing_feature (C++ is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (C++ does not implement per-subject evaluation telemetry opt-out) tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index b1cc5602bd1..7075d94ea0e 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -822,6 +822,8 @@ manifest: weblog: [poc, uds] - declaration: bug (FFL-2184) excluded_weblog: [poc, uds] + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) + tests/ffe/test_evaluation_telemetry_opt_out.py::Test_FFE_Evaluation_Telemetry_Opt_Out_EVP: missing_feature (.NET has no EVP evaluation producer; FFL-2446) tests/ffe/test_exposure_egress.py: v3.36.0 tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: missing_feature (Not yet implemented) tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct_Shutdown: missing_feature (Not yet implemented) diff --git a/manifests/golang.yml b/manifests/golang.yml index 5ea6d7bde10..5cf8ae798a5 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1728,6 +1728,7 @@ manifest: weblog: [echo, gin, chi, net-http-orchestrion, net-http, uds-echo] - declaration: bug (FFL-2182) excluded_weblog: [echo, gin, chi, net-http-orchestrion, net-http, uds-echo] + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_exposure_egress.py: v2.6.0-dev # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.5.0 tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: v2.12.0-dev tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct_Shutdown: diff --git a/manifests/java.yml b/manifests/java.yml index e5369c6f8dd..5a4c759dbc2 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3476,6 +3476,7 @@ manifest: weblog: [spring-boot] - declaration: bug (FFL-2184) excluded_weblog: [spring-boot] + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/java_lambda.yml b/manifests/java_lambda.yml index 83478d4eba8..bb44387817a 100644 --- a/manifests/java_lambda.yml +++ b/manifests/java_lambda.yml @@ -61,6 +61,7 @@ manifest: tests/appsec/waf/test_blocking.py::Test_Blocking: missing_feature tests/appsec/waf/test_blocking.py::Test_Blocking_strip_response_headers: missing_feature tests/appsec/waf/test_blocking.py::Test_CustomBlockingResponse: missing_feature + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (This server weblog scenario does not apply to Lambda) tests/ffe/test_flag_eval_evp.py: irrelevant (server-side FFE EVP weblog scenario is not applicable to lambda) tests/test_resource_renaming.py: missing_feature tests/test_rum_injection.py: irrelevant (RUM injection only supported for Java) diff --git a/manifests/java_otel.yml b/manifests/java_otel.yml index 0652748e59d..3a149b7e01e 100644 --- a/manifests/java_otel.yml +++ b/manifests/java_otel.yml @@ -12,6 +12,7 @@ manifest: tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (OpenTelemetry test library does not implement Datadog feature flag providers) tests/ffe/test_flag_eval_evp.py: irrelevant (OpenTelemetry test library does not implement server-side FFE EVP) tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_sqs.py::Test_SQS_PROPAGATION_VIA_AWS_XRAY_HEADERS: irrelevant (Localstack SQS does not support AWS Xray Header parsing) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 11846c06b7f..38757bde8ca 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1871,6 +1871,7 @@ manifest: weblog: [express5] - declaration: bug (FFL-2182) excluded_weblog: [express5] + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": incomplete_test_app diff --git a/manifests/nodejs_lambda.yml b/manifests/nodejs_lambda.yml index f1ff3b763b9..3b0b4007d34 100644 --- a/manifests/nodejs_lambda.yml +++ b/manifests/nodejs_lambda.yml @@ -171,6 +171,7 @@ manifest: tests/appsec/waf/test_blocking.py::Test_Blocking: missing_feature tests/appsec/waf/test_blocking.py::Test_Blocking_strip_response_headers: missing_feature tests/appsec/waf/test_blocking.py::Test_CustomBlockingResponse: missing_feature + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (This server weblog scenario does not apply to Lambda) tests/ffe/test_flag_eval_evp.py: irrelevant (server-side FFE EVP weblog scenario is not applicable to lambda) tests/test_resource_renaming.py: missing_feature tests/test_rum_injection.py: irrelevant diff --git a/manifests/nodejs_otel.yml b/manifests/nodejs_otel.yml index 0852007bf5d..eaab8733ff0 100644 --- a/manifests/nodejs_otel.yml +++ b/manifests/nodejs_otel.yml @@ -11,6 +11,7 @@ manifest: tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (OpenTelemetry test library does not implement Datadog feature flag providers) tests/ffe/test_flag_eval_evp.py: irrelevant (OpenTelemetry test library does not implement server-side FFE EVP) tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_sqs.py::Test_SQS_PROPAGATION_VIA_AWS_XRAY_HEADERS: irrelevant (Localstack SQS does not support AWS Xray Header parsing) diff --git a/manifests/php.yml b/manifests/php.yml index 4d4f1a2f563..81f733302d6 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1240,6 +1240,7 @@ manifest: "*": v1.21.0-dev laravel11x: incomplete_test_app symfony7x: incomplete_test_app + tests/ffe/test_evaluation_telemetry_opt_out.py: incomplete_test_app (Deferred until the PHP OpenFeature test-app foundation is available) tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": v1.21.0-dev diff --git a/manifests/python.yml b/manifests/python.yml index 206a6a0988f..30bc0e539a2 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1499,6 +1499,7 @@ manifest: flask-poc: v4.14.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622) + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_exposure_egress.py: v4.2.0-dev tests/ffe/test_exposure_egress.py::Test_FFE_Exposure_Egress_Agentless_Direct: - weblog_declaration: diff --git a/manifests/python_lambda.yml b/manifests/python_lambda.yml index 39432f50af5..7bff96a9e68 100644 --- a/manifests/python_lambda.yml +++ b/manifests/python_lambda.yml @@ -305,6 +305,7 @@ manifest: tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (This server weblog scenario does not apply to Lambda) tests/ffe/test_flag_eval_evp.py: irrelevant (server-side FFE EVP weblog scenario is not applicable to lambda) tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_sqs.py::Test_SQS_PROPAGATION_VIA_AWS_XRAY_HEADERS: irrelevant (Localstack SQS does not support AWS Xray Header parsing) diff --git a/manifests/python_otel.yml b/manifests/python_otel.yml index 241d0d3c919..7cf144e0493 100644 --- a/manifests/python_otel.yml +++ b/manifests/python_otel.yml @@ -11,6 +11,7 @@ manifest: tests/debugger/test_debugger_exception_replay.py::Test_Debugger_Exception_Replay::test_exception_replay_stackoverflow: missing_feature (Implemented only for dotnet) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot: missing_feature (Not yet implemented) tests/debugger/test_debugger_probe_snapshot.py::Test_Debugger_Line_Probe_Snaphots::test_process_tags_snapshot_svc: missing_feature (Not yet implemented) + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (OpenTelemetry test library does not implement Datadog feature flag providers) tests/ffe/test_flag_eval_evp.py: irrelevant (OpenTelemetry test library does not implement server-side FFE EVP) tests/integration_frameworks/llm/anthropic/test_anthropic_llmobs.py::TestAnthropicLlmObsMessages::test_create_error: bug (MLOB-1234) tests/integrations/crossed_integrations/test_sqs.py::Test_SQS_PROPAGATION_VIA_AWS_XRAY_HEADERS: irrelevant (Localstack SQS does not support AWS Xray Header parsing) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 20f6ac2dc5d..15ddc98e7aa 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2137,6 +2137,7 @@ manifest: - weblog_declaration: "*": irrelevant rails72: v2.23.0-dev + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_exposure_egress.py: - weblog_declaration: "*": irrelevant diff --git a/manifests/ruby_lambda.yml b/manifests/ruby_lambda.yml index 89f6602b30a..cd183c66b17 100644 --- a/manifests/ruby_lambda.yml +++ b/manifests/ruby_lambda.yml @@ -85,4 +85,5 @@ manifest: tests/appsec/waf/test_blocking.py::Test_Blocking::test_html_template_v2: missing_feature tests/appsec/waf/test_blocking.py::Test_Blocking_strip_response_headers: v3.30.0 tests/appsec/waf/test_blocking.py::Test_CustomBlockingResponse: v3.29.0 + tests/ffe/test_evaluation_telemetry_opt_out.py: irrelevant (This server weblog scenario does not apply to Lambda) tests/ffe/test_flag_eval_evp.py: irrelevant (server-side FFE EVP weblog scenario is not applicable to lambda) diff --git a/manifests/rust.yml b/manifests/rust.yml index e62fd119f0e..8a997c48e96 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -23,6 +23,7 @@ manifest: tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature tests/ffe/test_agentless_configuration.py: missing_feature (Rust is not in scope for v1 agentless server SDK coverage; FFL-2731 tracks the system-tests contract) tests/ffe/test_dynamic_evaluation.py: missing_feature + tests/ffe/test_evaluation_telemetry_opt_out.py: missing_feature (Provisional per-subject evaluation telemetry opt-out coverage; enable after the SDK release) tests/ffe/test_exposure_egress.py: missing_feature (Not yet implemented) tests/ffe/test_exposures_datadog_agent.py: missing_feature tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) diff --git a/tests/ffe/README.md b/tests/ffe/README.md index be43490db3c..dc76851c649 100644 --- a/tests/ffe/README.md +++ b/tests/ffe/README.md @@ -11,6 +11,7 @@ This directory contains system tests for the Feature Flags & Experimentation (FF | `test_exposures_datadog_agent.py` | Flag exposure tracking and reporting through the Datadog Agent | | `test_flag_eval_metrics.py` | Evaluation metrics (OTel counter) | | `test_flag_eval_evp.py` | Server-side EVP flagevaluation payloads, aggregation, and bounds | +| `test_evaluation_telemetry_opt_out.py` | Per-subject evaluation telemetry opt-out, default-off behavior, and unchanged exposures | ## Running FFE Tests @@ -18,14 +19,48 @@ This directory contains system tests for the Feature Flags & Experimentation (FF ./run.sh FEATURE_FLAGGING_AND_EXPERIMENTATION --library ``` +## Per-subject evaluation telemetry opt-out + +All four test classes report under [#649: Per-subject evaluation telemetry opt-out](https://feature-parity.us1.prod.dog/#/?feature=649). + +Candidate SDK implementations must pass the applicable local checks before delivery. +Released-version CI activation is a separate step. Manifests keep the tests disabled until supported SDK versions are known. +Use a local candidate SDK and force these tests to validate unreleased changes: + +```bash +./run.sh FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT tests/ffe/test_evaluation_telemetry_opt_out.py \ + -F tests/ffe/test_evaluation_telemetry_opt_out.py +./run.sh FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT_DISABLED tests/ffe/test_evaluation_telemetry_opt_out.py \ + -F tests/ffe/test_evaluation_telemetry_opt_out.py +``` + +The enabled scenario sets `DD_FEATURE_FLAGS_EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED=true`. +The disabled scenario leaves the setting absent. Both use the same requests. + +| Context value | Enabled | Disabled | +| --- | --- | --- | +| Boolean `true` | No evaluation telemetry | Evaluation telemetry remains | +| Boolean `false`, missing, string `"true"`, or number `1` | Evaluation telemetry remains | Evaluation telemetry remains | + +Separate tests check EVP counts, evaluation metrics, and automatic trace annotations. +Each test includes allowed subjects as positive controls. +Mixed subjects evaluate the same flag to detect a flag-scoped opt-out decision. +One rule targets the opt-out attribute to verify that it remains available for evaluation. +All requests must return the same flag value. Exposures must retain their ordinary deduplication and context attributes. +Validation reads final captures after the scenario stops its application and drains the Agent. + +PHP coverage is deferred until its OpenFeature test-app foundation is available. +.NET has no EVP evaluation producer. Exclude the EVP class when forcing its local runs with `-k 'not EVP'`. +Browser and mobile SDKs are outside this server harness and retain their repository integration tests. + ## Test-contract self-tests -The self-tests in `testing_tests/` validate the FFE assertions using synthetic +The self-tests validate the FFE assertions using synthetic captures. They do not run SDKs or contact an intake. With the Python 3.12 runner environment installed, run them explicitly: ```bash -./run.sh FEATURE_FLAGGING_CONTRACT_TESTS tests/ffe/testing_tests +./run.sh FEATURE_FLAGGING_CONTRACT_TESTS tests/ffe/test_tests.py tests/ffe/test_evaluation_telemetry_opt_out_contract.py ``` This no-container scenario has no CI workflow or scenario-group membership. diff --git a/tests/ffe/test_evaluation_telemetry_opt_out.py b/tests/ffe/test_evaluation_telemetry_opt_out.py new file mode 100644 index 00000000000..75f1b5ca066 --- /dev/null +++ b/tests/ffe/test_evaluation_telemetry_opt_out.py @@ -0,0 +1,208 @@ +"""Per-subject opt-out suppresses evaluation telemetry, not flag results or exposures. + +Run the same assertions with support disabled (the default) and +enabled. Each signal has its own class so manifests can describe SDK support. +Validation reads the final captures after the scenario flushes and stops its +containers. Positive controls prevent missing telemetry from passing as opt-out. +""" + +from concurrent.futures import ThreadPoolExecutor +from typing import Any, TypedDict + +from tests.ffe.utils.evaluation_telemetry import assert_count, assert_evaluation_responses, evaluation_count +from tests.ffe.utils.exposures import exposure_events_from_data +from tests.ffe.utils.fixtures import make_ufc_fixture +from utils import HttpResponse, context, features, interfaces, remote_config as rc, rfc, scenarios, weblog + + +ATTRIBUTE = "dd_evaluation_telemetry_opt_out" +RFC = "https://docs.google.com/document/d/153tRlj46l-XfBGCA2X_NvxW4vwYObaH8zm2moP_AqMk/edit" +ATTRIBUTES: dict[str, dict[str, Any]] = { + "true": {ATTRIBUTE: True}, + "false": {ATTRIBUTE: False}, + "missing": {}, + "string-true": {ATTRIBUTE: "true"}, + "number-one": {ATTRIBUTE: 1}, +} +ANNOTATIONS = {"ffe_flags_enc", "ffe_subjects_enc", "ffe_runtime_defaults"} + + +class EvaluationCase(TypedDict): + flag: str + subject: str + attributes: dict[str, Any] + expected_count: int + responses: list[HttpResponse] + + +class _TelemetryContract: + cases: list[EvaluationCase] + + def prepare(self, signal: str) -> None: + enabled = context.scenario == scenarios.feature_flagging_evaluation_telemetry_opt_out + self.cases = [] + fixture = make_ufc_fixture(f"opt-out-{signal}-true", serial_id=101) + fixture["flags"] = {} + for index, (label, attributes) in enumerate(ATTRIBUTES.items()): + flag = f"opt-out-{signal}-{label}" + fixture["flags"].update(make_ufc_fixture(flag, serial_id=101 + index)["flags"]) + if label == "true": + allocations = fixture["flags"][flag]["allocations"] + allocations[0]["rules"] = [ + {"conditions": [{"attribute": ATTRIBUTE, "operator": "ONE_OF", "value": ["true"]}]} + ] + # Removing the attribute before evaluation selects off-value, + # even if the original context is retained for exposure logging. + allocations.extend( + make_ufc_fixture(flag, "off", allocation_key="fallback", serial_id=201)["flags"][flag][ + "allocations" + ] + ) + self.cases.append( + EvaluationCase( + flag=flag, + subject=f"subject-{signal}-{label}", + attributes=attributes, + expected_count=0 if enabled and label == "true" else 3, + responses=[], + ) + ) + + independent_cases = list(self.cases) + shared_flag = f"opt-out-{signal}-shared" + fixture["flags"].update(make_ufc_fixture(shared_flag, serial_id=106)["flags"]) + shared_cases = [ + EvaluationCase( + flag=shared_flag, + subject=f"subject-{signal}-shared-{label}", + attributes={ATTRIBUTE: opted_out}, + expected_count=0 if enabled and opted_out else 3, + responses=[], + ) + for label, opted_out in [("true", True), ("false", False)] + ] + self.cases.extend(shared_cases) + + rc.tracer_rc_state.reset().set_config(f"datadog/2/FFE_FLAGS/opt-out-{signal}/config", fixture).apply() + + # Different subjects overlap. Repeated evaluations also exercise aggregation + # and the ordinary exposure cache without changing the targeting context. + def evaluate(case: EvaluationCase) -> None: + response = weblog.post( + "/ffe", + json={ + "flag": case["flag"], + "targetingKey": case["subject"], + "attributes": case["attributes"], + "variationType": "STRING", + "defaultValue": "default", + }, + ) + case["responses"].append(response) + + with ThreadPoolExecutor(max_workers=len(independent_cases)) as executor: + for _ in range(3): + list(executor.map(evaluate, independent_cases)) + + # Alternate subjects on one flag. A flag-scoped opt-out cache must not + # suppress the allowed subject or permit the opted-out subject. + for _ in range(3): + for case in shared_cases: + evaluate(case) + + def assert_evaluations(self) -> None: + for case in self.cases: + assert_evaluation_responses(case["responses"], expected_count=3, expected_value="on-value") + + def expected_counts(self) -> dict[str, int]: + counts: dict[str, int] = {} + for case in self.cases: + counts[case["flag"]] = counts.get(case["flag"], 0) + case["expected_count"] + return counts + + +@scenarios.feature_flagging_evaluation_telemetry_opt_out_disabled +@scenarios.feature_flagging_evaluation_telemetry_opt_out +@features.feature_flags_evaluation_telemetry_opt_out +@rfc(RFC) +class Test_FFE_Evaluation_Telemetry_Opt_Out_EVP(_TelemetryContract): + def setup_counts(self) -> None: + self.prepare("evp") + + def test_counts(self) -> None: + self.assert_evaluations() + captures = list(interfaces.agent.get_data(path_filters="/api/v2/flagevaluation")) + for flag, expected_count in self.expected_counts().items(): + assert_count(evaluation_count(captures, flag), expected_count, flag) + + +@scenarios.feature_flagging_evaluation_telemetry_opt_out_disabled +@scenarios.feature_flagging_evaluation_telemetry_opt_out +@features.feature_flags_evaluation_telemetry_opt_out +@rfc(RFC) +class Test_FFE_Evaluation_Telemetry_Opt_Out_Metrics(_TelemetryContract): + def setup_counts(self) -> None: + self.prepare("metrics") + + def test_counts(self) -> None: + self.assert_evaluations() + for flag, expected_count in self.expected_counts().items(): + total = 0.0 + for _, metric in interfaces.agent.get_metrics(): + if metric.get("metric") != "feature_flag.evaluations": + continue + if f"feature_flag.key:{flag}" not in metric.get("tags", []): + continue + for point in metric.get("points", []): + total += float(point["value"] if isinstance(point, dict) else point[1]) + assert_count(total, expected_count, flag) + + +@scenarios.feature_flagging_evaluation_telemetry_opt_out_disabled +@scenarios.feature_flagging_evaluation_telemetry_opt_out +@features.feature_flags_evaluation_telemetry_opt_out +@rfc(RFC) +class Test_FFE_Evaluation_Telemetry_Opt_Out_Exposures(_TelemetryContract): + def setup_unchanged(self) -> None: + self.prepare("exposures") + + def test_unchanged(self) -> None: + self.assert_evaluations() + captures = list(interfaces.agent.get_data(path_filters="/api/v2/exposures")) + for case in self.cases: + events = [ + event for data in captures for event in exposure_events_from_data(data, {case["flag"]}, case["subject"]) + ] + assert len(events) == 1, f"{case['flag']}: expected one deduplicated exposure, got {events}" + assert events[0]["variant"]["key"] == "on" + attributes = events[0]["subject"].get("attributes", {}) + # Some providers also include targetingKey in exposure attributes. + # Check the customer input without forbidding that existing behavior. + if ATTRIBUTE in case["attributes"]: + expected = case["attributes"][ATTRIBUTE] + assert attributes[ATTRIBUTE] == expected + if isinstance(expected, bool): + assert isinstance(attributes[ATTRIBUTE], bool) + else: + assert ATTRIBUTE not in attributes + + +@scenarios.feature_flagging_evaluation_telemetry_opt_out_disabled +@scenarios.feature_flagging_evaluation_telemetry_opt_out +@features.feature_flags_evaluation_telemetry_opt_out +@rfc(RFC) +class Test_FFE_Evaluation_Telemetry_Opt_Out_Trace_Annotations(_TelemetryContract): + def setup_annotations(self) -> None: + self.prepare("traces") + + def test_annotations(self) -> None: + self.assert_evaluations() + for case in self.cases: + for response in case["responses"]: + spans = [span for _, _, span in interfaces.library.get_spans(request=response, full_trace=True)] + assert spans, f"{case['flag']}: unrelated application tracing must remain enabled" + annotated = [span for span in spans if ANNOTATIONS.intersection(span.get("meta", {}))] + if case["expected_count"]: + assert annotated, f"{case['flag']}: missing positive-control flag annotation" + else: + assert not annotated, f"{case['flag']}: opted-out evaluation annotated a trace" diff --git a/tests/ffe/test_evaluation_telemetry_opt_out_contract.py b/tests/ffe/test_evaluation_telemetry_opt_out_contract.py new file mode 100644 index 00000000000..94d91e05782 --- /dev/null +++ b/tests/ffe/test_evaluation_telemetry_opt_out_contract.py @@ -0,0 +1,62 @@ +"""Self-tests ensure the opt-out oracle rejects leaks and missing positive controls.""" + +from typing import Any +from tests.ffe.utils.evaluation_telemetry import assert_count, assert_evaluation_responses, evaluation_count +from utils import HttpResponse, features, pytest, scenarios + + +@scenarios.feature_flagging_contract_tests +@features.not_reported +@pytest.mark.parametrize( + ("status_code", "text", "count", "passes"), + [ + (200, '{"value": "on-value"}', 3, True), + (500, '{"value": "on-value"}', 3, False), + (200, '{"value": "default"}', 3, False), + (200, "not JSON", 3, False), + (200, "{}", 3, False), + (200, '{"value": "on-value"}', 0, False), + ], +) +def test_evaluation_response_oracle(status_code: int, text: str, count: int, *, passes: bool) -> None: + response = HttpResponse( + { + "request": {"method": "POST", "url": "http://weblog/ffe", "params": None}, + "status_code": status_code, + "text": text, + "cookies": {}, + } + ) + responses = [response] * count + if passes: + assert_evaluation_responses(responses, expected_count=3, expected_value="on-value") + else: + with pytest.raises((AssertionError, ValueError, KeyError)): + assert_evaluation_responses(responses, expected_count=3, expected_value="on-value") + + +@scenarios.feature_flagging_contract_tests +@features.not_reported +@pytest.mark.parametrize( + ("suppressed_count", "allowed_count", "passes"), + [(0, 3, True), (3, 3, False), (0, 0, False), (0, 6, False)], +) +def test_opt_out_evp_oracle(suppressed_count: int, allowed_count: int, *, passes: bool) -> None: + events = [ + {"flag": {"key": flag}, "evaluation_count": count} + for flag, count in [("suppressed", suppressed_count), ("allowed", allowed_count)] + if count + ] + captures: list[dict[str, Any]] = [ + {"path": "/api/v2/flagevaluation", "request": {"content": {"flagEvaluations": events}}} + ] + + def validate() -> None: + assert_count(evaluation_count(captures, "suppressed"), 0, "suppressed") + assert_count(evaluation_count(captures, "allowed"), 3, "allowed") + + if passes: + validate() + else: + with pytest.raises(AssertionError, match="expected.*evaluations"): + validate() diff --git a/tests/ffe/utils/evaluation_telemetry.py b/tests/ffe/utils/evaluation_telemetry.py new file mode 100644 index 00000000000..28aba38c5c6 --- /dev/null +++ b/tests/ffe/utils/evaluation_telemetry.py @@ -0,0 +1,30 @@ +"""Validate evaluation responses and telemetry after the scenario stops.""" + +import json +from collections.abc import Iterable +from typing import Any + +from utils import HttpResponse + + +def assert_evaluation_responses(responses: list[HttpResponse], *, expected_count: int, expected_value: str) -> None: + assert len(responses) == expected_count, f"Expected {expected_count} evaluation responses, got {len(responses)}" + for response in responses: + assert response.status_code == 200, response.text + assert json.loads(response.text)["value"] == expected_value, response.text + + +def assert_count(actual: float, expected: int, flag: str) -> None: + assert actual == expected, f"{flag}: expected {expected} evaluations, captured {actual}" + + +def evaluation_count(captures: Iterable[dict[str, Any]], flag: str) -> int: + total = 0 + for data in captures: + content = data.get("request", {}).get("content", {}) + if not isinstance(content, dict): + continue + for event in content.get("flagEvaluations", []): + if event.get("flag", {}).get("key") == flag: + total += event["evaluation_count"] + return total diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index efe42703db9..b7870d22b70 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -62,6 +62,18 @@ "DD_APP_KEY": "mock_app_key", } +# Keep the opt-out scenario identical to the ordinary FFE scenario except for activation. +_FFE_WEBLOG_ENV: dict[str, str | None] = { + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS": "100", + "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", + "DD_METRICS_OTEL_ENABLED": "true", + "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": "http/protobuf", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://agent:4318/v1/metrics", + "OTEL_METRIC_EXPORT_INTERVAL": "1000", +} + class _Scenarios: todo = Scenario("TODO", doc="scenario that skips tests not yet executed", github_workflow=None) @@ -885,25 +897,35 @@ class _Scenarios: rc_api_enabled=True, # Allow final EVP batches to reach the backend after the weblog flushes and stops. agent_interface_timeout=15, + # Fail initial provider setup promptly; the fixture delivers RC after startup. + weblog_env=_FFE_WEBLOG_ENV, + doc="", + scenario_groups=[scenario_groups.ffe], + ) + + feature_flagging_evaluation_telemetry_opt_out = DdTraceEndToEndScenario( + "FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT", + rc_api_enabled=True, + agent_interface_timeout=15, weblog_env={ - "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": "true", - "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": "remote_config", - # set_provider() in Python blocks until we receive RC - # configuration for feature flags. But it is only sent - # after weblog sucessfully boots and tests start - # executing. Unfortunately, Python's OpenFeature SDK does - # not have "set provider and don't wait," so we reduce the - # timeout here, so that the provider initialization fails - # fast, weblog boots, and provider recovers when we set RC - # configuration later. - "DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS": "100", - "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS": "0.2", - "DD_METRICS_OTEL_ENABLED": "true", - "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": "http/protobuf", - "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://agent:4318/v1/metrics", - "OTEL_METRIC_EXPORT_INTERVAL": "1000", + **_FFE_WEBLOG_ENV, + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": "true", + "DD_FEATURE_FLAGS_EVALUATION_TELEMETRY_PER_SUBJECT_OPT_OUT_ENABLED": "true", }, - doc="", + doc="Per-subject evaluation telemetry opt-out, with unchanged assignments and exposures.", + scenario_groups=[scenario_groups.ffe], + ) + + feature_flagging_evaluation_telemetry_opt_out_disabled = DdTraceEndToEndScenario( + "FEATURE_FLAGGING_EVALUATION_TELEMETRY_OPT_OUT_DISABLED", + rc_api_enabled=True, + agent_interface_timeout=15, + weblog_env={ + **_FFE_WEBLOG_ENV, + "DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED": "true", + # Omit the opt-out option: an attribute must not activate the feature by itself. + }, + doc="Default-disabled control for per-subject evaluation telemetry opt-out.", scenario_groups=[scenario_groups.ffe], ) diff --git a/utils/_features.py b/utils/_features.py index 69a5e411a0e..9054138dace 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2726,6 +2726,14 @@ def feature_flags_serial_id_metadata(test_object): """ return _mark_test_object(test_object, feature_id=648, owner=_Owner.ffe) + @staticmethod + def feature_flags_evaluation_telemetry_opt_out(test_object): + """Per-subject evaluation telemetry opt-out + + https://feature-parity.us1.prod.dog/#/?feature=649 + """ + return _mark_test_object(test_object, feature_id=649, owner=_Owner.ffe) + @staticmethod def appsec_extended_data_collection(test_object): """AppSec supports extended data collection including headers and body diff --git a/utils/build/docker/dotnet/poc.Dockerfile b/utils/build/docker/dotnet/poc.Dockerfile index 44884499673..4798efbcbc8 100644 --- a/utils/build/docker/dotnet/poc.Dockerfile +++ b/utils/build/docker/dotnet/poc.Dockerfile @@ -1,6 +1,10 @@ FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-app WORKDIR /app +# Pin local SDK packages when building an unreleased implementation. +ARG DD_TRACE_PACKAGE_VERSION=* +ARG DD_OPENFEATURE_PACKAGE_VERSION=* + # Copy binaries folder for local NuGet packages (for testing with local builds) COPY binaries/ /binaries/ RUN if ls /binaries/*.nupkg 1> /dev/null 2>&1; then \ @@ -9,11 +13,15 @@ RUN if ls /binaries/*.nupkg 1> /dev/null 2>&1; then \ # dotnet restore COPY utils/build/docker/dotnet/weblog/app.csproj app.csproj -RUN dotnet restore +RUN dotnet restore \ + -p:DatadogTracePackageVersion="$DD_TRACE_PACKAGE_VERSION" \ + -p:DatadogOpenFeaturePackageVersion="$DD_OPENFEATURE_PACKAGE_VERSION" # dotnet publish COPY utils/build/docker/dotnet/weblog/* . -RUN dotnet publish --no-restore -c Release -f net8.0 -o out +RUN dotnet publish --no-restore -c Release -f net8.0 -o out \ + -p:DatadogTracePackageVersion="$DD_TRACE_PACKAGE_VERSION" \ + -p:DatadogOpenFeaturePackageVersion="$DD_OPENFEATURE_PACKAGE_VERSION" ######### diff --git a/utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs b/utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs index 183cd1bcf9f..5aafeba2ff6 100644 --- a/utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs +++ b/utils/build/docker/dotnet/weblog/Controllers/FeatureFlagEvaluatorController.cs @@ -139,16 +139,22 @@ private static EvaluationContext CreateContext(EvaluateRequest request, string t { foreach (var attr in request.Attributes) { - // System.Text.Json deserializes to JsonElement, not string - var value = attr.Value switch + // Preserve primitive types so boolean context controls stay boolean. + switch (attr.Value) { - JsonElement jsonElement => jsonElement.ValueKind == JsonValueKind.String - ? jsonElement.GetString() - : jsonElement.ToString(), - string s => s, - _ => attr.Value?.ToString() - }; - builder.Set(attr.Key, value); + case JsonElement jsonElement when jsonElement.ValueKind is JsonValueKind.True or JsonValueKind.False: + builder.Set(attr.Key, jsonElement.GetBoolean()); + break; + case JsonElement jsonElement when jsonElement.ValueKind == JsonValueKind.Number: + builder.Set(attr.Key, jsonElement.GetDouble()); + break; + case JsonElement jsonElement when jsonElement.ValueKind == JsonValueKind.String: + builder.Set(attr.Key, jsonElement.GetString()); + break; + default: + builder.Set(attr.Key, attr.Value?.ToString()); + break; + } } } return builder.Build(); diff --git a/utils/build/docker/dotnet/weblog/app.csproj b/utils/build/docker/dotnet/weblog/app.csproj index 07e8575ae29..d141eec0841 100644 --- a/utils/build/docker/dotnet/weblog/app.csproj +++ b/utils/build/docker/dotnet/weblog/app.csproj @@ -8,6 +8,8 @@ false enable false + * + *