From 272d58c77de562f68d449ab50c5d896e81b31f84 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Tue, 13 Oct 2026 09:12:40 +0200 Subject: [PATCH] feat(ffe): Add agentless and standalone OpenFeature smoke tests Extract the shared OpenFeature smoke test scenarios and run them against the SDK alone, the SDK with the agent in agentless mode, and the SDK with the agent through Remote Configuration. Document the new SDK layout in ARCHITECTURE.md. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 14 +- .../AbstractOpenFeatureSmokeTest.groovy | 213 ++++++++++++++++++ .../OpenFeatureAgentlessSmokeTest.groovy | 24 ++ .../OpenFeatureProviderSmokeTest.groovy | 185 +-------------- .../OpenFeatureStandaloneSmokeTest.groovy | 35 +++ 5 files changed, 290 insertions(+), 181 deletions(-) create mode 100644 dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.groovy create mode 100644 dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.groovy create mode 100644 dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.groovy diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1b67fae9ceb..985b323d8c6 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -64,7 +64,8 @@ to `.classdata` to prevent unintended loading. See [`docs/how_to_work_with_gradl - **`agent-tooling/`** — Instrumentation framework. Key types: - `InstrumenterModule` — Base class for all instrumentation modules. Declares a target system - (Tracing, AppSec, IAST, Profiling, CiVisibility, USM, etc.) and one or more instrumentations. + (Tracing, AppSec, IAST, Profiling, CiVisibility, USM, FeatureFlags, etc.) and one or more + instrumentations. - `Instrumenter` — Type matching interface: `ForSingleType`, `ForKnownTypes`, `ForTypeHierarchy`, `ForBootstrap`. - `muzzle/` — Build-time and runtime safety checks. Verifies that expected types and methods @@ -229,9 +230,8 @@ Current products: - `metrics/` — StatsD client and monitoring abstraction. Provides `Monitoring` interface with counters, timers, and histograms for internal agent metrics collection. -- `feature-flagging/` — Server-side feature flag evaluation driven by remote configuration. - Implements the OpenFeature SDK, handles the Unified Feature Control (UFC) protocol, - and tracks flag exposure per user/session. +- `feature-flagging/` — Agent services for the [`dd-openfeature/`](#dd-openfeature) SDK: + remote configuration, event delivery through the Datadog Agent, and span enrichment. ### `communication/` @@ -279,6 +279,12 @@ that implements the `io.opentracing.Tracer` interface by wrapping the Datadog `C This is a pure library for manual instrumentation only — there is no auto-instrumentation or bytecode advice. +### `dd-openfeature/` + +Feature Flags OpenFeature provider. Publishes a standalone JAR artifact (`dd-openfeature.jar`, Java 11+) +that works without the agent. When the agent is attached, an instrumentation connects it to the +`products/feature-flagging/` agent services. + ### `dd-smoke-tests/` End-to-end smoke tests. Each boots a real application with the agent jar and verifies traces, spans, diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.groovy new file mode 100644 index 00000000000..147c6fa2fde --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/AbstractOpenFeatureSmokeTest.groovy @@ -0,0 +1,213 @@ +package datadog.smoketest.springboot + +import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer + +import datadog.smoketest.AbstractServerSmokeTest +import datadog.trace.agent.test.server.http.TestHttpServer +import datadog.trace.agent.test.server.http.TestHttpServer.HandlerApi.RequestApi +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import java.nio.file.Files +import java.nio.file.Paths +import okhttp3.MediaType +import okhttp3.Request +import okhttp3.RequestBody +import spock.lang.AutoCleanup +import spock.lang.IgnoreIf +import spock.lang.Shared +import spock.lang.Stepwise +import spock.lang.Unroll +import spock.util.concurrent.PollingConditions + +/** Due to the exposure cache it's important to run the tests in the specified order */ +@Stepwise +abstract class AbstractOpenFeatureSmokeTest extends AbstractServerSmokeTest { + + @Shared + protected final ufcConfig = new JsonSlurper().parse(fetchResource("ffe-system-test-data/ufc-config.json")) as Map + + @Shared + protected final ufcPayload = JsonOutput.toJson(ufcConfig) + + @Shared + protected final loggedAllocations = buildLoggedAllocations(ufcConfig) + + /** Serves the flag configuration as the Datadog managed CDN does. */ + @Shared + @AutoCleanup + protected TestHttpServer cdn = httpServer { + handlers { + prefix("/api/v2/feature-flagging/config/rules-based/server") { + response.status(200).send(JsonOutput.toJson([ + data: [type: 'universal-flag-configuration', attributes: ufcConfig] + ])) + } + } + } + + /** @return whether exposures are delivered through the Datadog Agent event platform proxy. */ + abstract boolean deliversExposures() + + /** Publishes the flag configuration before each evaluation. */ + void publishConfiguration() {} + + @Override + Closure decodedTracesCallback() { + return {} + } + + @Override + Closure decodedEvpProxyMessageCallback() { + return { String path, RequestApi request -> + if (!path.contains('api/v2/exposures')) { + return null + } + return new JsonSlurper().parse(request.body) + } + } + + @IgnoreIf({ !instance.deliversExposures() }) + void 'test open feature exposures'() { + setup: + publishConfiguration() + final testCases = parseTestCases() + assert !testCases.isEmpty() + + when: + final results = testCases.collect { + testCase -> + final response = evaluate(testCase) + final responseBody = new JsonSlurper().parse(response.body().byteStream()) + return [testCase: testCase, response: response, body: responseBody] + } + final expectedExposures = uniqueExpectedExposures(results) + + then: + results.every { + it.response.code() == 200 + } + !expectedExposures.isEmpty() + new PollingConditions(timeout: 10).eventually { + final requests = evpProxyMessages*.getV2() as List> + final events = requests*.exposures.flatten() + assert events.size() == expectedExposures.size() + expectedExposures.each { + expected -> + assert events.find { + event -> + event.flag.key == expected.flag && + event.allocation.key == expected.allocation && + event.variant.key == expected.variant && + event.subject.id == expected.targetingKey + } != null : "Unable to find exposure ${expected}" + } + } + } + + @Unroll("test open feature evaluation - #testCase.fileName[#testCase.index] - flag=#testCase.flag") + void 'test open feature evaluation'() { + setup: + publishConfiguration() + + when: + final response = evaluate(testCase) + + then: + response.code() == 200 + final responseBody = new JsonSlurper().parse(response.body().byteStream()) + responseBody.value == testCase.result.value + responseBody.reason == testCase.result.reason + if (testCase.result.containsKey('errorCode')) { + assert responseBody.errorCode == testCase.result.errorCode + } + if (testCase.result.containsKey('variant')) { + assert responseBody.variant == testCase.result.variant + } + if (testCase.result.flagMetadata?.allocationKey) { + assert responseBody.flagMetadata?.allocationKey == testCase.result.flagMetadata?.allocationKey + } + + where: + testCase << parseTestCases() + } + + protected okhttp3.Response evaluate(final Map testCase) { + final request = new Request.Builder() + .url("http://localhost:${httpPort}/openfeature/evaluate") + .post(RequestBody.create(MediaType.parse('application/json'), JsonOutput.toJson(testCase))) + .build() + return client.newCall(request).execute() + } + + protected static URL fetchResource(final String name) { + return Thread.currentThread().getContextClassLoader().getResource(name) + } + + protected static List> parseTestCases() { + final folder = fetchResource('ffe-system-test-data/evaluation-cases') + final uri = folder.toURI() + final testsPath = Paths.get(uri) + final files = Files.list(testsPath) + .filter(path -> path.toString().endsWith('.json')) + .sorted(Comparator.comparing(path -> path.fileName.toString())) + final result = [] + final slurper = new JsonSlurper() + files.each { + path -> + final testCases = slurper.parse(path.toFile()) as List> + testCases.eachWithIndex { + testCase, index -> + testCase.fileName = path.fileName.toString() + testCase.index = index + } + result.addAll(testCases) + } + assert !result.isEmpty() + return result + } + + protected List> uniqueExpectedExposures(final List> results) { + final expected = [] + final seen = [] as Set + results.each { result -> + final testCase = result.testCase as Map + final body = result.body as Map + final flag = testCase.flag as String + final allocation = body.flagMetadata?.allocationKey as String + final variant = body.variant as String + if (!variant || !allocation || !allocationLogs(flag, allocation)) { + return + } + + final exposure = [ + flag: flag, + allocation: allocation, + variant: variant, + targetingKey: testCase.targetingKey + ] + final key = "${exposure.flag}\u0000${exposure.targetingKey}\u0000${exposure.allocation}\u0000${exposure.variant}" + if (seen.add(key)) { + expected.add(exposure) + } + } + return expected + } + + protected boolean allocationLogs(final String flag, final String allocation) { + return loggedAllocations["${flag}\u0000${allocation}"] == true + } + + protected static Map buildLoggedAllocations(final Map config) { + final logged = [:] + (config.flags as Map).each { flag, definition -> + if (definition.allocations instanceof List) { + definition.allocations.each { allocation -> + if (allocation instanceof Map) { + logged["${flag}\u0000${allocation.key}"] = allocation.doLog == true + } + } + } + } + return logged + } +} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.groovy new file mode 100644 index 00000000000..bf7932f7ea8 --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureAgentlessSmokeTest.groovy @@ -0,0 +1,24 @@ +package datadog.smoketest.springboot + +/** + * The SDK with the Datadog Java agent and the default agentless configuration source: the SDK + * polls the flag configuration from the CDN and delivers exposures through the agent. + */ +class OpenFeatureAgentlessSmokeTest extends AbstractOpenFeatureSmokeTest { + + @Override + ProcessBuilder createProcessBuilder() { + final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") + final command = [javaPath()] + command.addAll(defaultJavaProperties) + command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) + final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) + builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL', "http://localhost:${cdn.address.port}".toString()) + return builder + } + + @Override + boolean deliversExposures() { + return true + } +} diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy index 12115cb407d..d0808be6aea 100644 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy +++ b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy @@ -2,36 +2,18 @@ package datadog.smoketest.springboot import datadog.remoteconfig.Capabilities import datadog.remoteconfig.Product -import datadog.smoketest.AbstractServerSmokeTest -import datadog.trace.agent.test.server.http.TestHttpServer.HandlerApi.RequestApi import groovy.json.JsonOutput import groovy.json.JsonSlurper -import java.nio.file.Files -import java.nio.file.Paths import okhttp3.MediaType import okhttp3.Request import okhttp3.RequestBody -import spock.lang.Shared -import spock.lang.Stepwise -import spock.lang.Unroll -import spock.util.concurrent.PollingConditions -/** Due to the exposure cache it's important to run the tests in the specified order */ -@Stepwise -class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest { - - @Shared - private final rcConfig = new JsonSlurper().parse(fetchResource("ffe-system-test-data/ufc-config.json")) as Map - - @Shared - private final rcPayload = JsonOutput.toJson(rcConfig) - - @Shared - private final loggedAllocations = buildLoggedAllocations(rcConfig) +/** The SDK with the Datadog Java agent, receiving the flag configuration from Remote Configuration. */ +class OpenFeatureProviderSmokeTest extends AbstractOpenFeatureSmokeTest { @Override ProcessBuilder createProcessBuilder() { - setRemoteConfig("datadog/2/FFE_FLAGS/1/config", rcPayload) + setRemoteConfig("datadog/2/FFE_FLAGS/1/config", ufcPayload) final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") final command = [javaPath()] @@ -48,18 +30,13 @@ class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest { } @Override - Closure decodedTracesCallback() { - return {} + boolean deliversExposures() { + return true } @Override - Closure decodedEvpProxyMessageCallback() { - return { String path, RequestApi request -> - if (!path.contains('api/v2/exposures')) { - return null - } - return new JsonSlurper().parse(request.body) - } + void publishConfiguration() { + setRemoteConfig("datadog/2/FFE_FLAGS/1/config", ufcPayload) } void 'test first remote config poll asks agent for feature flags'() { @@ -79,7 +56,7 @@ class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest { void 'test open feature evaluation enriches the request span'() { setup: - setRemoteConfig("datadog/2/FFE_FLAGS/1/config", rcPayload) + publishConfiguration() final request = new Request.Builder() .url("http://localhost:${httpPort}/openfeature/evaluate") .post(RequestBody.create(MediaType.parse('application/json'), JsonOutput.toJson([ @@ -105,152 +82,6 @@ class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest { response.close() } - void 'test open feature exposures'() { - setup: - setRemoteConfig("datadog/2/FFE_FLAGS/1/config", rcPayload) - final url = "http://localhost:${httpPort}/openfeature/evaluate" - final testCases = parseTestCases() - assert !testCases.isEmpty() - - when: - final results = testCases.collect { - testCase -> - final request = new Request.Builder() - .url(url) - .post(RequestBody.create(MediaType.parse('application/json'), JsonOutput.toJson(testCase))) - .build() - final response = client.newCall(request).execute() - final responseBody = new JsonSlurper().parse(response.body().byteStream()) - return [testCase: testCase, response: response, body: responseBody] - } - final expectedExposures = uniqueExpectedExposures(results) - - then: - results.every { - it.response.code() == 200 - } - !expectedExposures.isEmpty() - new PollingConditions(timeout: 10).eventually { - final requests = evpProxyMessages*.getV2() as List> - final events = requests*.exposures.flatten() - assert events.size() == expectedExposures.size() - expectedExposures.each { - expected -> - assert events.find { - event -> - event.flag.key == expected.flag && - event.allocation.key == expected.allocation && - event.variant.key == expected.variant && - event.subject.id == expected.targetingKey - } != null : "Unable to find exposure ${expected}" - } - } - } - - @Unroll("test open feature evaluation - #testCase.fileName[#testCase.index] - flag=#testCase.flag") - void 'test open feature evaluation'() { - setup: - setRemoteConfig("datadog/2/FFE_FLAGS/1/config", rcPayload) - final url = "http://localhost:${httpPort}/openfeature/evaluate" - final request = new Request.Builder() - .url(url) - .post(RequestBody.create(MediaType.parse('application/json'), JsonOutput.toJson(testCase))) - .build() - - when: - final response = client.newCall(request).execute() - - then: - response.code() == 200 - final responseBody = new JsonSlurper().parse(response.body().byteStream()) - responseBody.value == testCase.result.value - responseBody.reason == testCase.result.reason - if (testCase.result.containsKey('errorCode')) { - assert responseBody.errorCode == testCase.result.errorCode - } - if (testCase.result.containsKey('variant')) { - assert responseBody.variant == testCase.result.variant - } - if (testCase.result.flagMetadata?.allocationKey) { - assert responseBody.flagMetadata?.allocationKey == testCase.result.flagMetadata?.allocationKey - } - - where: - testCase << parseTestCases() - } - - private static URL fetchResource(final String name) { - return Thread.currentThread().getContextClassLoader().getResource(name) - } - - private static List> parseTestCases() { - final folder = fetchResource('ffe-system-test-data/evaluation-cases') - final uri = folder.toURI() - final testsPath = Paths.get(uri) - final files = Files.list(testsPath) - .filter(path -> path.toString().endsWith('.json')) - .sorted(Comparator.comparing(path -> path.fileName.toString())) - final result = [] - final slurper = new JsonSlurper() - files.each { - path -> - final testCases = slurper.parse(path.toFile()) as List> - testCases.eachWithIndex { - testCase, index -> - testCase.fileName = path.fileName.toString() - testCase.index = index - } - result.addAll(testCases) - } - assert !result.isEmpty() - return result - } - - private List> uniqueExpectedExposures(final List> results) { - final expected = [] - final seen = [] as Set - results.each { result -> - final testCase = result.testCase as Map - final body = result.body as Map - final flag = testCase.flag as String - final allocation = body.flagMetadata?.allocationKey as String - final variant = body.variant as String - if (!variant || !allocation || !allocationLogs(flag, allocation)) { - return - } - - final exposure = [ - flag: flag, - allocation: allocation, - variant: variant, - targetingKey: testCase.targetingKey - ] - final key = "${exposure.flag}\u0000${exposure.targetingKey}\u0000${exposure.allocation}\u0000${exposure.variant}" - if (seen.add(key)) { - expected.add(exposure) - } - } - return expected - } - - private boolean allocationLogs(final String flag, final String allocation) { - return loggedAllocations["${flag}\u0000${allocation}"] == true - } - - private static Map buildLoggedAllocations(final Map config) { - final logged = [:] - (config.flags as Map).each { flag, definition -> - if (definition.allocations instanceof List) { - definition.allocations.each { allocation -> - if (allocation instanceof Map) { - logged["${flag}\u0000${allocation.key}"] = allocation.doLog == true - } - } - } - } - return logged - } - private static Set decodeProducts(final Map request) { return request.client.products.collect { Product.valueOf(it) } } diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.groovy new file mode 100644 index 00000000000..86bbf6e41be --- /dev/null +++ b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureStandaloneSmokeTest.groovy @@ -0,0 +1,35 @@ +package datadog.smoketest.springboot + +/** + * The SDK without the Datadog Java agent: the SDK polls the flag configuration from the CDN. Without + * an API key, events are not delivered. + */ +class OpenFeatureStandaloneSmokeTest extends AbstractOpenFeatureSmokeTest { + + @Override + ProcessBuilder createProcessBuilder() { + final springBootShadowJar = System.getProperty("datadog.smoketest.springboot.shadowJar.path") + final command = [javaPath()] + command.addAll(nativeJavaProperties) + command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) + final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) + builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL', "http://localhost:${cdn.address.port}".toString()) + return builder + } + + @Override + boolean deliversExposures() { + return false + } + + @Override + boolean testTelemetry() { + return false + } + + /** Disables direct intake delivery, which would otherwise reach the real Datadog intake. */ + @Override + String apiKey() { + return '' + } +}