diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fbdc4604..de0eaf2e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,43 @@ on: - main jobs: + # Fast canary for the Provider TCK: runs the full applicable conformance suite against the + # SDK's InMemoryProvider, and again against MultiProvider wrapping one of them, with no + # Docker, no Compose stack and no network. It finishes in seconds, so a broken step + # definition, a mis-wired capability gate or a regression in the shared harness is reported + # long before the containerised provider suites in `main` get there — and it points at the + # TCK rather than at whichever provider noticed first. + # + # Deliberately not a gate on `main`: the two run in parallel so a green run is not delayed. + # The same suite also runs inside `main` as part of the reactor build; this job exists to + # report it fast and in isolation. + provider-tck: + name: Provider TCK (no Docker) + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + # No submodules: this module's feature files, flags and control-API spec are in-repo. + + - name: Set up JDK 21 + uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5 + with: + java-version: 21 + distribution: 'temurin' + cache: maven + + - name: Cache local Maven repository + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}21-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}21-maven- + + - name: Verify the TCK against the in-memory provider + # No `e2e` profile and no Docker: the in-memory suite is not gated behind either. + run: mvn --batch-mode --activate-profiles codequality -pl tools/provider-tck -am clean verify + main: strategy: matrix: diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java index 8c3f833e2..4b712a078 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/AbstractFlagdTckTest.java @@ -3,9 +3,9 @@ import dev.openfeature.contrib.providers.flagd.Config; import dev.openfeature.contrib.providers.flagd.FlagdOptions; import dev.openfeature.contrib.providers.flagd.FlagdProvider; -import dev.openfeature.contrib.tools.providertck.AbstractProviderTckTest; import dev.openfeature.contrib.tools.providertck.BackendEndpoint; import dev.openfeature.contrib.tools.providertck.Capability; +import dev.openfeature.contrib.tools.providertck.ContainerizedProviderTckTest; import dev.openfeature.sdk.FeatureProvider; import java.io.File; import java.util.Collections; @@ -24,7 +24,7 @@ * one is running from the JUnit test plan, so adding a mode needs no registration or build * configuration. */ -abstract class AbstractFlagdTckTest extends AbstractProviderTckTest { +abstract class AbstractFlagdTckTest extends ContainerizedProviderTckTest { /** * A port nothing listens on, for the initialisation-failure scenarios. diff --git a/tools/provider-tck/README.md b/tools/provider-tck/README.md index 6c14dd0b2..85e252682 100644 --- a/tools/provider-tck/README.md +++ b/tools/provider-tck/README.md @@ -23,7 +23,8 @@ contract" is an unverified claim. This is the shared suite that makes it checkab ``` -Requires Java 11+, JUnit 5, and a working Docker daemon. +Requires Java 11+ and JUnit 5. A working Docker daemon is needed only for providers with an +external backend — see [Which base class to extend](#which-base-class-to-extend). ### OpenFeature SDK compatibility @@ -55,8 +56,114 @@ uses only long-stable API — `OpenFeatureAPI`, `Client`, typed evaluation, `Pro - the provider↔backend wire protocol. How you talk to your backend is your business. - SDK behaviour. That belongs to the SDK's own test suite. +## Which base class to extend + +Two, and the choice is made by one question: **does your provider talk to something outside the +JVM?** + +| | Extend | Backend control | You supply | +|---|---|---|---| +| Provider has an external backend | `ContainerizedProviderTckTest` | `HttpBackendControl`, over the HTTP control API | a Compose stack, a control API, a test class | +| Provider has no backend — in-memory, environment variables, a local file | `ProviderTckTest` | an in-process `BackendControl` | a test class | + +`ContainerizedProviderTckTest` is the normal case and everything in [Adopting it](#adopting-it) +below describes it. It extends `ProviderTckTest` and adds the Compose lifecycle, port discovery and +control API client on top. + +### In-process control is for backend-less providers only + +Step definitions never touch a backend directly. They go through one interface, `BackendControl`, +which is what lets the same Gherkin run against a container over HTTP and against an in-memory +provider manipulated in the same JVM. + +That seam is not an invitation to skip the control API. **If your provider has an external backend, +use `HttpBackendControl` via `ContainerizedProviderTckTest`.** The control API described in +[`openapi/control-api.yaml`](src/main/resources/openapi/control-api.yaml) is the normative contract +for those providers, and it is the whole basis of a portable conformance claim: another language's +TCK drives the same endpoints against the same stack and must get the same answers. + +A custom in-JVM `BackendControl` that reaches an external backend through a side channel — a +test-only admin client, a shared database handle, a static hook inside the provider — bypasses that +contract. It will pass, and it will prove nothing, because the path it exercised is not the path the +contract describes. + +In-process control exists for providers that have **nothing to contract with**, where "the backend" +is a data structure in the same JVM. For those, flag operations are map updates and a configuration +change is the provider's own update mechanism emitting its own event. + +### Adopting it without a backend + +`InProcessBackendControl` implements this for the SDK's `InMemoryProvider`, seeded with the +canonical flag set. The entire adoption is three methods: + +```java +public class MyProviderTckTest extends ProviderTckTest { + + private final InProcessBackendControl control = new InProcessBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public FeatureProvider createProvider() { + return control.createProvider(); + } + + @Override + public Set capabilities() { + return EnumSet.of( + Capability.EVENTS, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.STRICT_NUMERIC_TYPING); + } +} +``` + +One object backs both factory methods because in-process the flag store and the provider are the +same thing: `changeFlag()` has to reach the live provider instance to emit an event from it. + +**Connection control does not apply**, and the capability declaration is where you say so rather +than stubbing it out. An in-memory provider has no connection to lose, so +`InProcessBackendControl` leaves `disconnect()`, `reconnect()` and `disconnectFor()` unimplemented — +they throw. Leaving `STALE` and `UNAVAILABLE_INIT` out of `capabilities()` is what keeps that +honest: the scenarios needing them are skipped before any step can reach an unsupported operation. + +Get that pairing wrong — declare `STALE` against a control that cannot disconnect — and you get an +`UnsupportedOperationException` naming the fix, not a silent pass. That is deliberate. A +`BackendControl` may throw `UnsupportedOperationException` for operations it does not support, and +reaching one from a scenario that actually ran is a **test-configuration bug**, never a skip. + +### The TCK's own self-tests + +Two suites in this module are exactly the class above, and both run with no Docker in well under a +second. They are the reference adoption, and they are the fast CI canary. + +[`InMemoryProviderTckTest`](src/test/java/dev/openfeature/contrib/tools/providertck/InMemoryProviderTckTest.java) +runs the full applicable suite against the SDK's `InMemoryProvider` — 26 passed, 3 skipped by +capability. + +[`MultiProviderTckTest`](src/test/java/dev/openfeature/contrib/tools/providertck/MultiProviderTckTest.java) +runs it against `MultiProvider` wrapping **one** `InMemoryProvider`. A provider that delegates is +still a provider, and delegation is where the contract is easiest to drop: a variant that does not +survive the hop, a reason rewritten, an error code flattened, an event that never arrives. With a +single child the correct answer is precisely what the in-memory suite already asserts, so any +difference between the two suites is attributable to `MultiProvider` and nothing else. + +That suite has already paid for itself. It does **not** declare `CONFIGURATION_CHANGE`, because +`MultiProvider` extends `EventProvider` but never subscribes to its children — a child's +`PROVIDER_CONFIGURATION_CHANGED`, `PROVIDER_ERROR` and `PROVIDER_STALE` are all swallowed. Wrapping +a provider in a multi-provider silently costs you those events, with nothing in the API to hint at +it. That is a known SDK gap, +[open-feature/java-sdk#1882](https://github.com/open-feature/java-sdk/issues/1882) (gap 1, High), +which the suite reproduced from the outside — the gap was originally found by hand-comparing +implementations against the js-sdk reference. Everything else survives delegation unchanged. + ## Adopting it +This section describes a provider with an external backend — the common case. Four things to implement, then two small files. ### 1. A Docker Compose stack @@ -130,7 +237,7 @@ Two details are load-bearing: ### 4. The test class ```java -public class MyProviderTckTest extends AbstractProviderTckTest { +public class MyProviderTckTest extends ContainerizedProviderTckTest { @Override public File composeFile() { @@ -172,7 +279,7 @@ registration, no system property, no build configuration. Each class is its own own Compose stack, and they can share a base class: ```java -abstract class AbstractMyProviderTckTest extends AbstractProviderTckTest { +abstract class AbstractMyProviderTckTest extends ContainerizedProviderTckTest { protected abstract Mode mode(); // composeFile(), createProvider(), capabilities() ... shared here } @@ -218,10 +325,10 @@ green on scenarios it did not run is worse than no suite at all. | Capability | Tag | Meaning | |---|---|---| | `EVENTS` | `@events` | emits lifecycle events at all | -| `STALE` | `@stale` | enters `STALE` and emits `PROVIDER_STALE` on backend loss | +| `STALE` | `@stale` | enters `STALE` and emits `PROVIDER_STALE` on backend loss — *needs connection control* | | `CONFIGURATION_CHANGE` | `@configuration-change` | detects config changes, emits `PROVIDER_CONFIGURATION_CHANGED` | | `OBJECT` | `@object` | supports structured flag values | -| `UNAVAILABLE_INIT` | `@unavailable` | reports an error state instead of hanging on a dead backend | +| `UNAVAILABLE_INIT` | `@unavailable` | reports an error state instead of hanging on a dead backend — *needs connection control* | | `STRICT_NUMERIC_TYPING` | `@strict-numeric-typing` | does not coerce between integer and float | | `TARGETING` | `@targeting` | reserved, no scenarios yet | | `CACHING` | `@caching` | reserved, no scenarios yet | @@ -229,6 +336,12 @@ green on scenarios it did not run is worse than no suite at all. The default is every capability. **Narrow it, do not widen it**: start from the default, run the suite, and remove only what your provider genuinely cannot do. +`STALE` and `UNAVAILABLE_INIT` are the two that need a backend the provider can be cut off from. +They are what a backend-less provider leaves undeclared — see +[In-process control is for backend-less providers only](#in-process-control-is-for-backend-less-providers-only). +Declaring one against a `BackendControl` that cannot simulate an outage fails the scenario with an +`UnsupportedOperationException` naming the fix, rather than passing it. + ```java @Override public Set capabilities() { @@ -254,8 +367,8 @@ needs most of a poll interval. Every await timeout is therefore overridable. |---|---|---| | `eventTimeout()` | 12s | waiting for a provider event | | `readyTimeout()` | 30s | waiting for a provider to reach a lifecycle state | -| `startupTimeout()` | 60s | bringing the Compose stack up | -| `settleTime()` | 50ms | pause after a control API call | +| `startupTimeout()` | 60s | bringing the Compose stack up (`ContainerizedProviderTckTest` only) | +| `settleTime()` | 50ms | pause after a control API call (`ContainerizedProviderTckTest` only) | ```java @Override @@ -274,6 +387,9 @@ use the explicit `within {int}ms` step, which always wins. mvn test -Dtest=MyProviderTckTest ``` +A suite extending `ProviderTckTest` with in-process control needs no Docker and no network. A suite +extending `ContainerizedProviderTckTest` needs a working Docker daemon for its Compose stack. + Scenarios run **serially** and the suite enforces this, overriding any `cucumber.execution.parallel.enabled=true` in your module's `junit-platform.properties`. Control API state is global to the Compose stack, so concurrent scenarios corrupt each other — one scenario's @@ -335,6 +451,12 @@ consumers — the features stay on the classpath and stay inside the JAR. `@targeting` tag is reserved for context-passthrough scenarios once the gap above is closed. - **Caching.** Whether a stale provider keeps serving last-known values during an outage depends on whether it holds a local copy of the ruleset. The `@caching` tag is reserved; no scenarios yet. +- **Setting and removing individual flags.** `BackendControl` exposes `prepareScenario()` and + `changeFlag()` — reset to the canonical baseline, and mutate `changing-flag` — because those are + what the Gherkin needs and what the control API defines. Finer-grained `setFlag(key, value)` / + `removeFlag(key)` operations would need control API endpoints that do not exist yet, so adding + them to the interface would produce methods `HttpBackendControl` could not implement. They belong + to a control API revision, not to the Java seam. - **Hooks.** Not covered. - **Flag metadata.** The flagd harness has metadata scenarios; they are not yet ported. - **Multi-suite JVMs.** `TckRuntime` is static, so TCK suites run one at a time within a JVM fork. diff --git a/tools/provider-tck/pom.xml b/tools/provider-tck/pom.xml index 3de834808..ee5d7ce9e 100644 --- a/tools/provider-tck/pom.xml +++ b/tools/provider-tck/pom.xml @@ -25,11 +25,13 @@ provider-tck Language-agnostic conformance test suite (TCK) for OpenFeature providers. - Bundles the canonical Gherkin feature files, Cucumber step definitions and an - abstract JUnit Platform Suite base class that owns the full test lifecycle: - starting the vendor-supplied Docker Compose stack, discovering dynamically - mapped ports, driving the standardised control API and awaiting provider - events. Provider authors implement a single factory interface. + Bundles the canonical Gherkin feature files, Cucumber step definitions and + abstract JUnit Platform Suite base classes that own the full test lifecycle. + Providers with an external backend extend ContainerizedProviderTckTest, which + starts the vendor-supplied Docker Compose stack, discovers dynamically mapped + ports and drives the standardised HTTP control API. Providers with no backend + extend ProviderTckTest and supply in-process backend control. Provider authors + implement a small factory interface either way. https://openfeature.dev @@ -97,7 +99,7 @@ + OBJECT_FACTORY_PROPERTY_NAME constants used in ProviderTckTest --> io.cucumber cucumber-junit-platform-engine @@ -110,7 +112,7 @@ compile - org.junit.platform @@ -180,6 +182,21 @@ slf4j-api ${slf4j.version} + + + + org.junit.jupiter + junit-jupiter + + test + diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendControl.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendControl.java new file mode 100644 index 000000000..427f80d36 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendControl.java @@ -0,0 +1,119 @@ +package dev.openfeature.contrib.tools.providertck; + +import java.time.Duration; + +/** + * The single seam between the TCK's step definitions and whatever manipulates the backend. + * + *

Step definitions never talk to a backend directly. They talk to this interface, which is why + * the same Gherkin runs unchanged against a containerised backend driven over HTTP + * ({@link HttpBackendControl}) and against a provider manipulated in-process + * ({@link InProcessBackendControl}). Nothing below this line knows about ports, containers or + * transports. + * + *

Which implementation is right for your provider

+ * + *

If your provider talks to a backend — a server, a service, anything out of process — use + * {@link HttpBackendControl} by extending {@link ContainerizedProviderTckTest}. The HTTP control + * API in {@code openapi/control-api.yaml} is the normative contract for those providers, and it is + * what makes a conformance claim portable: another language's TCK drives the same endpoints against + * the same stack and must get the same answers. + * + *

Do not write a custom in-JVM {@code BackendControl} that reaches into an + * external backend through a side channel — a test-only admin client, a shared database handle, a + * static hook inside the provider. It will pass, and it will prove nothing, because the thing it + * exercised is not the thing the contract describes. + * + *

In-process control exists for providers that have no backend to contract with: + * in-memory, environment-variable and file-based providers, where "the backend" is a data structure + * in the same JVM. See {@link InProcessBackendControl}. + * + *

Operations a backend may not support

+ * + *

{@link #prepareScenario()} and {@link #changeFlag()} are mandatory: a backend that cannot reset + * itself or change a flag cannot run the suite at all. + * + *

The three connection operations are not. A provider with nothing to disconnect from leaves + * them at their defaults, which throw {@link UnsupportedOperationException}. That exception is a + * test-configuration bug, never a skip — the scenarios that need connection + * control are gated behind {@link Capability#STALE} and {@link Capability#UNAVAILABLE_INIT}, so + * reaching one of these defaults means a capability was declared that the backend cannot back up. + * Failing loudly there is deliberate: a silent no-op would report the scenario as passed. + * + * @see Capability + * @see ProviderTckTest + */ +public interface BackendControl { + + /** + * Brings the backend to the state every scenario starts from: reachable, with flag state at the + * baseline of the canonical flag set. + * + *

Called once before each scenario. This is the TCK's only isolation mechanism — scenarios + * share one backend for the whole suite, and containers are never restarted between them. + */ + void prepareScenario(); + + /** + * Mutates flag configuration so that a conforming provider observes a configuration change and + * resolves a different value for {@code changing-flag} afterwards. + * + *

Which value it changes to is deliberately unspecified; the suite asserts only that the + * resolved value differs from what it was before. + */ + void changeFlag(); + + /** + * Makes the backend unreachable for the rest of the scenario, without stopping any container. + * + * @throws UnsupportedOperationException if this backend has no connection to lose + */ + default void disconnect() { + throw unsupported("disconnect"); + } + + /** + * Makes the backend reachable again after {@link #disconnect()}, preserving flag state so the + * provider observes an availability change rather than a configuration change. + * + * @throws UnsupportedOperationException if this backend has no connection to restore + */ + default void reconnect() { + throw unsupported("reconnect"); + } + + /** + * Makes the backend unreachable for a bounded period, after which it comes back on its own. + * + * @param outage how long the backend stays unreachable + * @throws UnsupportedOperationException if this backend has no connection to lose + */ + default void disconnectFor(Duration outage) { + throw unsupported("disconnectFor"); + } + + /** + * Returns a short description of what is being controlled, for startup logging and for the + * failure messages of unsupported operations. + * + * @return a human-readable description of this backend control + */ + default String description() { + return getClass().getSimpleName(); + } + + /** + * Builds the exception the connection-control defaults throw. + * + * @param operation the operation that is not supported + * @return the exception to throw + */ + default UnsupportedOperationException unsupported(String operation) { + return new UnsupportedOperationException(description() + " does not support '" + operation + + "'. This is a test-configuration bug rather than a provider defect: a scenario " + + "needing connection control ran, so the harness declared Capability.STALE or " + + "Capability.UNAVAILABLE_INIT for a backend that cannot simulate an outage. " + + "Remove those capabilities from the harness, or supply a BackendControl that " + + "implements them."); + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendEndpoint.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendEndpoint.java index 40f7799cf..b3d89d587 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendEndpoint.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/BackendEndpoint.java @@ -4,7 +4,7 @@ /** * Addresses of the running backend stack, handed to - * {@link ProviderTckHarness#createProvider(BackendEndpoint)}. + * {@link ContainerizedProviderTckTest#createProvider(BackendEndpoint)}. * *

This type exists because external ports are only known after the Compose stack has * started. Compose stacks under test must not pin host ports — Docker assigns them dynamically, so @@ -53,7 +53,7 @@ public String host(String service) { * backend service. * * @param internalPort the container-internal port, as declared by - * {@link ProviderTckHarness#backendPorts()} + * {@link ContainerizedProviderTckTest#backendPorts()} * @return the host port the service is reachable on */ public int port(int internalPort) { @@ -64,7 +64,7 @@ public int port(int internalPort) { * Resolves the dynamically mapped host port for a container-internal port on a named service. * *

Use this for multi-service stacks — a proxy, an edge service, a sidecar. The service and - * port must have been declared via {@link ProviderTckHarness#additionalExposedPorts()}, + * port must have been declared via {@link ContainerizedProviderTckTest#additionalExposedPorts()}, * otherwise Testcontainers has not exposed it and this call fails. * * @param service the Compose service name diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/Capability.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/Capability.java index 8203db0d8..c1e9a4c0d 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/Capability.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/Capability.java @@ -17,6 +17,21 @@ * never as passed. Silently green scenarios would make a conformance suite worthless. * *

Scenarios with no capability tag are considered mandatory and always run. + * + *

The connection-dependent capabilities

+ * + *

{@link #STALE} and {@link #UNAVAILABLE_INIT} are the two that require a backend the provider + * can be cut off from. They are what a harness leaves undeclared when its {@link BackendControl} + * has no connection to control — an in-memory, environment-variable or file-based provider, where + * the backend is a data structure in the same JVM. Every step that would call + * {@link BackendControl#disconnect()}, {@link BackendControl#reconnect()} or + * {@link ProviderTckHarness#createUnavailableProvider()} lives in a scenario carrying one of these + * two tags, so undeclaring them skips those scenarios before an unsupported operation can be + * reached. + * + *

Getting that pairing wrong surfaces as an {@link UnsupportedOperationException} rather than a + * skip, which is deliberate: it means a capability was declared that the harness cannot back up, + * and that is a test-configuration bug. */ public enum Capability { diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ContainerizedProviderTckTest.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ContainerizedProviderTckTest.java new file mode 100644 index 000000000..63bfefe13 --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ContainerizedProviderTckTest.java @@ -0,0 +1,272 @@ +package dev.openfeature.contrib.tools.providertck; + +import dev.openfeature.sdk.FeatureProvider; +import java.io.File; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +/** + * Base JUnit Platform Suite for providers that talk to an external backend. + * + *

Adds everything {@link ProviderTckTest} deliberately leaves out: the Docker Compose lifecycle, + * discovery of dynamically mapped host ports, and construction of an {@link HttpBackendControl} + * against the backend's control API. This is the base class for the overwhelming majority of + * providers. + * + *

The HTTP control API described in {@code openapi/control-api.yaml} is the normative contract + * here, and that is the point: another language's TCK drives the same endpoints against the same + * stack and must get the same answers. Substituting a custom in-JVM {@link BackendControl} that + * manipulates an external backend through a side channel bypasses that contract — see + * {@link BackendControl} for why that is not an acceptable adoption path. + * + *

Provider authors implement three methods, optionally a fourth, and override the defaults their + * stack needs. The Compose stack is started once, before the first scenario, and + * stopped after the last one. It is never stopped or restarted in between: Testcontainers cannot + * reliably preserve dynamically mapped host ports across a container restart, so a restart would + * silently invalidate every provider already pointed at the old port. Backend unavailability is + * therefore always simulated inside the running stack through the control API. + * + *

Example — the entire adoption for a provider with one transport: + * + * 

{@code
+ * public class MyProviderTckTest extends ContainerizedProviderTckTest {
+ *
+ *     @Override
+ *     public File composeFile() {
+ *         return new File("src/test/resources/tck/docker-compose.yaml");
+ *     }
+ *
+ *     @Override
+ *     public List backendPorts() {
+ *         return Collections.singletonList(8013);
+ *     }
+ *
+ *     @Override
+ *     public FeatureProvider createProvider(BackendEndpoint endpoint) {
+ *         return new MyProvider(endpoint.host(), endpoint.port(8013));
+ *     }
+ *
+ *     @Override
+ *     public FeatureProvider createUnavailableProvider() {
+ *         return new MyProvider("localhost", 9999);
+ *     }
+ * }
+ * }
+ * + * @see ProviderTckTest + * @see HttpBackendControl + */ +public abstract class ContainerizedProviderTckTest extends ProviderTckTest { + + private static final Logger log = LoggerFactory.getLogger(ContainerizedProviderTckTest.class); + + private ComposeContainer compose; + private BackendEndpoint endpoint; + private HttpBackendControl control; + + // --------------------------------------------------------------------------------------- + // What a provider author supplies + // --------------------------------------------------------------------------------------- + + /** + * Returns the Docker Compose file describing the backend stack under test. + * + *

The path is resolved relative to the Maven module directory, so + * {@code new File("src/test/resources/tck/docker-compose.yaml")} is the idiomatic form. + * + *

The stack must not pin host ports — Docker assigns them dynamically and the TCK discovers + * them after startup. + * + * @return the Compose file describing the backend stack + */ + public abstract File composeFile(); + + /** + * Returns the container-internal ports on {@link #backendService()} that the provider connects + * to, so Testcontainers can expose and map them. + * + *

The control API port from {@link #controlPort()} is exposed automatically and does not + * need to be listed here. + * + * @return container-internal ports the provider connects to + */ + public abstract List backendPorts(); + + /** + * Creates the provider under test, configured against the running backend. + * + *

Called after the Compose stack is up and the control API has seeded the canonical flag + * set. The endpoint carries the dynamically mapped host ports, which is why this is a factory + * rather than a field: the ports do not exist until the stack has started. + * + *

The TCK owns the provider lifecycle from here. Do not call {@code setProvider} or + * {@code initialize} yourself. + * + * @param endpoint host and mapped ports of the running backend stack + * @return a configured, uninitialised provider + */ + public abstract FeatureProvider createProvider(BackendEndpoint endpoint); + + /** + * {@inheritDoc} + * + *

Delegates to {@link #createProvider(BackendEndpoint)} with the running stack's endpoint. + */ + @Override + public final FeatureProvider createProvider() { + return createProvider(endpoint()); + } + + /** + * {@inheritDoc} + * + *

Abstract here rather than defaulted: a provider with a real backend can always be pointed + * at a closed port, so there is no reason for one not to cover the initialisation-failure + * scenarios. + */ + @Override + public abstract FeatureProvider createUnavailableProvider(); + + /** + * Returns the Compose service name that hosts the control API and the backend the provider + * connects to. + * + * @return the Compose service name, {@code backend} by default + */ + public String backendService() { + return "backend"; + } + + /** + * Returns the container-internal port the control API listens on. + * + * @return the control API port, {@code 8080} by default + */ + public int controlPort() { + return 8080; + } + + /** + * Returns extra services and container-internal ports to expose, for stacks that contain more + * than the backend service. + * + *

Keys are Compose service names, values are container-internal ports. Resolve the mapped + * ports with {@link BackendEndpoint#port(String, int)}. + * + * @return additional services and ports to expose, empty by default + */ + public Map> additionalExposedPorts() { + return Collections.emptyMap(); + } + + /** + * Returns the control API configuration name used to seed the canonical flag set. + * + * @return the configuration name passed to {@code POST /start}, {@code default} by default + */ + public String defaultConfig() { + return "default"; + } + + /** + * Returns how long to wait for the Compose stack and its control API to become reachable. + * + * @return the stack startup timeout, 60 seconds by default + */ + public Duration startupTimeout() { + return Duration.ofSeconds(60); + } + + /** + * Returns how long to pause after a control API call before continuing. + * + *

Covers the gap between the control API acknowledging a command and the backend actually + * having acted on it. Raise it if you see flakiness immediately after + * {@code the flag was modified} or a provider setup step. + * + * @return the settle time, 50 milliseconds by default + */ + public Duration settleTime() { + return Duration.ofMillis(50); + } + + // --------------------------------------------------------------------------------------- + // The lifecycle-agnostic contract, implemented in terms of the Compose stack + // --------------------------------------------------------------------------------------- + + /** + * {@inheritDoc} + * + *

Starts the Compose stack, resolves the control API's mapped port and waits for it to + * accept commands. + */ + @Override + public final void startSuite() { + compose = startCompose(); + endpoint = new BackendEndpoint(compose, backendService()); + control = new HttpBackendControl( + "http://" + endpoint.host() + ":" + endpoint.port(controlPort()), defaultConfig(), settleTime()); + control.awaitReady(startupTimeout()); + log.info("Control API ready at {}", control.baseUrl()); + } + + /** {@inheritDoc} */ + @Override + public final void stopSuite() { + if (compose != null) { + compose.stop(); + } + compose = null; + endpoint = null; + control = null; + } + + /** {@inheritDoc} */ + @Override + public final BackendControl backendControl() { + return control; + } + + /** + * Returns the host and mapped ports of the running stack. + * + * @return the backend endpoint + * @throws IllegalStateException if the stack has not been started + */ + protected final BackendEndpoint endpoint() { + if (endpoint == null) { + throw new IllegalStateException("The Compose stack has not been started yet."); + } + return endpoint; + } + + private ComposeContainer startCompose() { + File composeFile = composeFile(); + if (!composeFile.isFile()) { + throw new IllegalStateException("Compose file not found: " + composeFile.getAbsolutePath() + + ". ContainerizedProviderTckTest.composeFile() is resolved relative to the module directory."); + } + ComposeContainer stack = new ComposeContainer(composeFile); + + stack.withExposedService(backendService(), controlPort(), Wait.forListeningPort()); + for (Integer port : backendPorts()) { + stack.withExposedService(backendService(), port, Wait.forListeningPort()); + } + for (Map.Entry> service : additionalExposedPorts().entrySet()) { + for (Integer port : service.getValue()) { + stack.withExposedService(service.getKey(), port, Wait.forListeningPort()); + } + } + stack.withStartupTimeout(startupTimeout()); + + log.info("Starting Compose stack {} (started once per suite, never restarted)", composeFile.getAbsolutePath()); + stack.start(); + return stack; + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ControlApiClient.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/HttpBackendControl.java similarity index 63% rename from tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ControlApiClient.java rename to tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/HttpBackendControl.java index 6c0a19e57..42183b52c 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ControlApiClient.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/HttpBackendControl.java @@ -10,38 +10,52 @@ import org.slf4j.LoggerFactory; /** - * Client for the standardised backend control API. + * {@link BackendControl} backed by the standardised HTTP control API. * - *

Implements the contract in {@code openapi/control-api.yaml}, including the documented fallback - * for the optional {@code /reset} operation. Uses the JDK HTTP client so that adopting the TCK does - * not drag an HTTP library onto a provider's test classpath. + *

This is the normative implementation for every provider that talks to an external backend. It + * implements the contract in {@code openapi/control-api.yaml}, including the documented fallback + * for the optional {@code /reset} operation. It uses the JDK HTTP client so that adopting the TCK + * does not drag an HTTP library onto a provider's test classpath. * *

Every operation here manipulates the backend process or its flag state. None of them * touch containers — that is the no-container-restart invariant, and it is the reason a provider * built once at suite start stays valid for every scenario. + * + *

Constructed by {@link ContainerizedProviderTckTest} once the Compose stack is up and the + * control API host port is known. Provider authors do not build one themselves. */ -public final class ControlApiClient { +public final class HttpBackendControl implements BackendControl { - private static final Logger log = LoggerFactory.getLogger(ControlApiClient.class); + private static final Logger log = LoggerFactory.getLogger(HttpBackendControl.class); private final HttpClient http; private final String baseUrl; + private final String defaultConfig; private final Duration settleTime; /** * Tri-state cache of whether the backend implements the optional {@code /reset} operation. - * {@code null} until the first {@link #reset(String)} call probes it. + * {@code null} until the first {@link #reset()} call probes it. */ private Boolean resetSupported; /** - * Whether the backend was last known to be unreachable. Conservative: {@code restart} sets it - * even though the backend comes back on its own, because a scenario may end before it does. + * Whether the backend was last known to be unreachable. Conservative: {@link #disconnectFor} + * sets it even though the backend comes back on its own, because a scenario may end before it + * does. */ private boolean backendStopped; - ControlApiClient(String baseUrl, Duration settleTime) { + /** + * Creates a control client for a running backend. + * + * @param baseUrl the control API base URL, without a trailing slash + * @param defaultConfig the configuration name defining the canonical baseline + * @param settleTime how long to pause after a command before continuing + */ + HttpBackendControl(String baseUrl, String defaultConfig, Duration settleTime) { this.baseUrl = baseUrl; + this.defaultConfig = defaultConfig; this.settleTime = settleTime; this.http = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); @@ -56,105 +70,80 @@ public String baseUrl() { return baseUrl; } - /** - * Starts the backend with a named configuration, seeding flag state to that configuration's - * baseline. - * - * @param config the configuration name - */ - public void start(String config) { - post("/start?config=" + config); - backendStopped = false; + @Override + public String description() { + return "HTTP control API at " + baseUrl; } /** - * Makes the backend unreachable without stopping its container. + * {@inheritDoc} + * + *

Prefers {@code POST /reset} when the backend is already running, because restoring the + * baseline without an availability blip means the previous scenario teardown cannot leak a + * spurious lifecycle event into the next scenario. When the previous scenario left the backend + * unreachable, {@code /reset} alone would not bring it back, so this falls through to + * {@code POST /start?config=...}. */ - public void stop() { - post("/stop"); - backendStopped = true; + @Override + public void prepareScenario() { + if (backendStopped) { + start(); + } else { + reset(); + } } /** - * Makes the backend unreachable for a bounded duration, then starts it again. - * - *

Flag state is preserved across the outage, so a provider observes an availability change - * and not a configuration change. + * {@inheritDoc} * - * @param seconds how long the backend stays unreachable + *

Issues {@code POST /change}. */ - public void restart(int seconds) { - post("/restart?seconds=" + seconds); - backendStopped = true; + @Override + public void changeFlag() { + post("/change"); } /** - * Puts the backend into the state every scenario starts from: running, with flag state at the - * baseline of the default configuration. + * {@inheritDoc} * - *

Prefers {@link #reset(String)} when the backend is already running, because restoring the - * baseline without an availability blip means the previous scenario's teardown cannot leak a - * spurious lifecycle event into the next scenario. When the previous scenario left the backend - * unreachable, {@code /reset} alone would not bring it back, so this falls through to - * {@link #start(String)}. - * - * @param defaultConfig the configuration name defining the baseline + *

Issues {@code POST /stop}, which makes the backend unreachable without stopping its + * container. */ - public void prepareScenario(String defaultConfig) { - if (backendStopped) { - start(defaultConfig); - } else { - reset(defaultConfig); - } + @Override + public void disconnect() { + post("/stop"); + backendStopped = true; } /** - * Mutates flag configuration so that a conforming provider observes a configuration change and - * resolves a different value for {@code changing-flag} afterwards. + * {@inheritDoc} + * + *

Issues {@code POST /start?config=...}, which also restores the baseline flag state. */ - public void change() { - post("/change"); + @Override + public void reconnect() { + start(); } /** - * Restores flag state to the seeded baseline for scenario isolation. - * - *

Prefers the optional {@code POST /reset}, which causes no availability blip. When the - * backend answers {@code 404} or {@code 501} the result is cached and every subsequent call - * falls back to {@code POST /start?config=...}, which resets state at the cost of a process - * restart. Both paths are conformant; see {@code openapi/control-api.yaml}. + * {@inheritDoc} * - * @param defaultConfig the configuration name to fall back to + *

Issues {@code POST /restart?seconds=...}. Flag state is preserved across the outage, so a + * provider observes an availability change and not a configuration change. The control API + * takes whole seconds, so a sub-second outage is rounded up to one second. */ - public void reset(String defaultConfig) { - if (Boolean.FALSE.equals(resetSupported)) { - start(defaultConfig); - return; - } - HttpResponse response = send("/reset"); - if (response.statusCode() == 404 || response.statusCode() == 501) { - if (resetSupported == null) { - log.info( - "Control API at {} does not implement POST /reset (HTTP {}); " - + "falling back to POST /start?config={} for scenario isolation.", - baseUrl, - response.statusCode(), - defaultConfig); - } - resetSupported = false; - start(defaultConfig); - return; - } - expectSuccess("/reset", response); - resetSupported = true; - settle(); + @Override + public void disconnectFor(Duration outage) { + int seconds = (int) Math.max(1, Math.ceil(outage.toMillis() / 1000.0)); + post("/restart?seconds=" + seconds); + backendStopped = true; } /** * Waits until the control API accepts commands. * - *

Probes the optional {@code GET /healthz}. A {@code 404} is a conformant answer meaning "not - * implemented", in which case readiness has already been established by the Testcontainers + *

Probes the optional {@code GET /healthz}. A {@code 404} is a conformant answer meaning + * "not implemented", in which case readiness has already been established by the Testcontainers * listening-port wait strategy and this returns immediately. * * @param timeout how long to keep probing @@ -185,6 +174,47 @@ public void awaitReady(Duration timeout) { throw new IllegalStateException("control API at " + baseUrl + " did not become ready within " + timeout, last); } + /** + * Starts the backend with the default configuration, seeding flag state to that configuration + * baseline. + */ + private void start() { + post("/start?config=" + defaultConfig); + backendStopped = false; + } + + /** + * Restores flag state to the seeded baseline without an availability blip. + * + *

{@code POST /reset} is optional. When the backend answers {@code 404} or {@code 501} the + * result is cached and every subsequent call falls back to {@code POST /start?config=...}, + * which resets state at the cost of a process restart. Both paths are conformant; see + * {@code openapi/control-api.yaml}. + */ + private void reset() { + if (Boolean.FALSE.equals(resetSupported)) { + start(); + return; + } + HttpResponse response = send("/reset"); + if (response.statusCode() == 404 || response.statusCode() == 501) { + if (resetSupported == null) { + log.info( + "Control API at {} does not implement POST /reset (HTTP {}); " + + "falling back to POST /start?config={} for scenario isolation.", + baseUrl, + response.statusCode(), + defaultConfig); + } + resetSupported = false; + start(); + return; + } + expectSuccess("/reset", response); + resetSupported = true; + settle(); + } + private void post(String path) { expectSuccess(path, send(path)); settle(); diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControl.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControl.java new file mode 100644 index 000000000..91ef2fb6f --- /dev/null +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControl.java @@ -0,0 +1,226 @@ +package dev.openfeature.contrib.tools.providertck; + +import dev.openfeature.sdk.MutableStructure; +import dev.openfeature.sdk.Value; +import dev.openfeature.sdk.providers.memory.Flag; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * {@link BackendControl} that manipulates the SDK's {@link InMemoryProvider} directly, with no + * backend, no container and no HTTP. + * + *

This exists so that providers with nothing to connect to — in-memory, environment-variable and + * file-based providers — can run the TCK. For those, "the backend" is a data structure in the same + * JVM: seeding flags is building a map, and changing one is + * {@link InMemoryProvider#updateFlag(String, Flag)}, which emits + * {@code PROVIDER_CONFIGURATION_CHANGED} through the provider's own event mechanism rather than + * through a simulated one. + * + *

This is not a shortcut for providers that do have a backend. Reaching into an + * external backend from inside the JVM — a test-only admin client, a shared database handle, a + * static hook in the provider — produces a suite that passes while proving nothing, because the + * path it exercised is not the path the contract describes. Those providers use + * {@link HttpBackendControl} via {@link ContainerizedProviderTckTest}, and the control API in + * {@code openapi/control-api.yaml} stays the normative contract. See {@link BackendControl}. + * + *

Connection control

+ * + *

{@link #disconnect()}, {@link #reconnect()} and {@link #disconnectFor} are not implemented, so + * they inherit the interface defaults and throw. An in-memory provider has no connection to lose, + * and pretending otherwise with a no-op would report {@code @stale} scenarios as passed. The + * harness instead leaves {@link Capability#STALE} and {@link Capability#UNAVAILABLE_INIT} + * undeclared, and those scenarios are reported as skipped. + * + *

Ownership of the provider

+ * + *

This class both seeds the flags and creates the provider that serves them, because in-process + * they are the same object: {@link #changeFlag()} has to reach the live provider instance to emit + * an event from it. A harness therefore wires both of its factory methods to one instance: + * + * 

{@code
+ * private final InProcessBackendControl control = new InProcessBackendControl();
+ *
+ * @Override
+ * public BackendControl backendControl() {
+ *     return control;
+ * }
+ *
+ * @Override
+ * public FeatureProvider createProvider() {
+ *     return control.createProvider();
+ * }
+ * }
+ */ +public final class InProcessBackendControl implements BackendControl { + + /** The flag {@link #changeFlag()} mutates, as defined by {@code flags/canonical-flags.json}. */ + private static final String CHANGING_FLAG = "changing-flag"; + + private static final String CHANGING_BASELINE = "foo"; + private static final String CHANGING_CHANGED = "bar"; + + /** + * The canonical flag set, never mutated after construction. + * + *

Scenario isolation depends on that: {@link InMemoryProvider} copies the map it is given, + * and {@code updateFlag} writes only to the provider's copy, so every provider handed out by + * {@link #createProvider()} starts from an untouched baseline. + */ + private final Map> baseline = canonicalFlags(); + + /** The provider serving the current scenario, or {@code null} between scenarios. */ + private InMemoryProvider current; + + /** Which variant {@code changing-flag} currently resolves to. */ + private String changingVariant = CHANGING_BASELINE; + + /** + * Creates the provider for the scenario about to run, seeded with the canonical flag set. + * + *

Each call returns a fresh instance over a fresh copy of the baseline, which is what makes + * {@link #prepareScenario()} nothing more than dropping the previous reference. + * + * @return a configured, uninitialised in-memory provider + */ + @SuppressFBWarnings( + value = "EI_EXPOSE_REP", + justification = "Handing out the live provider is the contract, not a leak: in-process " + + "the flag store and the provider are one object, and changeFlag() must reach " + + "the same instance the TCK registered in order to emit an event from it") + public InMemoryProvider createProvider() { + changingVariant = CHANGING_BASELINE; + current = new InMemoryProvider(new HashMap<>(baseline)); + return current; + } + + @Override + public String description() { + return "in-process control of " + InMemoryProvider.class.getSimpleName(); + } + + /** + * {@inheritDoc} + * + *

Drops the reference to the previous scenario's provider. That is the whole reset: the + * baseline map is never mutated, so the {@link #createProvider()} call that follows produces a + * provider already at the baseline. Clearing the reference rather than leaving it dangling + * means a scenario that manipulates flags without creating a provider fails with a clear + * message instead of mutating a provider that has already been shut down. + */ + @Override + public void prepareScenario() { + current = null; + } + + /** + * {@inheritDoc} + * + *

Flips {@code changing-flag} between its two variants through + * {@link InMemoryProvider#updateFlag(String, Flag)}, so the event the suite awaits is the + * provider's own {@code PROVIDER_CONFIGURATION_CHANGED} — carrying {@code changing-flag} in + * {@code flagsChanged} — and not a signal the TCK synthesised. + * + *

Alternating rather than assigning a fixed variant keeps repeated calls within one scenario + * meaningful; the suite asserts that the resolved value differs, not what it became. + */ + @Override + public void changeFlag() { + changingVariant = CHANGING_CHANGED.equals(changingVariant) ? CHANGING_BASELINE : CHANGING_CHANGED; + requireProvider().updateFlag(CHANGING_FLAG, changingFlag(changingVariant)); + } + + private InMemoryProvider requireProvider() { + if (current == null) { + throw new IllegalStateException("No in-memory provider exists for this scenario. In-process backend " + + "control manipulates the provider itself, so the scenario must create one — with " + + "'Given a stable provider' — before any step that changes flag state."); + } + return current; + } + + /** + * Builds the canonical flag set as {@link InMemoryProvider} flags. + * + *

Mirrors {@code flags/canonical-flags.json} entry for entry. The two load-bearing details + * from that file hold here too: {@code missing-flag} is absent, which is what the + * {@code FLAG_NOT_FOUND} scenario tests, and no flag carries a + * {@link dev.openfeature.sdk.providers.memory.ContextEvaluator}, so every evaluation reports + * reason {@code STATIC} as the feature files expect. + * + * @return the canonical flag set + */ + private static Map> canonicalFlags() { + Map> flags = new LinkedHashMap<>(); + + flags.put( + "boolean-flag", + Flag.builder() + .variant("on", true) + .variant("off", false) + .defaultVariant("on") + .build()); + + flags.put( + "string-flag", + Flag.builder() + .variant("greeting", "hi") + .variant("parting", "bye") + .defaultVariant("greeting") + .build()); + + flags.put( + "integer-flag", + Flag.builder() + .variant("one", 1) + .variant("ten", 10) + .defaultVariant("ten") + .build()); + + flags.put( + "float-flag", + Flag.builder() + .variant("tenth", 0.1) + .variant("half", 0.5) + .defaultVariant("half") + .build()); + + flags.put( + "object-flag", + Flag.builder() + .variant("empty", new Value(new MutableStructure())) + .variant( + "template", + new Value(new MutableStructure() + .add("showImages", true) + .add("title", "Check out these pics!") + .add("imagesPerPage", 100))) + .defaultVariant("template") + .build()); + + // A string flag, evaluated as a boolean by the TYPE_MISMATCH scenario. + flags.put( + "wrong-flag", + Flag.builder() + .variant("one", "uno") + .variant("two", "dos") + .defaultVariant("one") + .build()); + + flags.put(CHANGING_FLAG, changingFlag(CHANGING_BASELINE)); + + return Collections.unmodifiableMap(flags); + } + + private static Flag changingFlag(String defaultVariant) { + return Flag.builder() + .variant(CHANGING_BASELINE, CHANGING_BASELINE) + .variant(CHANGING_CHANGED, CHANGING_CHANGED) + .defaultVariant(defaultVariant) + .build(); + } +} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java index 1cc671faf..b7b76115a 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckHarness.java @@ -1,97 +1,86 @@ package dev.openfeature.contrib.tools.providertck; import dev.openfeature.sdk.FeatureProvider; -import java.io.File; import java.time.Duration; -import java.util.Collections; import java.util.EnumSet; -import java.util.List; -import java.util.Map; import java.util.Set; /** - * The complete contract a provider author implements to run the OpenFeature Provider TCK. + * The lifecycle-agnostic contract a provider author implements to run the OpenFeature Provider TCK. * - *

Four methods have no default and must be supplied. Everything else is a convention with a - * working default. If you find yourself needing to add lifecycle code, container handling or event - * plumbing to your implementation, that is a bug in the TCK's base class rather than something to - * work around here. + *

Two methods have no default: what provider to test, and what manipulates the backend it reads + * from. Everything else is a convention with a working default. Nothing here mentions containers, + * ports or HTTP — that belongs to {@link ContainerizedProviderTckTest}, which implements this + * interface in terms of a Compose stack. * - *

Implementations are discovered through {@link java.util.ServiceLoader}. Extend - * {@link AbstractProviderTckTest} — which implements this interface and carries all the Cucumber - * configuration — and register the concrete class in - * {@code META-INF/services/dev.openfeature.contrib.tools.providertck.ProviderTckHarness}. + *

Which base class to extend: * - *

Example — the entire adoption for a provider: + *

+ * + *

Implementations are discovered through the executing JUnit suite, and through + * {@link java.util.ServiceLoader} as a fallback. Extend one of the two base classes — each is both + * the JUnit suite and the harness — and no registration is needed. + * + *

Example — the entire adoption for a backend-less provider: * * 

{@code
- * public class MyProviderTckTest extends AbstractProviderTckTest {
+ * public class MyProviderTckTest extends ProviderTckTest {
  *
- *     @Override
- *     public File composeFile() {
- *         return new File("src/test/resources/tck/docker-compose.yaml");
- *     }
+ *     private final InProcessBackendControl control = new InProcessBackendControl();
  *
  *     @Override
- *     public List backendPorts() {
- *         return Collections.singletonList(8013);
+ *     public BackendControl backendControl() {
+ *         return control;
  *     }
  *
  *     @Override
- *     public FeatureProvider createProvider(BackendEndpoint endpoint) {
- *         return new MyProvider(endpoint.host(), endpoint.port(8013));
+ *     public FeatureProvider createProvider() {
+ *         return control.createProvider();
  *     }
  *
  *     @Override
- *     public FeatureProvider createUnavailableProvider() {
- *         return new MyProvider("localhost", 9999);
+ *     public Set capabilities() {
+ *         return EnumSet.of(Capability.EVENTS, Capability.CONFIGURATION_CHANGE, Capability.OBJECT);
  *     }
  * }
  * }
+ * + * @see ProviderTckTest + * @see ContainerizedProviderTckTest */ public interface ProviderTckHarness { /** - * Returns the Docker Compose file describing the backend stack under test. + * Creates the provider under test, configured against a backend that is already running and + * seeded with the canonical flag set. * - *

The path is resolved relative to the Maven module directory, so - * {@code new File("src/test/resources/tck/docker-compose.yaml")} is the idiomatic form. - * - *

The stack is started once before the first scenario and stopped after the last one. It is - * never restarted in between — see {@link #createProvider(BackendEndpoint)} and the - * no-container-restart invariant documented in {@code openapi/control-api.yaml}. The stack must - * not pin host ports. - * - * @return the Compose file describing the backend stack - */ - File composeFile(); - - /** - * Returns the container-internal ports on {@link #backendService()} that the provider connects - * to, so Testcontainers can expose and map them. + *

Called once per scenario. This is a factory rather than a field because a provider cannot + * always be configured before the suite starts — a Compose stack's host ports do not exist + * until it is up — and because each scenario gets its own provider instance. * - *

The control API port from {@link #controlPort()} is exposed automatically and does not - * need to be listed here. + *

The TCK owns the provider lifecycle from here: it registers the provider with the + * OpenFeature API under a scenario-scoped domain, waits for it to become ready, and shuts it + * down afterwards. Do not call {@code setProvider} or {@code initialize} yourself. * - * @return container-internal ports the provider connects to + * @return a configured, uninitialised provider */ - List backendPorts(); + FeatureProvider createProvider(); /** - * Creates the provider under test, configured against the running backend. + * Returns the seam through which the TCK manipulates the backend. * - *

Called after the Compose stack is up and the control API has seeded the canonical flag - * set. The endpoint carries the dynamically mapped host ports, which is why this is a factory - * rather than a field: the ports do not exist until the stack has started. - * - *

The TCK owns the provider lifecycle from here — it registers the provider with the - * OpenFeature API under a scenario-scoped domain, waits for it to become ready, and shuts it - * down afterwards. Do not call {@code setProvider} or {@code initialize} yourself. + *

Called after {@link #startSuite()}, so an implementation may build it there and return the + * same instance on every call. It must not be {@code null}. * - * @param endpoint host and mapped ports of the running backend stack - * @return a configured, uninitialised provider + * @return the backend control for this suite */ - FeatureProvider createProvider(BackendEndpoint endpoint); + BackendControl backendControl(); /** * Creates a provider pointed at a backend that does not exist. @@ -100,15 +89,26 @@ public interface ProviderTckHarness { * reach its backend settles into {@code ERROR} and emits {@code PROVIDER_ERROR} rather than * hanging or throwing out of {@code setProvider}. * - *

Point this at a closed port on localhost. Do not point it at the Compose stack — the stack - * must stay up and reachable, and simulated outages belong to the control API. + *

Point this at a closed port on localhost. Do not point it at the backend under test — that + * must stay up and reachable, and simulated outages belong to {@link BackendControl}. * *

Configure a short connection deadline. The scenario allows a bounded time for the error * event to arrive, and a provider with a 30-second connect timeout will not make it. * + *

Defaults to throwing, because a provider with no backend has no way to be unreachable. + * Such a harness leaves {@link Capability#UNAVAILABLE_INIT} undeclared and the scenarios that + * would call this are reported as skipped, so the default is never reached. Reaching it means a + * capability was declared that the harness cannot back up. + * * @return a configured provider that cannot reach a backend */ - FeatureProvider createUnavailableProvider(); + default FeatureProvider createUnavailableProvider() { + throw new UnsupportedOperationException(getClass().getName() + " does not implement " + + "createUnavailableProvider(). This is a test-configuration bug rather than a provider " + + "defect: an @unavailable scenario ran, so the harness declared " + + "Capability.UNAVAILABLE_INIT without supplying a provider that cannot reach its " + + "backend. Remove that capability, or implement this method."); + } /** * Declares which optional parts of the provider contract this provider supports. @@ -126,53 +126,27 @@ default Set capabilities() { } /** - * Returns the Compose service name that hosts the control API and the backend the provider - * connects to. - * - * @return the Compose service name, {@code backend} by default - */ - default String backendService() { - return "backend"; - } - - /** - * Returns the container-internal port the control API listens on. - * - * @return the control API port, {@code 8080} by default - */ - default int controlPort() { - return 8080; - } - - /** - * Returns extra services and container-internal ports to expose, for stacks that contain more - * than the backend service. - * - *

Keys are Compose service names, values are container-internal ports. Resolve the mapped - * ports with {@link BackendEndpoint#port(String, int)}. + * Prepares whatever must exist before the first scenario — a container stack, a temporary + * directory, a local server. * - * @return additional services and ports to expose, empty by default - */ - default Map> additionalExposedPorts() { - return Collections.emptyMap(); - } - - /** - * Returns the control API configuration name used to seed the canonical flag set. + *

Called once, before any scenario, and always paired with {@link #stopSuite()}. Defaults to + * doing nothing, which is right for a harness whose backend is a data structure in this JVM. * - * @return the configuration name passed to {@code POST /start}, {@code default} by default + *

{@link #backendControl()} is called immediately afterwards, so this is where to build it + * if it needs something that only exists once the suite has started. */ - default String defaultConfig() { - return "default"; + default void startSuite() { + // Nothing to start by default. } /** - * Returns how long to wait for the Compose stack to become reachable. + * Releases whatever {@link #startSuite()} created. * - * @return the stack startup timeout, 60 seconds by default + *

Called once, after the last scenario, and also if suite startup fails partway through, so + * it must tolerate being called when startup did not complete. */ - default Duration startupTimeout() { - return Duration.ofSeconds(60); + default void stopSuite() { + // Nothing to stop by default. } /** @@ -184,8 +158,8 @@ default Duration startupTimeout() { * interval before it notices. Set this to comfortably exceed your worst-case detection latency, * or the suite will report timeouts that are really just impatience. * - *

Individual scenarios can tighten this with the explicit - * {@code within {int}ms} step, which always wins over this value. + *

Individual scenarios can tighten this with the explicit {@code within {int}ms} step, which + * always wins over this value. * * @return the default event await timeout, 12 seconds by default */ @@ -201,17 +175,4 @@ default Duration eventTimeout() { default Duration readyTimeout() { return Duration.ofSeconds(30); } - - /** - * Returns how long to pause after a control API call before continuing. - * - *

Covers the gap between the control API acknowledging a command and the backend actually - * having acted on it. Raise it if you see flakiness immediately after - * {@code the flag was modified} or a provider setup step. - * - * @return the settle time, 50 milliseconds by default - */ - default Duration settleTime() { - return Duration.ofMillis(50); - } } diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/AbstractProviderTckTest.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckTest.java similarity index 51% rename from tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/AbstractProviderTckTest.java rename to tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckTest.java index 4f3304d94..f3ac3f43a 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/AbstractProviderTckTest.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/ProviderTckTest.java @@ -13,27 +13,39 @@ * infrastructure at all. The canonical feature files are packaged inside this JAR and selected from * the classpath, so consumers need no git submodule of their own. * - *

To adopt the TCK, extend this class, implement the four abstract methods of - * {@link ProviderTckHarness}, and register the concrete class in - * {@code src/test/resources/META-INF/services/dev.openfeature.contrib.tools.providertck.ProviderTckHarness}. + *

Which base class to extend

+ * + *

This one is lifecycle-agnostic: it starts nothing and knows nothing about how the backend is + * reached. Extend it directly when your provider has no backend — an in-memory, + * environment-variable or file-based provider — and supply an in-process {@link BackendControl} + * such as {@link InProcessBackendControl}. + * + *

When your provider talks to an external backend, extend {@link ContainerizedProviderTckTest} + * instead. It adds the Compose stack lifecycle, port discovery and {@link HttpBackendControl}, and + * the HTTP control API in {@code openapi/control-api.yaml} remains the normative contract for that + * conformance claim. In-process control is for backend-less providers only; an external backend + * driven through a custom in-JVM {@code BackendControl} bypasses that contract and proves nothing. + * + *

Serial execution

* *

Scenarios run serially, and this class enforces that rather than merely - * asking for it. Control API state — which flags are seeded, whether the backend is reachable — is - * global to the Compose stack, so concurrent scenarios corrupt each other: one scenario's - * {@code /start} restarts the backend underneath another's disconnect assertion. The failure looks - * like a flaky provider rather than a broken test, which makes it expensive to diagnose. + * asking for it. Backend state — which flags are seeded, whether the backend is reachable — is + * global to the suite, so concurrent scenarios corrupt each other: one scenario's reconnect + * restarts the backend underneath another's disconnect assertion. The failure looks like a flaky + * provider rather than a broken test, which makes it expensive to diagnose. * *

The suite therefore pins {@code cucumber.execution.parallel.enabled=false} here, where it * overrides any {@code junit-platform.properties} the consuming module happens to ship. Several * providers already enable Cucumber parallelism for their own suites, and inheriting that setting * silently breaks the TCK. * - *

Note this class carries no lifecycle code. The Compose stack, the control API client, provider - * registration and event awaiting are all owned by the step definitions in - * {@code dev.openfeature.contrib.tools.providertck.steps}, which reach the harness through - * {@link TckRuntime}. + *

Note this class carries no lifecycle code of its own. Provider registration, event awaiting + * and backend manipulation are owned by the step definitions in + * {@code dev.openfeature.contrib.tools.providertck.steps}, which reach the harness and its + * {@link BackendControl} through {@link TckRuntime}. * * @see ProviderTckHarness + * @see ContainerizedProviderTckTest */ @Suite @IncludeEngines("cucumber") @@ -43,4 +55,4 @@ @ConfigurationParameter(key = Constants.EXECUTION_MODE_FEATURE_PROPERTY_NAME, value = "same_thread") @ConfigurationParameter(key = Constants.GLUE_PROPERTY_NAME, value = "dev.openfeature.contrib.tools.providertck.steps") @ConfigurationParameter(key = Constants.OBJECT_FACTORY_PROPERTY_NAME, value = "io.cucumber.picocontainer.PicoFactory") -public abstract class AbstractProviderTckTest implements ProviderTckHarness {} +public abstract class ProviderTckTest implements ProviderTckHarness {} diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java index 82da5aa11..4489f4bc3 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/TckRuntime.java @@ -1,37 +1,31 @@ package dev.openfeature.contrib.tools.providertck; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.io.File; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.ServiceLoader; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.containers.ComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; /** - * Suite-scoped runtime: discovers the provider's harness, owns the Compose stack, and exposes the - * control API client to the step definitions. + * Suite-scoped runtime: discovers the provider's harness, drives its suite lifecycle, and exposes + * its {@link BackendControl} to the step definitions. * - *

The Compose stack is started once, before the first scenario, and stopped - * after the last one. It is never stopped or restarted in between. Testcontainers cannot reliably - * preserve dynamically mapped host ports across a container restart, so a restart would silently - * invalidate every provider already pointed at the old port. Backend unavailability is therefore - * always simulated inside the running stack through the control API. See the normative statement of - * this invariant in {@code openapi/control-api.yaml}. + *

This class knows nothing about containers, ports or transports. Whatever must exist before the + * first scenario is created by {@link ProviderTckHarness#startSuite()} and released by + * {@link ProviderTckHarness#stopSuite()} — a Compose stack for + * {@link ContainerizedProviderTckTest}, nothing at all for a harness whose backend is a data + * structure in this JVM. + * + *

The lifecycle runs once: started before the first scenario, stopped after the + * last one, never cycled in between. Scenario isolation is achieved through + * {@link BackendControl#prepareScenario()} instead. * *

State is static because Cucumber's {@code @BeforeAll} / {@code @AfterAll} hooks are static and - * the stack must outlive individual scenarios. Consequently only one TCK suite may run per JVM fork - * at a time. + * the runtime must outlive individual scenarios. Consequently only one TCK suite may run per JVM + * fork at a time. */ -@SuppressFBWarnings( - value = "EI_EXPOSE_REP", - justification = "The harness and control API client are shared collaborators by design; " - + "step definitions must act on the same instances the suite started") public final class TckRuntime { private static final Logger log = LoggerFactory.getLogger(TckRuntime.class); @@ -42,42 +36,52 @@ public final class TckRuntime { private static TckRuntime instance; private final ProviderTckHarness harness; - private final ComposeContainer compose; - private final ControlApiClient controlApi; - private final BackendEndpoint endpoint; + private final BackendControl backendControl; - private TckRuntime(ProviderTckHarness harness, ComposeContainer compose) { + private TckRuntime(ProviderTckHarness harness, BackendControl backendControl) { this.harness = harness; - this.compose = compose; - this.endpoint = new BackendEndpoint(compose, harness.backendService()); - String baseUrl = "http://" + compose.getServiceHost(harness.backendService(), null) + ":" - + compose.getServicePort(harness.backendService(), harness.controlPort()); - this.controlApi = new ControlApiClient(baseUrl, harness.settleTime()); + this.backendControl = backendControl; } /** - * Starts the Compose stack if it is not already running, and returns the shared runtime. + * Starts the suite lifecycle if it is not already running, and returns the shared runtime. * * @return the suite-scoped runtime */ public static synchronized TckRuntime startIfNeeded() { - if (instance == null) { - ProviderTckHarness harness = discoverHarness(); - log.info("Provider TCK harness: {}", harness.getClass().getName()); - instance = new TckRuntime(harness, startCompose(harness)); - instance.controlApi.awaitReady(harness.startupTimeout()); - log.info("Control API ready at {}", instance.controlApi.baseUrl()); + if (instance != null) { + return instance; + } + ProviderTckHarness harness = discoverHarness(); + log.info("Provider TCK harness: {}", harness.getClass().getName()); + + harness.startSuite(); + try { + BackendControl control = harness.backendControl(); + if (control == null) { + throw new IllegalStateException(harness.getClass().getName() + + ".backendControl() returned null. Every harness must supply the seam through " + + "which the TCK manipulates the backend — HttpBackendControl for an external " + + "backend, InProcessBackendControl for a provider that has none."); + } + log.info("Backend control: {}", control.description()); + instance = new TckRuntime(harness, control); + } catch (RuntimeException e) { + // startSuite() may have allocated a container stack before this failed. + harness.stopSuite(); + throw e; } return instance; } /** - * Stops the Compose stack and releases the shared runtime. + * Runs the harness's suite teardown and releases the shared runtime. */ public static synchronized void stop() { if (instance != null) { - instance.compose.stop(); + ProviderTckHarness harness = instance.harness; instance = null; + harness.stopSuite(); } } @@ -85,7 +89,7 @@ public static synchronized void stop() { * Returns the running runtime. * * @return the suite-scoped runtime - * @throws IllegalStateException if the stack has not been started + * @throws IllegalStateException if the suite has not been started */ public static synchronized TckRuntime get() { if (instance == null) { @@ -104,46 +108,12 @@ public ProviderTckHarness harness() { } /** - * Returns the client for the backend's control API. - * - * @return the control API client - */ - public ControlApiClient controlApi() { - return controlApi; - } - - /** - * Returns the host and mapped ports of the running stack. + * Returns the seam through which the TCK manipulates the backend. * - * @return the backend endpoint + * @return the backend control for this suite */ - public BackendEndpoint endpoint() { - return endpoint; - } - - private static ComposeContainer startCompose(ProviderTckHarness harness) { - File composeFile = harness.composeFile(); - if (!composeFile.isFile()) { - throw new IllegalStateException("Compose file not found: " + composeFile.getAbsolutePath() - + ". ProviderTckHarness.composeFile() is resolved relative to the module directory."); - } - ComposeContainer compose = new ComposeContainer(composeFile); - - compose.withExposedService(harness.backendService(), harness.controlPort(), Wait.forListeningPort()); - for (Integer port : harness.backendPorts()) { - compose.withExposedService(harness.backendService(), port, Wait.forListeningPort()); - } - for (Map.Entry> service : - harness.additionalExposedPorts().entrySet()) { - for (Integer port : service.getValue()) { - compose.withExposedService(service.getKey(), port, Wait.forListeningPort()); - } - } - compose.withStartupTimeout(harness.startupTimeout()); - - log.info("Starting Compose stack {} (started once per suite, never restarted)", composeFile.getAbsolutePath()); - compose.start(); - return compose; + public BackendControl backendControl() { + return backendControl; } /** @@ -169,7 +139,8 @@ private static ProviderTckHarness discoverHarness() { if (found.isEmpty()) { throw new IllegalStateException("No ProviderTckHarness found. Write a test class extending " - + "AbstractProviderTckTest; it is both the JUnit suite and the harness."); + + "ContainerizedProviderTckTest (external backend) or ProviderTckTest (no backend); " + + "it is both the JUnit suite and the harness."); } if (found.size() == 1) { return found.get(0); diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/AbstractSteps.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/AbstractSteps.java index 526068cb4..6a3ff9cf6 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/AbstractSteps.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/AbstractSteps.java @@ -1,5 +1,6 @@ package dev.openfeature.contrib.tools.providertck.steps; +import dev.openfeature.contrib.tools.providertck.BackendControl; import dev.openfeature.contrib.tools.providertck.ProviderTckHarness; import dev.openfeature.contrib.tools.providertck.TckRuntime; import dev.openfeature.contrib.tools.providertck.TckState; @@ -7,8 +8,13 @@ /** * Base for the TCK step definition classes. * - *

Holds the PicoContainer-injected scenario state and gives subclasses convenience access to the - * suite-scoped runtime. + *

Holds the PicoContainer-injected scenario state and gives subclasses the only two collaborators + * a step is allowed to reach: the provider author's harness, and the {@link BackendControl} that + * manipulates the backend. + * + *

Deliberately no accessor for the runtime itself. Steps must not know whether the backend is a + * container reached over HTTP or a map in this JVM — that is exactly what {@link BackendControl} + * exists to hide, and it is what lets the same Gherkin run in both modes. */ public abstract class AbstractSteps { @@ -20,20 +26,20 @@ protected AbstractSteps(TckState state) { } /** - * Returns the suite-scoped runtime that owns the Compose stack and control API. + * Returns the provider author's harness. * - * @return the running TCK runtime + * @return the discovered harness */ - protected TckRuntime runtime() { - return TckRuntime.get(); + protected ProviderTckHarness harness() { + return TckRuntime.get().harness(); } /** - * Returns the provider author's harness. + * Returns the seam through which the backend is manipulated. * - * @return the discovered harness + * @return the backend control for this suite */ - protected ProviderTckHarness harness() { - return TckRuntime.get().harness(); + protected BackendControl backend() { + return TckRuntime.get().backendControl(); } } diff --git a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ProviderSteps.java b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ProviderSteps.java index 8fe239414..35652f943 100644 --- a/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ProviderSteps.java +++ b/tools/provider-tck/src/main/java/dev/openfeature/contrib/tools/providertck/steps/ProviderSteps.java @@ -19,6 +19,7 @@ import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; +import java.time.Duration; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -27,9 +28,13 @@ import org.slf4j.LoggerFactory; /** - * Lifecycle and control API steps: bringing the Compose stack up, gating scenarios on declared + * Lifecycle and backend-control steps: bringing the suite up, gating scenarios on declared * capabilities, creating and registering the provider under test, and simulating backend outages. * + *

Every step that touches the backend goes through {@link #backend()}. Nothing here knows whether + * that is a container driven over HTTP or an in-memory provider manipulated directly, which is what + * lets one set of feature files cover both. + * *

Step vocabulary is inherited from the flagd test harness so that existing feature files port * with a near-zero diff. The only change is dropping the word {@code flagd} from the provider setup * step: {@code a stable flagd provider} becomes {@code a stable provider}. @@ -43,7 +48,7 @@ public ProviderSteps(TckState state) { } /** - * Starts the Compose stack once, before the first scenario. + * Runs the harness's suite startup once, before the first scenario. */ @BeforeAll public static void beforeAll() { @@ -51,7 +56,7 @@ public static void beforeAll() { } /** - * Stops the Compose stack after the last scenario. + * Runs the harness's suite teardown after the last scenario. */ @AfterAll public static void afterAll() { @@ -66,6 +71,11 @@ public static void afterAll() { * configuration-change events should see those scenarios visibly excluded, never silently * green. * + *

This is also how a backend with no connection to lose stays honest. A harness whose + * {@link dev.openfeature.contrib.tools.providertck.BackendControl} cannot simulate an outage + * leaves {@link Capability#STALE} and {@link Capability#UNAVAILABLE_INIT} undeclared, and the + * scenarios needing them are skipped here — before any step can reach an unsupported operation. + * * @param scenario the scenario about to run */ @Before(order = 0) @@ -81,19 +91,18 @@ public void gateOnCapabilities(Scenario scenario) { } /** - * Restores the backend to a running, freshly seeded state before each scenario. + * Restores the backend to the state every scenario starts from. * - *

Scenario isolation is achieved here, through the control API, and never by restarting - * containers — see the no-container-restart invariant in {@code openapi/control-api.yaml}. + *

Scenario isolation is achieved here and nowhere else — never by restarting containers, and + * never by relying on scenarios happening not to interfere. */ @Before(order = 10) public void prepareBackend() { - ProviderTckHarness harness = harness(); - runtime().controlApi().prepareScenario(harness.defaultConfig()); + backend().prepareScenario(); } /** - * Tears the provider down without disturbing the Compose stack. + * Tears the provider down without disturbing the backend. * *

Replaces the domain's provider with a {@link NoOpProvider} through the SDK lifecycle rather * than calling {@code shutdown()} directly, because only the former makes the SDK detach the @@ -111,10 +120,10 @@ public void tearDown() { * Creates the provider under test and registers it under a scenario-scoped domain. * *

Two provider flavours are recognised. A {@code stable} provider is built by the harness - * against the running stack and registered with {@code setProviderAndWait}, so the step does not - * return until the provider is ready. An {@code unavailable} provider points at a dead backend - * and is registered with {@code setProvider}, deliberately without waiting — the scenario's - * whole point is that readiness never arrives. + * against the running backend and registered with {@code setProviderAndWait}, so the step does + * not return until the provider is ready. An {@code unavailable} provider points at a dead + * backend and is registered with {@code setProvider}, deliberately without waiting — the + * scenario's whole point is that readiness never arrives. * * @param flavour either {@code stable} or {@code unavailable} */ @@ -126,7 +135,7 @@ public void createProvider(String flavour) { switch (flavour) { case "stable": - provider = harness.createProvider(runtime().endpoint()); + provider = harness.createProvider(); waitForReady = true; break; case "unavailable": @@ -161,7 +170,7 @@ public void createProvider(String flavour) { */ @When("the connection is lost") public void theConnectionIsLost() { - runtime().controlApi().stop(); + backend().disconnect(); } /** @@ -171,7 +180,7 @@ public void theConnectionIsLost() { */ @When("the connection is lost for {int}s") public void theConnectionIsLostFor(int seconds) { - runtime().controlApi().restart(seconds); + backend().disconnectFor(Duration.ofSeconds(seconds)); } /** @@ -184,7 +193,7 @@ public void theConnectionIsLostFor(int seconds) { */ @When("the connection is restored") public void theConnectionIsRestored() { - runtime().controlApi().start(harness().defaultConfig()); + backend().reconnect(); } /** @@ -192,7 +201,7 @@ public void theConnectionIsRestored() { */ @When("the flag was modified") public void theFlagWasModified() { - runtime().controlApi().change(); + backend().changeFlag(); } /** diff --git a/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InMemoryProviderTckTest.java b/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InMemoryProviderTckTest.java new file mode 100644 index 000000000..3d8ac5e25 --- /dev/null +++ b/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InMemoryProviderTckTest.java @@ -0,0 +1,78 @@ +package dev.openfeature.contrib.tools.providertck; + +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.util.EnumSet; +import java.util.Set; + +/** + * Runs the OpenFeature Provider TCK against the SDK's own {@link InMemoryProvider}. + * + *

This is the TCK's self-test, and it earns its keep twice over. + * + *

It is the reference adoption for a provider with no backend. Everything a + * file-based or environment-variable provider needs to write is here, and it is three methods: hand + * over a {@link BackendControl}, hand over a provider, and say which capabilities hold. + * + *

It is also the Docker-free canary. Because it needs no container, no Compose + * stack and no network, it runs in seconds on any machine and in any CI job, which makes it the + * fast check that catches a broken step definition, a mis-wired capability gate or a regression in + * the shared harness long before the containerised suites get a chance to. When a change breaks + * both this and the flagd suite, this one tells you within seconds and points at the TCK rather + * than at a provider. + * + *

Note what it does not do: it is not a licence for providers that have a backend to + * test themselves this way. See {@link BackendControl} for why. + */ +public class InMemoryProviderTckTest extends ProviderTckTest { + + /** + * Both the flag store and the factory for the provider that serves it. + * + *

In-process the two are the same thing — {@code changeFlag()} has to reach the live provider + * instance to emit an event from it — so one instance backs both harness methods below. + */ + private final InProcessBackendControl control = new InProcessBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public FeatureProvider createProvider() { + return control.createProvider(); + } + + /** + * {@inheritDoc} + * + *

Four capabilities, and each omission is a fact about {@link InMemoryProvider} rather than a + * convenience: + * + *

+ * + *

{@link Capability#STRICT_NUMERIC_TYPING} is declared, and that is worth stating + * plainly: {@link InMemoryProvider} refuses to narrow {@code float-flag} (0.5) to an integer and + * reports {@code TYPE_MISMATCH} instead. It is the reference behaviour the capability describes. + */ + @Override + public Set capabilities() { + return EnumSet.of( + Capability.EVENTS, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.STRICT_NUMERIC_TYPING); + } +} diff --git a/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControlTest.java b/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControlTest.java new file mode 100644 index 000000000..14df30bb3 --- /dev/null +++ b/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/InProcessBackendControlTest.java @@ -0,0 +1,104 @@ +package dev.openfeature.contrib.tools.providertck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.openfeature.sdk.ImmutableContext; +import dev.openfeature.sdk.ProviderEvaluation; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.time.Duration; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Guards the two properties of {@link InProcessBackendControl} that the Gherkin cannot assert + * about itself. + * + *

The first is that unsupported operations fail loudly. The whole + * skipped-by-capability design collapses into false confidence if a connection operation quietly + * does nothing, and a scenario that never runs cannot prove that it would have failed. These tests + * call the operations directly. + * + *

The second is that scenario isolation actually isolates. {@link InMemoryProviderTckTest} would + * still pass if {@code changeFlag()} leaked into the next scenario, because no scenario evaluates + * {@code changing-flag} before modifying it. + */ +class InProcessBackendControlTest { + + @Test + @DisplayName("connection operations throw rather than silently doing nothing") + void connectionOperationsThrow() { + InProcessBackendControl control = new InProcessBackendControl(); + + // The message has to name the fix, because whoever hits this is looking at a red scenario + // that reads like a provider defect and is in fact a capability declared in error. + assertThatThrownBy(control::disconnect) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support 'disconnect'") + .hasMessageContaining("test-configuration bug") + .hasMessageContaining("Capability.STALE"); + + assertThatThrownBy(control::reconnect) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support 'reconnect'"); + + assertThatThrownBy(() -> control.disconnectFor(Duration.ofSeconds(1))) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support 'disconnectFor'"); + } + + @Test + @DisplayName("changing a flag without a provider fails instead of being lost") + void changeFlagWithoutProviderThrows() { + InProcessBackendControl control = new InProcessBackendControl(); + control.prepareScenario(); + + assertThatThrownBy(control::changeFlag) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Given a stable provider"); + } + + @Test + @DisplayName("changeFlag changes the resolved value and the next scenario starts from baseline") + void changeFlagIsVisibleAndDoesNotLeak() throws Exception { + InProcessBackendControl control = new InProcessBackendControl(); + + control.prepareScenario(); + InMemoryProvider first = control.createProvider(); + first.initialize(new ImmutableContext()); + assertThat(resolveChangingFlag(first)).isEqualTo("foo"); + + control.changeFlag(); + assertThat(resolveChangingFlag(first)) + .as("changeFlag must actually change what the provider resolves, not merely emit an event") + .isEqualTo("bar"); + + // The next scenario must not inherit that change. The baseline map is shared between every + // provider this control hands out, so a mutation that reached it would leak forwards. + control.prepareScenario(); + InMemoryProvider second = control.createProvider(); + second.initialize(new ImmutableContext()); + assertThat(resolveChangingFlag(second)) + .as("each scenario starts from the canonical baseline") + .isEqualTo("foo"); + } + + @Test + @DisplayName("the canonical flag set omits missing-flag") + void missingFlagIsAbsent() throws Exception { + InProcessBackendControl control = new InProcessBackendControl(); + InMemoryProvider provider = control.createProvider(); + provider.initialize(new ImmutableContext()); + + // Absence is what the FLAG_NOT_FOUND scenario tests, so seeding it by accident would turn + // that scenario green for the wrong reason. + assertThatThrownBy(() -> provider.getStringEvaluation("missing-flag", "fallback", new ImmutableContext())) + .hasMessageContaining("missing-flag"); + } + + private static String resolveChangingFlag(InMemoryProvider provider) { + ProviderEvaluation evaluation = + provider.getStringEvaluation("changing-flag", "unset", new ImmutableContext()); + return evaluation.getValue(); + } +} diff --git a/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/MultiProviderTckTest.java b/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/MultiProviderTckTest.java new file mode 100644 index 000000000..2d34c0094 --- /dev/null +++ b/tools/provider-tck/src/test/java/dev/openfeature/contrib/tools/providertck/MultiProviderTckTest.java @@ -0,0 +1,77 @@ +package dev.openfeature.contrib.tools.providertck; + +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.multiprovider.MultiProvider; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +/** + * Runs the OpenFeature Provider TCK against the SDK's {@link MultiProvider}, wrapping a single + * {@link InMemoryProvider}. + * + *

A provider that delegates is still a provider, and delegation is where the contract is easiest + * to drop on the floor: a variant that does not survive the hop, a reason rewritten to + * {@code DEFAULT}, an error code flattened to {@code GENERAL}, an event that never reaches the + * client. Wrapping exactly one child makes every one of those observable, because the correct + * answer is precisely what {@link InMemoryProviderTckTest} already asserts. Any difference between + * these two suites is attributable to {@link MultiProvider} and nothing else. + * + *

That framing is the point of running it here rather than in the SDK: this is not a test of + * aggregation across several backends, it is a test that delegation is transparent. + * + *

It costs one class, needs no Docker, and it has already earned its place — see the capability + * note below. + */ +public class MultiProviderTckTest extends ProviderTckTest { + + private final InProcessBackendControl control = new InProcessBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + /** + * {@inheritDoc} + * + *

Exactly one child. See the class javadoc for why that is the interesting configuration + * rather than a degenerate one. + */ + @Override + public FeatureProvider createProvider() { + return new MultiProvider(Collections.singletonList(control.createProvider())); + } + + /** + * {@inheritDoc} + * + *

{@link Capability#CONFIGURATION_CHANGE} is not declared, and that is a + * finding rather than a configuration choice. + * + *

{@link MultiProvider} extends {@code EventProvider} but never subscribes to its children, + * so a child's {@code PROVIDER_CONFIGURATION_CHANGED} — along with its {@code PROVIDER_ERROR} + * and {@code PROVIDER_STALE} — is swallowed and never reaches the client. Wrapping an + * in-memory provider in a multi-provider therefore silently costs you configuration-change + * events, with nothing in the API to suggest it. + * + *

This is a known gap, tracked as + * open-feature/java-sdk#1882 + * (gap 1, "child provider event aggregation and status tracking", High). The suite reproduced + * it from the outside, which is a reasonable advertisement for what the TCK is for: the gap was + * originally found by hand-comparing implementations against the js-sdk reference. + * + *

Delete this omission once #1882 is fixed. Until then the + * {@code @configuration-change} scenario is reported as skipped-with-reason rather than passing + * on a provider that cannot satisfy it. + * + *

Everything else holds. Values, variants, reasons, the full type-mismatch matrix, + * {@code FLAG_NOT_FOUND}, structured values, strict numeric typing and reaching {@code READY} + * all survive the delegation hop unchanged. + */ + @Override + public Set capabilities() { + return EnumSet.of(Capability.EVENTS, Capability.OBJECT, Capability.STRICT_NUMERIC_TYPING); + } +}