diff --git a/.github/component_owners.yml b/.github/component_owners.yml index 4e4a03415..2d20b2b5e 100644 --- a/.github/component_owners.yml +++ b/.github/component_owners.yml @@ -46,6 +46,8 @@ components: - toddbaert tools/flagd-http-connector: - liran2000 + tools/tck: + - aepfli ignored-authors: - renovate-bot diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfdcc24e2..d91af3ab0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,41 @@ on: - reopened branches: - main + # temporary, so stacked tck PRs get CI; remove once merged (open-feature/spec#417) + - 'feat/provider-tck*' jobs: + # fast no-docker tck canary (in-memory, multi, controllable providers) + # not a gate on main; runs in parallel + tck: + name: Provider TCK (no Docker) + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + with: + # spec submodule supplies the tck assets + submodules: recursive + + - name: Set up JDK 21 + uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5 + with: + java-version: 21 + distribution: 'temurin' + cache: maven + + - name: Cache local Maven repository + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}21-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}21-maven- + + - name: Verify the TCK against the in-memory provider + # no e2e profile or docker needed + run: mvn --batch-mode --activate-profiles codequality -pl tools/tck -am clean verify + main: strategy: matrix: diff --git a/.gitmodules b/.gitmodules index 636acb4c5..730b25ff8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -18,3 +18,7 @@ path = tools/flagd-api-testkit/test-harness url = https://github.com/open-feature/test-harness.git branch = v3.10.4 +[submodule "tools/tck/spec"] + path = tools/tck/spec + url = https://github.com/open-feature/spec.git + branch = specification/assets/provider-tck/v0.1.0 diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 6bf06cffc..a259c348a 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -18,6 +18,7 @@ "tools/flagd-api": "1.0.0", "tools/flagd-api-testkit": "0.3.0", "tools/flagd-core": "3.0.0", + "tools/tck": "0.1.0", ".": "1.0.0", "providers/optimizely": "1.0.0" } diff --git a/pom.xml b/pom.xml index 6d94d070c..36b7fb164 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,7 @@ + tools/tck tools/flagd-api-testkit tools/flagd-api tools/flagd-core diff --git a/release-please-config.json b/release-please-config.json index d3281c2d5..54129737e 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -205,6 +205,17 @@ "README.md" ] }, + "tools/tck": { + "package-name": "dev.openfeature.contrib.tools.tck", + "release-type": "simple", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true, + "versioning": "default", + "extra-files": [ + "pom.xml", + "README.md" + ] + }, "tools/flagd-api-testkit": { "package-name": "dev.openfeature.contrib.tools.flagdapitestkit", "release-type": "simple", diff --git a/tools/tck/.gitignore b/tools/tck/.gitignore new file mode 100644 index 000000000..cca79086c --- /dev/null +++ b/tools/tck/.gitignore @@ -0,0 +1,4 @@ +# copied from the spec submodule at build time; do not edit +src/main/resources/gherkin/ +src/main/resources/flags/ +src/main/resources/openapi/ diff --git a/tools/tck/README.md b/tools/tck/README.md new file mode 100644 index 000000000..451e16859 --- /dev/null +++ b/tools/tck/README.md @@ -0,0 +1,309 @@ +# OpenFeature Provider TCK (Java) + +A conformance suite any OpenFeature Java provider can adopt to verify it implements the provider +contract of the [specification](https://openfeature.dev/specification/). It is the Java implementation +of [Appendix F][appendix-f], which defines the shared scenarios, canonical flag set, control API and +capability vocabulary; read it for anything true of the suite rather than this artifact (what is +tested, rules for declaring capabilities, known deviations, CI guidance). Tracking issue: +[open-feature/spec#417][tracking]. + +> **Status: proof of concept.** Scenarios cover each mechanism once rather than exhaustively. Expect +> breaking changes. + +## Installation + + +```xml + + dev.openfeature.contrib.tools + tck + 0.1.0 + test + +``` + + +- Java 11+ and JUnit 5; Docker only for providers with an external backend. +- Testcontainers is `provided`/`optional`; containerized adopters add `org.testcontainers:testcontainers` themselves. +- The SDK is a `provided` version range (`[1.21.0,1.99999)`), never pinned. + +## Quick start + +| | Extend | Backend control | +| --- | --- | --- | +| External backend | `ContainerizedProviderTckTest` | built in, over the HTTP control API | +| No backend (in-memory, env vars, local file) | `ProviderTckTest` | an in-process `BackendControl` | + +### A provider with a backend + +Write a Compose file exposing your backend and its [control API][control-api], seed it with the +[canonical flag set][flags] (see the [assets README][assets]; several properties are easy to break +while seeding), then: + +```java +public class MyProviderTest extends ContainerizedProviderTckTest { + + @Override + public File composeFile() { + return new File("src/test/resources/tck/docker-compose.yaml"); + } + + @Override + public List backendPorts() { + return Collections.singletonList(5000); + } + + @Override + public FeatureProvider createProvider(BackendEndpoint endpoint) { + return new MyProvider(endpoint.host(), endpoint.port(5000)); + } + + @Override + public FeatureProvider createUnavailableProvider() { + return new MyProvider("localhost", 9999); + } +} +``` + +Note `endpoint.port(5000)` rather than `5000`: declared ports are container-internal, the +stack must not pin host ports, and Compose maps each to an arbitrary one. + +That's the whole adoption. The TCK owns the Compose lifecycle, port discovery, control calls, provider +registration, event awaiting and teardown, and the assets are packaged in the JAR (no submodule +needed). If you find yourself adding test infrastructure here, please open an issue. + +`createUnavailableProvider()` should target a closed local port, not your stack (the stack stays up; +outages are simulated through the control API), with a short connection deadline since failure +scenarios assert promptness. + +### A provider with no backend + +`BackendControl.inProcess()` supports the SDK's `InMemoryProvider`. Copy the shape; for your own provider, +`createProvider()` returns yours and the `BackendControl` reaches its live flag store (it must reach the +live provider instance so `changeFlag()` can emit events). + +```java +public class InMemoryProviderTckTest extends ProviderTckTest { + + private final InProcessControl control = BackendControl.inProcess(); + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public FeatureProvider createProvider() { + return control.createProvider(); + } + + @Override + public Set capabilities() { + return EnumSet.of(Capability.EVENTS, Capability.CONFIGURATION_CHANGE, Capability.OBJECT); + } +} +``` + +This path is for providers with nothing to contract with (see Appendix F, "Providers with no +backend"). In-process control cannot `disconnect()`/`reconnect()`, so withhold `STALE` and +`UNAVAILABLE_INIT`; declaring them fails with `UnsupportedOperationException` (a configuration bug, +never a skip). + +### Several provider modes + +Write one class per mode; nothing else is needed. Each class is its own suite with its own Compose +stack, and abstract classes are not run, so a shared intermediate base is safe. Modes may differ in +timing too (e.g. flagd in-process needs a longer init deadline than RPC). + +## Options + +Overridable on `ContainerizedProviderTckTest`; names and defaults match every language's TCK. + +| Option | Default | Notes | +| --- | --- | --- | +| `composeFile()` | required | relative to the Maven module directory | +| `backendPorts()` | required | container-internal ports the provider connects to; don't list the control port. Resolve each with `endpoint.port(..)` | +| `backendService()` | `backend` | service hosting both the backend and control API | +| `controlPort()` | `8080` | | +| `additionalPorts()` | none | `Map>` of extra service to ports, resolved via `BackendEndpoint` by service name | +| `backendConfiguration()` | `default` | name passed to the control API's `POST /start` | +| `startupTimeout()` | 60s | stack and control API becoming reachable | +| `eventTimeout()` | 12s | waiting for a provider event; exceed your worst-case detection latency | +| `readyTimeout()` | 30s | waiting for a provider lifecycle state | + +- Never pin host ports; they're mapped dynamically and discovered after startup. +- Timeouts are bounds on awaits, never pauses; nothing sleeps after a control call. +- `backendConfiguration()` names a **backend** configuration; `configuration()` names the **provider** + mode. `configuration()` defaults to the hyphenated class name minus the JUnit suffix (`InProcessTest` + becomes `in-process`), so set it explicitly when the class name doesn't identify the provider + (flagd uses `flagd-rpc` and `flagd-in-process`). +- `BackendControl.backendType()` states what drove the run: `BackendType.HTTP`, `IN_PROCESS`, or + `CUSTOM`, which is the default for a control the TCK does not ship. + +## Capabilities + +Scenarios for optional behavior are tagged; undeclared capabilities are reported as **skipped with a +reason**, never passed. When to declare or withhold is [Appendix F's][appendix-f] ("Capabilities", +"Rules for declaring"); the decision is per scenario, not per tag. + +| Capability | Tag | | Capability | Tag | +| --- | --- | --- | --- | --- | +| `LIFECYCLE` | `@lifecycle` | | `UNAVAILABLE_INIT` | `@unavailable` | +| `REINITIALIZATION` | `@reinitialization` | | `NUMERIC_COERCION` | `@numeric-coercion` | +| `EVENTS` | `@events` | | `TARGETING` | `@targeting` | +| `STALE` | `@stale` | | `STANDARD_REASONS` | `@standard-reasons` | +| `CONFIGURATION_CHANGE` | `@configuration-change` | | `STRING_TYPING` | `@string-typing` | +| `OBJECT` | `@object` | | `LARGE_INTEGERS` | `@large-integers` (1) | +| `VARIANTS` | `@variants` | | `CACHING` | `@caching` (2) | +| `DISABLED_FLAGS` | `@disabled-flags` | | `FULLY_TYPED_VALUES` | `@fully-typed-values` | + +(1) not declarable in Java (2) reserved, not declarable + +The default is every declarable capability. Narrow it, don't widen it: run the suite and remove only +what your provider genuinely can't do. + +```java +@Override +public Set capabilities() { + return Capability.declarableExcept(Capability.STALE); +} +``` + +- Use `Capability.declarable()`/`declarableExcept(...)`, not `EnumSet.complementOf(...)` (which would + include reserved and inexpressible capabilities); naming a refused capability is an error. +- `LARGE_INTEGERS` is inexpressible: `Client.getIntegerDetails` is 32-bit. Its scenario is skipped with + a reason naming the SDK, distinct from "provider declined". Neither this nor a reserved capability + needs a `KnownDeviation`. +- `FULLY_TYPED_VALUES` (floats and structures requested as strings) implies `STRING_TYPING` (booleans + and integers); declare it only alongside. A store may legitimately support only the latter. + +### Known deviations + +A `knownDeviations()` entry means the provider fails something it's **required** to do (a numbered +`MUST`, or a rule the implementation bound itself to). If the spec permits the behavior, withhold the +capability instead. + +```java +@Override +public List knownDeviations() { + return List.of(KnownDeviation.tracked( + Capability.NUMERIC_COERCION, + "https://github.com/open-feature/java-sdk-contrib/issues/1234", + "float-flag through the integer API returns 0 with no error code")); +} +``` + +- `summary` is required; use `KnownDeviation.untracked(...)` without an issue. +- Capability may be `null` for an ungated scenario; `CACHING` and `LARGE_INTEGERS` are refused. +- Empty (the default) is silence, not a claim of having none. + +## Running it + +```bash +# once, if the module is not installed locally +mvn -pl tools/tck -am -DskipTests install + +mvn -Ptck -pl providers/ test +``` + +Don't add `-am` to the second command; it runs the dependencies' unit suites first and mixes their +failures into the `-Ptck` result. + +Scenarios run serially by default, and the stack starts once per suite and is never restarted. + +The suite does not pin that any more. Set `cucumber.execution.parallel.enabled=true` and each thread +gets its own harness and its own stack, so scenarios no longer share backend state -- but set +`cucumber.execution.parallel.config.strategy=fixed` with a matching `fixed.parallelism` and +`fixed.max-pool-size` as well. The parallelism setting is a ForkJoinPool target, not a cap, and these +steps block: left uncapped against flagd the pool grew to 18 threads and started 18 stacks. Capped at +four it ran the two suites in 53s against 83s serial, for eight stacks. + +Put the adoption in its own `tck` test package (e.g. `.../providers/flagd/tck/`, a sibling of `e2e`), +keep the `*Test` suffix for Surefire, and check `configuration()` when renaming. Exclude it from the +default build and select it with a `tck` profile: + +```xml + + **/e2e/*.java,**/tck/*.java + + + + tck + + **/e2e/*.java + + + org.apache.maven.plugins + maven-surefire-plugin + **/tck/*.java + + +``` + +- Both halves of the profile are needed: dropping the exclusion alone also runs unit tests; narrowing + includes alone runs nothing. +- `testExclusions` is a Surefire (not compiler) exclusion, so the adoption still compiles in every build. +- Verify the property under every profile your CI activates: + `mvn -Pe2e -pl providers/ help:evaluate -Dexpression=testExclusions -DforceStdout` + +## Extensions + +Provider-specific scenarios (e.g. flagd's `fractional`) run inside the same suite: + +``` +src/test/resources/extensions/fractional.feature +src/test/java/openfeature/tck/extensions/FractionalSteps.java // package openfeature.tck.extensions +``` + +- Same backend lifecycle and `BackendControl`; canonical steps are on the glue path, so an extension + can open with `Given a stable provider`. +- Step classes take what they need as constructor arguments (`TckState`, `ProviderTckHarness`, + `BackendControl`). Don't build your own client; it would resolve against a provider the suite never + registered. +- Step classes and step methods must be `public`; otherwise Cucumber reports scenarios as undefined. +- Don't put files under `gherkin/`; a same-named file would silently shadow a canonical one. +- Unknown tags are tolerated in `extensions/` (they gate nothing) but fail the run in `gherkin/`. +- `ProviderTck` exposes `EXTENSIONS`, `EXTENSION_GLUE` and `ALL_GLUE` for custom + `@ConfigurationParameter`s; always use `ALL_GLUE` or canonical steps go undefined. +- Portable scenarios belong in the TCK, not in an extension. + +## Java notes + +- **Suite discovery** relies on the JUnit Platform auto-registering the TCK's `TestExecutionListener` + (Surefire, Gradle and IDEs do by default). If disabled, register your harness in + `src/test/resources/META-INF/services/dev.openfeature.contrib.tools.tck.ProviderTckHarness` and pick + between several with `-Dopenfeature.tck.harness=RpcTest`. +- **Reference adoptions** (Docker-free, under a second): `InMemoryProviderTckTest`, + `ControllableProviderTckTest` (real initialization; covers `@lifecycle`) and `MultiProviderTckTest` + (can't declare `CONFIGURATION_CHANGE`; see + [java-sdk#1882](https://github.com/open-feature/java-sdk/issues/1882)). +- **Packaged assets are generated** from the `spec` submodule (`specification/assets/provider-tck/`): + the submodule is updated at `initialize` (skip with `-Dtck.spec.checkout.skip=true`) and copied into + `src/main/resources/` at `generate-resources`. Don't edit `gherkin/`, `flags/` or `openapi/` there; + they're git-ignored and changes belong in [open-feature/spec][spec]. `CanonicalAssetDigestTest` + fails on a stale checkout, and `CanonicalTagCoverageTest` fails on a pin carrying a capability tag + this module doesn't know. +- **Step vocabulary** is inherited from the [flagd test harness](https://github.com/open-feature/test-harness) + where provider-neutral; only `Given a stable/unavailable flagd provider` were renamed to drop the + vendor. Read the javadoc on the step classes before writing an extension step. +- **Object factory**: TCK suites use their own Cucumber `ObjectFactory` (PicoContainer-based), registered + via `META-INF/services`. If the same module has other Cucumber suites, set `cucumber.object-factory` + on them explicitly (e.g. `io.cucumber.picocontainer.PicoFactory`) or Cucumber fails on multiple + factories. + +## Known gaps + +- See [Appendix F][appendix-f] for suite-wide gaps (context passthrough beyond the targeting key, + per-flag control, caching, `@stale` without containers, hooks, flag metadata). + +## Contributing + +See [CONTRIBUTING.md](../../CONTRIBUTING.md). New scenarios must be portable across providers; vendor +specific behavior belongs in that provider's own suite or an extension. + +[appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md +[assets]: https://github.com/open-feature/spec/blob/main/specification/assets/provider-tck/README.md +[control-api]: https://github.com/open-feature/spec/blob/main/specification/assets/provider-tck/openapi/control-api.yaml +[flags]: https://github.com/open-feature/spec/blob/main/specification/assets/provider-tck/flags/canonical-flags.json +[spec]: https://github.com/open-feature/spec +[tracking]: https://github.com/open-feature/spec/issues/417 diff --git a/tools/tck/lombok.config b/tools/tck/lombok.config new file mode 100644 index 000000000..df71bb6a0 --- /dev/null +++ b/tools/tck/lombok.config @@ -0,0 +1,2 @@ +config.stopBubbling = true +lombok.addLombokGeneratedAnnotation = true diff --git a/tools/tck/pom.xml b/tools/tck/pom.xml new file mode 100644 index 000000000..70d653128 --- /dev/null +++ b/tools/tck/pom.xml @@ -0,0 +1,266 @@ + + + 4.0.0 + + dev.openfeature.contrib + parent + [1.0,2.0) + ../../pom.xml + + dev.openfeature.contrib.tools + tck + 0.1.0 + + + ${groupId}.tck + + + false + + 3.27.7 + 4.3.0 + 2.22.1 + 2.0.17 + 2.0.4 + 1.3.0 + + + tck + + Language-agnostic conformance test suite (TCK) for OpenFeature providers. + Bundles the canonical Gherkin feature files, Cucumber step definitions and + abstract JUnit Platform Suite base classes that own the full test lifecycle. + Providers with an external backend extend ContainerizedProviderTckTest, which + starts the vendor-supplied Docker Compose stack, discovers dynamically mapped + ports and drives the standardised HTTP control API. Providers with no backend + extend ProviderTckTest and supply in-process backend control. Provider authors + implement a small factory interface either way. + + https://openfeature.dev + + + + aepfli + Simon Schrottner + OpenFeature + https://openfeature.dev/ + + + + + + + + + + + io.cucumber + cucumber-java + + + + + + io.cucumber + cucumber-junit-platform-engine + + + + + io.cucumber + cucumber-picocontainer + compile + + + + + org.junit.platform + junit-platform-suite + compile + + + + + org.junit.platform + junit-platform-launcher + compile + + + + + org.opentest4j + opentest4j + ${opentest4j.version} + compile + + + + + org.assertj + assertj-core + ${assertj.version} + compile + + + + + org.awaitility + awaitility + ${awaitility.version} + compile + + + + + org.testcontainers + testcontainers + ${testcontainers.version} + provided + true + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + org.junit.jupiter + junit-jupiter + + test + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.6.3 + + + update-spec-submodule + initialize + + exec + + + ${tck.spec.checkout.skip} + git + + submodule + update + --init + spec + + + + + + + + + maven-clean-plugin + 3.5.0 + + + clear-generated-spec-assets + generate-resources + + clean + + + true + + + ${basedir}/src/main/resources/gherkin + + + ${basedir}/src/main/resources/flags + + + ${basedir}/src/main/resources/openapi + + + + + + + + + + maven-resources-plugin + 3.5.0 + + + copy-provider-tck-gherkin + generate-resources + + copy-resources + + + ${basedir}/src/main/resources/gherkin/ + + + ${basedir}/spec/specification/assets/provider-tck/gherkin/ + + **/*.feature + + + + + + + copy-provider-tck-flags + generate-resources + + copy-resources + + + ${basedir}/src/main/resources/flags/ + + + ${basedir}/spec/specification/assets/provider-tck/flags/ + + **/*.json + + + + + + + copy-provider-tck-openapi + generate-resources + + copy-resources + + + ${basedir}/src/main/resources/openapi/ + + + ${basedir}/spec/specification/assets/provider-tck/openapi/ + + **/*.yaml + + + + + + + + + + + diff --git a/tools/tck/spec b/tools/tck/spec new file mode 160000 index 000000000..10b1a610e --- /dev/null +++ b/tools/tck/spec @@ -0,0 +1 @@ +Subproject commit 10b1a610ed0afe6bf0e38b441d7362f5a76cb47d diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendControl.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendControl.java new file mode 100644 index 000000000..a16a989c9 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendControl.java @@ -0,0 +1,85 @@ +package dev.openfeature.contrib.tools.tck; + +/** + * The seam between the TCK's step definitions and whatever manipulates the backend. + * + *

Use {@link #overHttp} for a provider with a backend, or {@link #inProcess()} for one without. + * + * @see Capability + * @see ProviderTckTest + */ +public interface BackendControl { + + /** + * Returns a control over the canonical control API, for a backend this suite did not start. + * + * @param baseUrl root of the control API, without a trailing slash + * @param backendConfiguration the configuration to seed, or {@code null} for none + * @return a control speaking the canonical control API + */ + static BackendControl overHttp(String baseUrl, String backendConfiguration) { + return new HttpBackendControl(baseUrl, backendConfiguration); + } + + /** + * Returns a control for a provider with no backend, which also supplies the provider. + * + * @return an in-process control over a copy of the canonical flag set + */ + static InProcessControl inProcess() { + return new InProcessBackendControl(); + } + + /** Brings the backend to the baseline state before each scenario. */ + void prepareScenario(); + + /** Mutates flag configuration so {@code changing-flag} resolves to a different value. */ + void changeFlag(); + + /** + * Makes the backend unreachable for the rest of the scenario, without stopping any container. + * + * @throws UnsupportedOperationException if this backend has no connection to lose + */ + default void disconnect() { + throw unsupported("disconnect"); + } + + /** + * Makes the backend reachable again after {@link #disconnect()}, preserving flag state. + * + * @throws UnsupportedOperationException if this backend has no connection to restore + */ + default void reconnect() { + throw unsupported("reconnect"); + } + + /** + * Returns a short description of what is being controlled, for logs and failure messages. + * + * @return a human-readable description of this backend control + */ + default String description() { + return getClass().getSimpleName(); + } + + /** + * Returns what kind of backend this run drives; defaults to {@link BackendType#CUSTOM}. + * + * @return what kind of backend this run drives + */ + default BackendType backendType() { + return BackendType.CUSTOM; + } + + /** + * Builds the exception the connection-control defaults throw. + * + * @param operation the operation that is not supported + * @return the exception to throw + */ + default UnsupportedOperationException unsupported(String operation) { + return new UnsupportedOperationException(description() + " does not support '" + operation + + "'; test-configuration bug, remove Capability.STALE and Capability.UNAVAILABLE_INIT."); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendEndpoint.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendEndpoint.java new file mode 100644 index 000000000..e64ffcb52 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendEndpoint.java @@ -0,0 +1,60 @@ +package dev.openfeature.contrib.tools.tck; + +import org.testcontainers.containers.ComposeContainer; + +/** + * Addresses of the running backend stack, handed to + * {@link ContainerizedProviderTckTest#createProvider(BackendEndpoint)} once host ports are known. + */ +public final class BackendEndpoint { + + private final ComposeContainer compose; + private final String defaultService; + + BackendEndpoint(ComposeContainer compose, String defaultService) { + this.compose = compose; + this.defaultService = defaultService; + } + + /** + * Returns the host the stack is reachable on; not necessarily {@code localhost}. + * + * @return the Docker host serving the backend stack + */ + public String host() { + return compose.getServiceHost(defaultService, null); + } + + /** + * Returns the host the named service is reachable on. + * + * @param service the Compose service name + * @return the Docker host serving that service + */ + public String host(String service) { + return compose.getServiceHost(service, null); + } + + /** + * Resolves the mapped host port for a container-internal port on the default backend service. + * + * @param internalPort the container-internal port, as declared by + * {@link ContainerizedProviderTckTest#backendPorts()} + * @return the host port the service is reachable on + */ + public int port(int internalPort) { + return port(defaultService, internalPort); + } + + /** + * Resolves the mapped host port for a container-internal port on a named service; it must be + * declared via {@link ContainerizedProviderTckTest#additionalPorts()}. + * + * @param service the Compose service name + * @param internalPort the container-internal port + * @return the host port the service is reachable on + */ + public int port(String service, int internalPort) { + return compose.getServicePort(service, internalPort); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendType.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendType.java new file mode 100644 index 000000000..c52cce525 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/BackendType.java @@ -0,0 +1,46 @@ +package dev.openfeature.contrib.tools.tck; + +import com.fasterxml.jackson.annotation.JsonValue; + +/** + * What kind of backend a run drove. + * + * @see BackendControl#backendType() + */ +public enum BackendType { + + /** The normative control API: a real backend driven over {@code openapi/control-api.yaml}. */ + HTTP("http"), + + /** Control of a provider with no backend (in-memory, env var, file), inside this JVM. */ + IN_PROCESS("in-process"), + + /** A control the TCK does not ship; the default for custom {@link BackendControl}s. */ + CUSTOM("custom"); + + private final String wireValue; + + BackendType(String wireValue) { + this.wireValue = wireValue; + } + + /** + * Returns the form this value takes in a conformance report. + * + * @return {@code http}, {@code in-process} or {@code custom} + */ + @JsonValue + public String wireValue() { + return wireValue; + } + + /** + * Returns the wire form. + * + * @return {@code http}, {@code in-process} or {@code custom} + */ + @Override + public String toString() { + return wireValue; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/CanonicalFlags.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/CanonicalFlags.java new file mode 100644 index 000000000..3716c3732 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/CanonicalFlags.java @@ -0,0 +1,175 @@ +package dev.openfeature.contrib.tools.tck; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.openfeature.sdk.Value; +import dev.openfeature.sdk.providers.memory.Flag; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The canonical flag set, decoded from {@code flags/canonical-flags.json} (copied from the spec + * submodule and packaged on the classpath). + * + *

Decoding preserves what the scenarios test: {@code missing-flag} stays absent, no flag gets a + * context evaluator, and numbers keep their written kind and width (see {@link #number}). + */ +final class CanonicalFlags { + + /** Classpath location of the canonical flag definition. */ + static final String RESOURCE = "flags/canonical-flags.json"; + + private static final String COMMENT = "$comment"; + + private static final String ENABLED = "ENABLED"; + private static final String DISABLED = "DISABLED"; + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private CanonicalFlags() {} + + /** + * Reads the packaged canonical flag definition. + * + * @return the raw JSON bytes + * @throws IllegalStateException if the definition is not on the classpath + */ + static byte[] definition() { + ClassLoader loader = CanonicalFlags.class.getClassLoader(); + try (InputStream in = loader.getResourceAsStream(RESOURCE)) { + if (in == null) { + throw new IllegalStateException("The canonical flag definition " + RESOURCE + " is not on the " + + "classpath; check out the spec submodule and run 'mvn generate-resources' on tools/tck."); + } + return in.readAllBytes(); + } catch (IOException e) { + throw new UncheckedIOException("Could not read the canonical flag definition " + RESOURCE, e); + } + } + + /** + * Decodes the packaged canonical flag definition into {@code InMemoryProvider} flags. + * + * @return the canonical flag set, in file order, unmodifiable + * @throws IllegalStateException if the definition is missing or malformed + */ + static Map> flagSet() { + return decode(definition()); + } + + /** + * Decodes a canonical flag definition. + * + * @param json the definition, in flagd's flag-definition format + * @return the flag set, in document order, unmodifiable + * @throws IllegalStateException if the document is not the expected shape + */ + static Map> decode(byte[] json) { + JsonNode root; + try { + root = MAPPER.readTree(json); + } catch (IOException e) { + throw new IllegalStateException("The canonical flag definition " + RESOURCE + " is not valid JSON", e); + } + + JsonNode flags = root.path("flags"); + if (!flags.isObject()) { + throw new IllegalStateException("The canonical flag definition " + RESOURCE + + " has no 'flags' object. The format is {\"flags\": {\"\": {...}}}."); + } + + Map> decoded = new LinkedHashMap<>(); + for (Iterator> it = flags.fields(); it.hasNext(); ) { + Map.Entry entry = it.next(); + if (COMMENT.equals(entry.getKey())) { + continue; + } + decoded.put(entry.getKey(), flag(entry.getKey(), entry.getValue())); + } + if (decoded.isEmpty()) { + throw new IllegalStateException("The canonical flag definition " + RESOURCE + " defines no flags."); + } + return Collections.unmodifiableMap(decoded); + } + + private static Flag flag(String key, JsonNode definition) { + String state = definition.path("state").asText(null); + if (!ENABLED.equals(state) && !DISABLED.equals(state)) { + throw new IllegalStateException("Canonical flag '" + key + "' has state '" + state + "', which is neither " + + ENABLED + " nor " + DISABLED + "."); + } + + String defaultVariant = definition.path("defaultVariant").asText(null); + if (defaultVariant == null) { + throw new IllegalStateException("Canonical flag '" + key + "' names no defaultVariant."); + } + + JsonNode variants = definition.path("variants"); + if (!variants.isObject()) { + throw new IllegalStateException("Canonical flag '" + key + "' has no 'variants' object."); + } + + Map values = new LinkedHashMap<>(); + for (Iterator> it = variants.fields(); it.hasNext(); ) { + Map.Entry variant = it.next(); + if (COMMENT.equals(variant.getKey())) { + continue; + } + values.put(variant.getKey(), value(key, variant.getKey(), variant.getValue())); + } + if (!values.containsKey(defaultVariant)) { + throw new IllegalStateException("Canonical flag '" + key + "' resolves to variant '" + defaultVariant + + "', which it does not define. Its variants are " + values.keySet() + "."); + } + + return Flag.builder() + .variants(values) + .defaultVariant(defaultVariant) + .disabled(DISABLED.equals(state)) + .build(); + } + + private static Object value(String key, String variant, JsonNode node) { + switch (node.getNodeType()) { + case BOOLEAN: + return node.booleanValue(); + case STRING: + return node.textValue(); + case NUMBER: + return number(key, variant, node); + case OBJECT: + case ARRAY: + // same conversion as feature-file object values, see TckValues + return Value.objectToValue(MAPPER.convertValue(node, Object.class)); + case NULL: + return null; + default: + throw new IllegalStateException("Canonical flag '" + key + "', variant '" + variant + "' is a " + + node.getNodeType() + ", which is not a flag value."); + } + } + + /** + * Decodes a number as written: fractional to {@link Double}, integral to {@link Integer} or, if + * too wide, {@link Long}. {@code InMemoryProvider} matches by type, so {@code 10.0} must not + * become {@code 10}. + */ + private static Object number(String key, String variant, JsonNode node) { + if (!node.isIntegralNumber()) { + return node.doubleValue(); + } + if (node.canConvertToInt()) { + return node.intValue(); + } + if (node.canConvertToLong()) { + return node.longValue(); + } + throw new IllegalStateException("Canonical flag '" + key + "', variant '" + variant + "' is " + node.asText() + + ", which does not fit a Long."); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/Capability.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/Capability.java new file mode 100644 index 000000000..8a85044f8 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/Capability.java @@ -0,0 +1,309 @@ +package dev.openfeature.contrib.tools.tck; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import org.opentest4j.TestAbortedException; + +/** + * An optional part of the provider contract, gated by one Gherkin tag and declared through + * {@link ProviderTckHarness#capabilities()}. Scenarios with undeclared tags are skipped. + * + *

Tag semantics are defined by + * Appendix + * F. Build declarations with {@link #declarable()} or {@link #declarableExcept}, not + * {@code EnumSet.allOf}/{@code complementOf}, which include {@linkplain #reserved() reserved} and + * {@linkplain #inexpressible() inexpressible} entries and fail the run. + */ +public enum Capability { + + /** Provider performs an initialisation that reaches its backend, with an observable outcome. */ + LIFECYCLE("@lifecycle"), + + /** Provider can be initialised again after {@code shutdown} and serves flags afterwards. */ + REINITIALIZATION("@reinitialization"), + + /** Provider emits lifecycle events at all ({@code PROVIDER_READY}, {@code PROVIDER_ERROR}). */ + EVENTS("@events"), + + /** Provider enters {@code STALE} and emits {@code PROVIDER_STALE} when the backend is lost. */ + STALE("@stale"), + + /** Provider detects flag configuration changes and emits {@code PROVIDER_CONFIGURATION_CHANGED}. */ + CONFIGURATION_CHANGE("@configuration-change"), + + /** Provider supports structured (object) flag values. */ + OBJECT("@object"), + + /** Provider names the variant it resolved. */ + VARIANTS("@variants"), + + /** + * Provider resolves a flag disabled in the management system to the caller's default value. + * Withhold it if the backend response cannot distinguish disabled from absent. + */ + DISABLED_FLAGS("@disabled-flags"), + + /** Provider reports an error state rather than hanging when initialised against a dead backend. */ + UNAVAILABLE_INIT("@unavailable"), + + /** + * Provider coerces between integer and float only when lossless; lossy coercion is a + * {@code TYPE_MISMATCH}. + */ + NUMERIC_COERCION("@numeric-coercion"), + + /** + * Provider reports {@code TYPE_MISMATCH} for a boolean or integer flag requested as a string. + */ + STRING_TYPING("@string-typing"), + + /** + * Backend records native types for float and structured values too; always declared alongside + * {@link #STRING_TYPING}. + */ + FULLY_TYPED_VALUES("@fully-typed-values"), + + /** Provider resolves integers up to 2^53 - 1 exactly; inexpressible in Java. */ + LARGE_INTEGERS("@large-integers", "Client.getIntegerDetails is 32-bit, so 9007199254740991 cannot be requested"), + + /** Provider resolves a flag differently for a matching evaluation context. */ + TARGETING("@targeting"), + + /** Provider reports the standard resolution reasons, with the meanings Appendix F gives them. */ + STANDARD_REASONS("@standard-reasons"), + + /** Provider caches results and invalidates them on change; reserved, no scenario uses it yet. */ + CACHING("@caching", true); + + private final String tag; + private final boolean reserved; + private final String inexpressibleBecause; + + Capability(String tag) { + this.tag = tag; + this.reserved = false; + this.inexpressibleBecause = null; + } + + Capability(String tag, boolean reserved) { + this.tag = tag; + this.reserved = reserved; + this.inexpressibleBecause = null; + } + + Capability(String tag, String inexpressibleBecause) { + this.tag = tag; + this.reserved = false; + this.inexpressibleBecause = inexpressibleBecause; + } + + /** + * Returns the Gherkin tag, including the leading {@code @}, that gates this capability. + * + * @return the Gherkin tag for this capability + */ + String tag() { + return tag; + } + + /** + * Returns whether this capability is reserved and so must not be declared. + * + * @return {@code true} if no scenario carries this capability's tag + */ + boolean reserved() { + return reserved; + } + + /** + * Returns whether the Java SDK cannot express this capability, and so no provider may declare it. + * + * @return {@code true} if this SDK cannot express this capability + */ + boolean inexpressible() { + return inexpressibleBecause != null; + } + + /** + * Returns why this SDK cannot express this capability. + * + * @return the reason, or {@code null} if this capability is expressible + */ + String inexpressibleBecause() { + return inexpressibleBecause; + } + + /** + * Looks up the capability gated by a Gherkin tag. + * + * @param tag a Gherkin tag including the leading {@code @} + * @return the matching capability, or empty if the tag does not gate a capability + */ + static Optional fromTag(String tag) { + return Arrays.stream(values()).filter(c -> c.tag.equals(tag)).findFirst(); + } + + /** + * Returns every capability a provider may declare, excluding reserved and inexpressible ones. + * + * @return the declarable capabilities, as a fresh mutable set + */ + public static EnumSet declarable() { + EnumSet declarable = EnumSet.allOf(Capability.class); + declarable.removeIf(capability -> capability.reserved() || capability.inexpressible()); + return declarable; + } + + /** + * Returns every declarable capability except the given ones. + * + * @param excluded capabilities to withhold + * @return the declarable capabilities minus {@code excluded}, as a fresh mutable set + */ + public static EnumSet declarableExcept(Capability... excluded) { + EnumSet declared = declarable(); + for (Capability capability : excluded) { + declared.remove(capability); + } + return declared; + } + + /** + * Rejects a declaration containing reserved or inexpressible capabilities. + * + * @param declared the capabilities a harness declares + * @throws IllegalArgumentException if any of them is reserved or inexpressible + */ + public static void requireDeclarable(Collection declared) { + List reservedTags = new ArrayList<>(); + List inexpressibleTags = new ArrayList<>(); + for (Capability capability : declared) { + if (capability.reserved()) { + reservedTags.add(capability.name() + " (" + capability.tag() + ")"); + } else if (capability.inexpressible()) { + inexpressibleTags.add( + capability.name() + " (" + capability.tag() + "): " + capability.inexpressibleBecause()); + } + } + if (!reservedTags.isEmpty()) { + throw new IllegalArgumentException("capabilities() declares reserved " + reservedTags + + ", which no scenario in the suite carries; use Capability.declarable() instead."); + } + if (!inexpressibleTags.isEmpty()) { + throw new IllegalArgumentException("capabilities() declares " + inexpressibleTags + + ", which the Java SDK cannot express. This is not a reserved capability; " + + "the scenarios exist and are asked in languages whose API is wide enough."); + } + } + + // scenario gate; static so harnesses cannot override it + + /** + * Applies the gate rules to a scenario of unknown source; the vocabulary check is skipped. + * + * @param tags the scenario's Gherkin tags, including the leading at-sign + * @param declared the capabilities the provider declares + * @throws IllegalStateException if a tag names a reserved capability + * @throws TestAbortedException if a tag gates an inexpressible or an undeclared capability + */ + static void requireDeclared(Collection tags, Set declared) { + requireDeclared(null, tags, declared); + } + + /** + * Applies all gate rules to a scenario. + * + * @param source the scenario's feature file, as Cucumber reports it, or {@code null} if unknown + * @param tags the scenario's Gherkin tags, including the leading at-sign + * @param declared the capabilities the provider declares + * @throws IllegalStateException if a tag is reserved, or a canonical scenario has an unknown tag + * @throws TestAbortedException if a tag gates an inexpressible or an undeclared capability + */ + public static void requireDeclared(URI source, Collection tags, Set declared) { + requireNoExpiredReservation(tags); + requireKnownVocabulary(source, tags); + for (String tag : tags) { + Optional found = fromTag(tag); + if (!found.isPresent()) { + // adopter tag; gates nothing + continue; + } + Capability capability = found.get(); + if (capability.inexpressible()) { + throw new TestAbortedException("Skipped: the Java SDK cannot express capability " + + capability.name() + " (tag " + tag + "); " + capability.inexpressibleBecause() + + "; not the provider under test declining."); + } + if (!declared.contains(capability)) { + throw new TestAbortedException("Skipped: provider does not declare capability " + capability.name() + + " (tag " + tag + "). Declared capabilities: " + declared); + } + } + } + + /** + * Fails if a canonical (non-extension) scenario carries a tag unknown to {@link Capability}. + * + * @param source the scenario's feature file, as Cucumber reports it, or {@code null} if unknown + * @param tags the scenario's Gherkin tags, including the leading at-sign + * @throws IllegalStateException if the scenario is canonical and carries an unknown tag + */ + private static void requireKnownVocabulary(URI source, Collection tags) { + if (!isCanonical(source)) { + return; + } + List unknown = new ArrayList<>(); + for (String tag : tags) { + if (!fromTag(tag).isPresent()) { + unknown.add(tag); + } + } + if (unknown.isEmpty()) { + return; + } + throw new IllegalStateException("Canonical scenario " + source + " carries unknown tags " + unknown + + "; add them to Capability or move the feature under " + ProviderTck.EXTENSIONS + "/."); + } + + // canonical if the parent directory is FEATURES; null is not canonical + private static boolean isCanonical(URI source) { + if (source == null) { + return false; + } + String path = source.toString().replace('\\', '/'); + int lastSeparator = path.lastIndexOf('/'); + if (lastSeparator < 0) { + return false; + } + String parent = path.substring(0, lastSeparator); + int start = Math.max(parent.lastIndexOf('/'), parent.lastIndexOf(':')) + 1; + return ProviderTck.FEATURES.equals(parent.substring(start)); + } + + /** + * Fails if a scenario carries a reserved capability's tag, rather than silently skipping it. + * + * @param tags the scenario's Gherkin tags, including the leading at-sign + * @throws IllegalStateException if a tag names a reserved capability + */ + private static void requireNoExpiredReservation(Collection tags) { + List expired = new ArrayList<>(); + for (String tag : tags) { + Optional capability = fromTag(tag); + if (capability.isPresent() && capability.get().reserved()) { + expired.add(capability.get().name() + " (" + capability.get().tag() + ")"); + } + } + if (expired.isEmpty()) { + return; + } + throw new IllegalStateException("Scenario carries reserved " + expired + + "; unreserve it in Capability or use your own tag in extensions/."); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ContainerizedProviderTckTest.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ContainerizedProviderTckTest.java new file mode 100644 index 000000000..cd2004b72 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ContainerizedProviderTckTest.java @@ -0,0 +1,192 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import java.io.File; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +/** + * Base JUnit Platform Suite for providers that talk to an external backend. + * + *

Manages the Docker Compose lifecycle (started once, never restarted), port discovery and the + * control over the canonical control API. See {@code tools/tck/README.md} for an example. + * + *

Declared ports are container-internal; resolve host ports via {@link BackendEndpoint}. + * + * @see ProviderTckTest + * @see BackendControl#overHttp + */ +public abstract class ContainerizedProviderTckTest extends ProviderTckTest { + + private static final Logger log = LoggerFactory.getLogger(ContainerizedProviderTckTest.class); + + private ComposeContainer compose; + private BackendEndpoint endpoint; + private HttpBackendControl control; + + /** + * Returns the Docker Compose file for the backend stack, relative to the module directory. The + * stack must not pin host ports. + * + * @return the Compose file describing the backend stack + */ + public abstract File composeFile(); + + /** + * Returns the container-internal ports on {@link #backendService()} the provider connects to. + * {@link #controlPort()} is exposed automatically. + * + * @return container-internal ports the provider connects to + */ + public abstract List backendPorts(); + + /** + * Creates the provider under test, configured against the running backend. Do not call + * {@code setProvider} or {@code initialize}; the TCK owns the lifecycle. + * + * @param endpoint host and mapped ports of the running backend stack + * @return a configured, uninitialised provider + */ + public abstract FeatureProvider createProvider(BackendEndpoint endpoint); + + /** + * {@inheritDoc} + * + *

Delegates to {@link #createProvider(BackendEndpoint)} with the running stack's endpoint. + */ + @Override + public final FeatureProvider createProvider() { + return createProvider(endpoint()); + } + + /** + * {@inheritDoc} + * + *

Abstract here; a provider with a real backend can always be pointed at a closed port. + */ + @Override + public abstract FeatureProvider createUnavailableProvider(); + + /** + * Returns the Compose service name hosting the control API and backend. + * + * @return the Compose service name, {@code backend} by default + */ + public String backendService() { + return "backend"; + } + + /** + * Returns the container-internal port the control API listens on. + * + * @return the control API port, {@code 8080} by default + */ + public int controlPort() { + return 8080; + } + + /** + * Returns extra Compose services and their container-internal ports to expose; resolve them + * with {@link BackendEndpoint#port(String, int)}. + * + * @return additional services and ports to expose, empty by default + */ + public Map> additionalPorts() { + return Collections.emptyMap(); + } + + /** + * Returns the backend configuration name passed to {@code POST /start?config=...}; distinct + * from {@link ProviderTckHarness#configuration()}. + * + * @return the backend configuration name passed to {@code POST /start}, {@code default} by + * default + */ + public String backendConfiguration() { + return "default"; + } + + /** + * Returns how long to wait for the Compose stack and its control API to become reachable. + * + * @return the stack startup timeout, 60 seconds by default + */ + public Duration startupTimeout() { + return Duration.ofSeconds(60); + } + + /** + * {@inheritDoc} + * + *

Starts the Compose stack and waits up to {@link #startupTimeout()} for the control API. + */ + @Override + public final void startSuite() { + compose = startCompose(); + endpoint = new BackendEndpoint(compose, backendService()); + control = new HttpBackendControl( + "http://" + endpoint.host() + ":" + endpoint.port(controlPort()), backendConfiguration()); + control.awaitReady(startupTimeout()); + log.info("Control API ready at {}", control.baseUrl()); + } + + /** {@inheritDoc} */ + @Override + public final void stopSuite() { + if (compose != null) { + compose.stop(); + } + compose = null; + endpoint = null; + control = null; + } + + /** {@inheritDoc} */ + @Override + public final BackendControl backendControl() { + return control; + } + + /** + * Returns the host and mapped ports of the running stack. + * + * @return the backend endpoint + * @throws IllegalStateException if the stack has not been started + */ + protected final BackendEndpoint endpoint() { + if (endpoint == null) { + throw new IllegalStateException("The Compose stack has not been started yet."); + } + return endpoint; + } + + private ComposeContainer startCompose() { + File composeFile = composeFile(); + if (!composeFile.isFile()) { + throw new IllegalStateException("Compose file not found: " + composeFile.getAbsolutePath() + + ". ContainerizedProviderTckTest.composeFile() is resolved relative to the module directory."); + } + ComposeContainer stack = new ComposeContainer(composeFile); + + stack.withExposedService(backendService(), controlPort(), Wait.forListeningPort()); + for (Integer port : backendPorts()) { + stack.withExposedService(backendService(), port, Wait.forListeningPort()); + } + for (Map.Entry> service : additionalPorts().entrySet()) { + for (Integer port : service.getValue()) { + stack.withExposedService(service.getKey(), port, Wait.forListeningPort()); + } + } + stack.withStartupTimeout(startupTimeout()); + + log.info("Starting Compose stack {} (started once per suite, never restarted)", composeFile.getAbsolutePath()); + stack.start(); + return stack; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/FlagUnderTest.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/FlagUnderTest.java new file mode 100644 index 000000000..9bd8e1673 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/FlagUnderTest.java @@ -0,0 +1,53 @@ +package dev.openfeature.contrib.tools.tck; + +/** + * The flag a scenario is exercising: key, declared type (which selects the typed evaluation call), + * and code default. + */ +public final class FlagUnderTest { + + private final String key; + private final String type; + private final Object defaultValue; + + /** + * Creates a flag under test. + * + * @param key the flag key + * @param type the declared type, one of {@code Boolean}, {@code String}, {@code Integer}, + * {@code Float} or {@code Object} + * @param defaultValue the code default passed to the evaluation call + */ + public FlagUnderTest(String key, String type, Object defaultValue) { + this.key = key; + this.type = type; + this.defaultValue = defaultValue; + } + + /** + * Returns the flag key. + * + * @return the flag key + */ + public String key() { + return key; + } + + /** + * Returns the declared flag type. + * + * @return the declared flag type + */ + public String type() { + return type; + } + + /** + * Returns the code default passed to the evaluation call. + * + * @return the code default + */ + public Object defaultValue() { + return defaultValue; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/HttpBackendControl.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/HttpBackendControl.java new file mode 100644 index 000000000..a8229e3f8 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/HttpBackendControl.java @@ -0,0 +1,212 @@ +package dev.openfeature.contrib.tools.tck; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * {@link BackendControl} backed by the HTTP control API in {@code openapi/control-api.yaml}. + * + *

Constructed by {@link ContainerizedProviderTckTest}; never restarts containers. + */ +final class HttpBackendControl implements BackendControl { + + private static final Logger log = LoggerFactory.getLogger(HttpBackendControl.class); + + private final HttpClient http; + private final String baseUrl; + private final String backendConfiguration; + + // whether optional /reset is supported; null until probed + private Boolean resetSupported; + + // whether the backend was last left unreachable, so /reset alone won't suffice + private boolean backendStopped; + + /** + * Creates a control client for a running backend. No post-command settle; control calls return + * once the backend has acted. + * + * @param baseUrl the control API base URL, without a trailing slash + * @param backendConfiguration the backend configuration name defining the canonical baseline + */ + HttpBackendControl(String baseUrl, String backendConfiguration) { + this.baseUrl = baseUrl; + this.backendConfiguration = backendConfiguration; + this.http = + HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); + } + + /** + * Returns the base URL of the control API, for diagnostics. + * + * @return the control API base URL + */ + public String baseUrl() { + return baseUrl; + } + + @Override + public String description() { + return "HTTP control API at " + baseUrl; + } + + /** + * {@inheritDoc} + * + *

Always {@link BackendType#HTTP}. + */ + @Override + public BackendType backendType() { + return BackendType.HTTP; + } + + /** + * {@inheritDoc} + * + *

Uses {@code POST /reset} if the backend is running, else {@code POST /start?config=...}. + */ + @Override + public void prepareScenario() { + if (backendStopped) { + start(); + } else { + reset(); + } + } + + /** + * {@inheritDoc} + * + *

Issues {@code POST /change}. + */ + @Override + public void changeFlag() { + post("/change"); + } + + /** + * {@inheritDoc} + * + *

Issues {@code POST /stop}. + */ + @Override + public void disconnect() { + post("/stop"); + backendStopped = true; + } + + /** + * {@inheritDoc} + * + *

Issues {@code POST /start?config=...}, which also restores the baseline flag state. + */ + @Override + public void reconnect() { + start(); + } + + /** + * Waits until the control API accepts commands, probing optional {@code GET /healthz} (a + * {@code 404} counts as ready). + * + * @param timeout how long to keep probing + */ + public void awaitReady(Duration timeout) { + long deadline = System.nanoTime() + timeout.toNanos(); + RuntimeException last = null; + while (System.nanoTime() < deadline) { + try { + HttpResponse response = http.send( + HttpRequest.newBuilder(URI.create(baseUrl + "/healthz")) + .GET() + .timeout(Duration.ofSeconds(5)) + .build(), + HttpResponse.BodyHandlers.discarding()); + if (response.statusCode() == 200 || response.statusCode() == 404) { + return; + } + last = new IllegalStateException("control API not ready, HTTP " + response.statusCode()); + } catch (IOException e) { + last = new IllegalStateException("control API not reachable at " + baseUrl, e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while waiting for the control API", e); + } + sleep(Duration.ofMillis(200)); + } + throw new IllegalStateException("control API at " + baseUrl + " did not become ready within " + timeout, last); + } + + /** Starts the backend, seeding the baseline flag state. */ + private void start() { + post("/start?config=" + backendConfiguration); + backendStopped = false; + } + + /** Restores baseline via {@code POST /reset}, falling back to {@code /start} on 404/501. */ + private void reset() { + if (Boolean.FALSE.equals(resetSupported)) { + start(); + return; + } + HttpResponse response = send("/reset"); + if (response.statusCode() == 404 || response.statusCode() == 501) { + if (resetSupported == null) { + log.info( + "Control API at {} does not implement POST /reset (HTTP {}); " + + "falling back to POST /start?config={}.", + baseUrl, + response.statusCode(), + backendConfiguration); + } + resetSupported = false; + start(); + return; + } + expectSuccess("/reset", response); + resetSupported = true; + } + + private void post(String path) { + expectSuccess(path, send(path)); + } + + private HttpResponse send(String path) { + HttpRequest request = HttpRequest.newBuilder(URI.create(baseUrl + path)) + .POST(HttpRequest.BodyPublishers.noBody()) + .timeout(Duration.ofSeconds(30)) + .build(); + try { + return http.send(request, HttpResponse.BodyHandlers.discarding()); + } catch (IOException e) { + throw new IllegalStateException("control API call POST " + baseUrl + path + " failed", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted during control API call POST " + path, e); + } + } + + private void expectSuccess(String path, HttpResponse response) { + if (response.statusCode() != 200) { + throw new IllegalStateException( + "control API call POST " + baseUrl + path + " returned HTTP " + response.statusCode() + + ", expected 200. See openapi/control-api.yaml for the expected contract."); + } + } + + /** Pauses between readiness probes. */ + private static void sleep(Duration duration) { + try { + Thread.sleep(duration.toMillis()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while waiting for the control API to become ready", e); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/InProcessBackendControl.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/InProcessBackendControl.java new file mode 100644 index 000000000..24909fdbb --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/InProcessBackendControl.java @@ -0,0 +1,131 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.providers.memory.Flag; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.util.HashMap; +import java.util.Map; + +/** + * {@link BackendControl} that manipulates the SDK's {@link InMemoryProvider} directly. Only for + * providers with no backend; others use {@link ContainerizedProviderTckTest}. + * + *

{@link #disconnect()} and {@link #reconnect()} throw, so leave {@link Capability#STALE} and + * {@link Capability#UNAVAILABLE_INIT} undeclared. Wire both harness factory methods to one instance: + * + * 

{@code
+ * private final InProcessControl control = BackendControl.inProcess();
+ *
+ * @Override
+ * public BackendControl backendControl() {
+ *     return control;
+ * }
+ *
+ * @Override
+ * public FeatureProvider createProvider() {
+ *     return control.createProvider();
+ * }
+ * }
+ */ +final class InProcessBackendControl implements InProcessControl { + + // flag mutated by changeFlag(); also named by POST /change + private static final String CHANGING_FLAG = "changing-flag"; + + // canonical flag set; never mutated, which provides scenario isolation + private final Map> baseline = CanonicalFlags.flagSet(); + + // changing-flag as defined, source of its variants + private final Flag changingBaseline = requireChangingFlag(); + + // provider for the current scenario, null between scenarios + private InMemoryProvider current; + + // variant changing-flag currently resolves to + private String changingVariant = changingBaseline.getDefaultVariant(); + + /** + * Creates a fresh provider for the next scenario, seeded with the canonical flag set. + * + * @return a configured, uninitialised in-memory provider + */ + public InMemoryProvider createProvider() { + changingVariant = changingBaseline.getDefaultVariant(); + current = new InMemoryProvider(new HashMap<>(baseline)); + return current; + } + + @Override + public String description() { + return "in-process control of " + InMemoryProvider.class.getSimpleName(); + } + + /** + * {@inheritDoc} + * + *

Always {@link BackendType#IN_PROCESS}. + */ + @Override + public BackendType backendType() { + return BackendType.IN_PROCESS; + } + + /** + * {@inheritDoc} + * + *

Drops the previous provider; the next {@link #createProvider()} starts from the baseline. + */ + @Override + public void prepareScenario() { + current = null; + } + + /** + * {@inheritDoc} + * + *

Flips {@code changing-flag} between variants via + * {@link InMemoryProvider#updateFlag(String, Flag)}, which emits the provider's own event. + */ + @Override + public void changeFlag() { + changingVariant = otherVariant(changingVariant); + requireProvider().updateFlag(CHANGING_FLAG, changingFlag(changingVariant)); + } + + /** Returns a variant of {@code changing-flag} other than the given one. */ + private String otherVariant(String resolved) { + for (String variant : changingBaseline.getVariants().keySet()) { + if (!variant.equals(resolved)) { + return variant; + } + } + throw new IllegalStateException("The canonical definition of '" + CHANGING_FLAG + "' has only the variant '" + + resolved + "'; changeFlag() needs at least two."); + } + + /** Rebuilds {@code changing-flag} resolving to the given variant. */ + private Flag changingFlag(String defaultVariant) { + return Flag.builder() + .variants(changingBaseline.getVariants()) + .defaultVariant(defaultVariant) + .disabled(changingBaseline.isDisabled()) + .build(); + } + + /** Returns the canonical {@code changing-flag}, failing if absent. */ + private Flag requireChangingFlag() { + Flag flag = baseline.get(CHANGING_FLAG); + if (flag == null) { + throw new IllegalStateException("The canonical flag definition " + CanonicalFlags.RESOURCE + + " does not define '" + CHANGING_FLAG + "', required by POST /change."); + } + return flag; + } + + private InMemoryProvider requireProvider() { + if (current == null) { + throw new IllegalStateException("No in-memory provider exists for this scenario; use " + + "'Given a stable provider' before any step that changes flag state."); + } + return current; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/InProcessControl.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/InProcessControl.java new file mode 100644 index 000000000..b17b7625c --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/InProcessControl.java @@ -0,0 +1,19 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.providers.memory.InMemoryProvider; + +/** + * A {@link BackendControl} that also supplies the provider, for a provider with no backend. + * + *

In-process the flag store and the provider are one object, so {@code changeFlag()} has to + * reach the live instance. Obtain one from {@link BackendControl#inProcess()}. + */ +public interface InProcessControl extends BackendControl { + + /** + * Returns the provider this control is the flag store for. + * + * @return a provider over an untouched copy of the canonical flag set + */ + InMemoryProvider createProvider(); +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/KnownDeviation.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/KnownDeviation.java new file mode 100644 index 000000000..e85a9ae51 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/KnownDeviation.java @@ -0,0 +1,94 @@ +package dev.openfeature.contrib.tools.tck; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * A declared gap: this provider fails a requirement (a {@code MUST}), as opposed to an optional + * capability it chose not to offer. Declared via {@link ProviderTckHarness#knownDeviations()}. + * + *

Prefer declaring the capability and letting the scenario fail. {@link #capability} is + * {@code null} for mandatory scenarios and must not be reserved or inexpressible. See + * Appendix + * F. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public final class KnownDeviation { + + /** The capability tag the deviation concerns, or {@code null} when it maps to none. */ + public final String capability; + + /** Where the gap is tracked, or {@code null} when it is not tracked anywhere. */ + public final String issue; + + /** What the gap is; never {@code null}. */ + public final String summary; + + private KnownDeviation(Capability capability, String issue, String summary) { + requireDeviable(capability); + requireSummary(summary); + this.capability = capability == null ? null : capability.tag(); + this.issue = issue; + this.summary = summary; + } + + /** + * Refuses a deviation without a summary. + * + * @param summary the summary the adopter supplied + * @throws IllegalArgumentException if it is null or blank + */ + private static void requireSummary(String summary) { + if (summary == null || summary.trim().isEmpty()) { + throw new IllegalArgumentException( + "knownDeviations() records a deviation with no summary; describe what the provider gets wrong."); + } + } + + /** + * Refuses a deviation against a reserved or inexpressible capability. + * + * @param capability the capability the deviation names, or {@code null} + * @throws IllegalArgumentException if the capability is reserved or inexpressible + */ + private static void requireDeviable(Capability capability) { + if (capability == null) { + return; + } + if (capability.reserved()) { + throw new IllegalArgumentException("knownDeviations() records a deviation against reserved " + + capability.name() + " (" + capability.tag() + "), which no scenario in the suite " + + "carries; use null for a gap against a mandatory scenario."); + } + if (capability.inexpressible()) { + throw new IllegalArgumentException("knownDeviations() records a deviation against " + + capability.name() + " (" + capability.tag() + "), which the Java SDK cannot " + + "express: " + capability.inexpressibleBecause() + "; its scenarios are never run " + + "against Java providers, so remove this deviation."); + } + } + + /** + * Records a deviation tracked in an issue; the preferred form. + * + * @param capability the capability the gap is about, or {@code null} for none + * @param issue a URI where the gap is tracked + * @param summary what the gap is; required + * @return the deviation, ready to declare + * @throws IllegalArgumentException if the capability is reserved or inexpressible + */ + public static KnownDeviation tracked(Capability capability, String issue, String summary) { + return new KnownDeviation(capability, issue, summary); + } + + /** + * Records a deviation not yet tracked anywhere; prefer {@link #tracked}. + * + * @param capability the capability the gap is about, or {@code null} for none + * @param summary what the gap is; required + * @return the deviation, ready to declare + * @throws IllegalArgumentException if the capability is reserved or inexpressible + */ + public static KnownDeviation untracked(Capability capability, String summary) { + return new KnownDeviation(capability, null, summary); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderEventRecord.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderEventRecord.java new file mode 100644 index 000000000..b93ee0f6a --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderEventRecord.java @@ -0,0 +1,44 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.EventDetails; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * A provider event observed by a scenario's event handler, tagged with the Gherkin word that + * registered the handler ({@code ready}, {@code error}, {@code stale}, {@code change}). + */ +@SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "SDK event payload is handed on as-is for assertion") +public final class ProviderEventRecord { + + private final String type; + private final EventDetails details; + + /** + * Records an observed event. + * + * @param type the Gherkin event word the handler was registered under + * @param details the event payload delivered by the SDK + */ + public ProviderEventRecord(String type, EventDetails details) { + this.type = type; + this.details = details; + } + + /** + * Returns the Gherkin event word. + * + * @return the event type word + */ + public String type() { + return type; + } + + /** + * Returns the event payload delivered by the SDK. + * + * @return the event details + */ + public EventDetails details() { + return details; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTck.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTck.java new file mode 100644 index 000000000..1c14985cd --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTck.java @@ -0,0 +1,44 @@ +package dev.openfeature.contrib.tools.tck; + +/** + * The suite's Cucumber configuration. Only the three constants an extension composes with are + * public; the lifecycle lives with the glue it serves. + */ +public final class ProviderTck { + + // compile-time constants, so the suite classes can use them in annotations + + /** Classpath directory of the canonical feature files; reserved, use {@link #EXTENSIONS}. */ + static final String FEATURES = "gherkin"; + + /** + * Classpath directory for adopter feature files; separate from {@link #FEATURES} so they cannot + * shadow canonical files. + */ + public static final String EXTENSIONS = "extensions"; + + /** Package holding the canonical step definitions. */ + static final String GLUE = "dev.openfeature.contrib.tools.tck.internal.steps"; + + /** Package for adopter step definitions; may be absent. */ + public static final String EXTENSION_GLUE = "openfeature.tck.extensions"; + + /** The glue path: canonical steps plus the extension package. */ + public static final String ALL_GLUE = GLUE + "," + EXTENSION_GLUE; + + /** The Cucumber plugins the suite registers. */ + static final String PLUGINS = "summary"; + + /** Parallel execution off; control API state is global to the backend. */ + static final String PARALLEL = "false"; + + /** Scenarios of one feature stay on one thread. */ + static final String FEATURE_EXECUTION_MODE = "same_thread"; + + /** Object factory that injects {@link TckState} into every step class. */ + static final String OBJECT_FACTORY = "dev.openfeature.contrib.tools.tck.internal.steps.TckObjectFactory"; + + private ProviderTck() { + // constants only + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckHarness.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckHarness.java new file mode 100644 index 000000000..03eb28866 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckHarness.java @@ -0,0 +1,111 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * The lifecycle-agnostic contract a provider author implements to run the Provider TCK. + * + *

Extend {@link ContainerizedProviderTckTest} for a provider with an external backend, or + * {@link ProviderTckTest} for one with none. See {@code tools/tck/README.md}. + * + * @see ProviderTckTest + * @see ContainerizedProviderTckTest + */ +public interface ProviderTckHarness { + + /** + * Creates the provider under test, against a running backend seeded with the canonical flags. + * Called once per scenario; do not call {@code setProvider} or {@code initialize}. + * + * @return a configured, uninitialised provider + */ + FeatureProvider createProvider(); + + /** + * Returns the seam through which the TCK manipulates the backend. Called after + * {@link #startSuite()}; must not be {@code null}. + * + * @return the backend control for this suite + */ + BackendControl backendControl(); + + /** + * Creates a provider pointed at a closed port, with a short connection timeout. Only needed + * if {@link Capability#UNAVAILABLE_INIT} is declared. + * + * @return a configured provider that cannot reach a backend + */ + default FeatureProvider createUnavailableProvider() { + throw new UnsupportedOperationException(getClass().getName() + " does not implement " + + "createUnavailableProvider(); test-configuration bug: remove Capability.UNAVAILABLE_INIT " + + "or implement this method."); + } + + /** + * Declares which optional capabilities this provider supports; scenarios for others are + * skipped. Defaults to {@link Capability#declarable()}; narrow with + * {@link Capability#declarableExcept}. + * + * @return the capabilities this provider supports + */ + default Set capabilities() { + return Capability.declarable(); + } + + /** + * Declares known gaps against required parts of the contract; see {@link KnownDeviation}. + * + * @return the deviations this provider acknowledges, empty by default + */ + default List knownDeviations() { + return Collections.emptyList(); + } + + /** + * Returns the name of the provider configuration (mode) this suite exercises. Derived from the + * class name by default, e.g. {@code MyProviderInProcessTest} becomes + * {@code my-provider-in-process}. + * + * @return a short name for this configuration + */ + default String configuration() { + return ReportNames.configurationOf(getClass()); + } + + /** + * Prepares whatever must exist before the first scenario. Called once, paired with + * {@link #stopSuite()}. + */ + default void startSuite() { + // nothing to start by default + } + + /** + * Releases whatever {@link #startSuite()} created; must tolerate a partial start. + */ + default void stopSuite() { + // nothing to stop by default + } + + /** + * Returns how long to wait for a provider event; should exceed worst-case detection latency. + * + * @return the default event await timeout, 12 seconds by default + */ + default Duration eventTimeout() { + return Duration.ofSeconds(12); + } + + /** + * Returns how long to wait for a provider to reach {@code READY} during initialisation. + * + * @return the readiness timeout, 30 seconds by default + */ + default Duration readyTimeout() { + return Duration.ofSeconds(30); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckTest.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckTest.java new file mode 100644 index 000000000..31da2bda5 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckTest.java @@ -0,0 +1,33 @@ +package dev.openfeature.contrib.tools.tck; + +import io.cucumber.junit.platform.engine.Constants; +import org.junit.platform.suite.api.ConfigurationParameter; +import org.junit.platform.suite.api.IncludeEngines; +import org.junit.platform.suite.api.SelectClasspathResource; +import org.junit.platform.suite.api.Suite; + +/** + * Base JUnit Platform Suite for the Provider TCK, carrying all Cucumber configuration. + * + *

Extend directly for a provider with no backend (e.g. with {@link BackendControl#inProcess()}); + * otherwise extend {@link ContainerizedProviderTckTest}. Scenarios run sequentially. + * + *

Extra scenarios go in {@code src/test/resources/extensions/} with steps in package + * {@code openfeature.tck.extensions}. + * + * @see ProviderTckHarness + * @see ContainerizedProviderTckTest + * @see ProviderTck + */ +@Suite +@IncludeEngines("cucumber") +@SelectClasspathResource(ProviderTck.FEATURES) +@SelectClasspathResource(ProviderTck.EXTENSIONS) +@ConfigurationParameter(key = Constants.PLUGIN_PROPERTY_NAME, value = ProviderTck.PLUGINS) +@ConfigurationParameter( + key = Constants.EXECUTION_MODE_FEATURE_PROPERTY_NAME, + value = ProviderTck.FEATURE_EXECUTION_MODE) +@ConfigurationParameter(key = Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, value = ProviderTck.PARALLEL) +@ConfigurationParameter(key = Constants.GLUE_PROPERTY_NAME, value = ProviderTck.ALL_GLUE) +@ConfigurationParameter(key = Constants.OBJECT_FACTORY_PROPERTY_NAME, value = ProviderTck.OBJECT_FACTORY) +public abstract class ProviderTckTest implements ProviderTckHarness {} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ReportNames.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ReportNames.java new file mode 100644 index 000000000..d2a12cd06 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ReportNames.java @@ -0,0 +1,36 @@ +package dev.openfeature.contrib.tools.tck; + +import java.util.Locale; + +/** Derives the names a run of the suite is identified by. */ +final class ReportNames { + + /** Suite class name suffixes stripped when deriving a name. */ + private static final String[] SUITE_SUFFIXES = {"TckTest", "TCKTest", "TckSuite", "Test", "IT"}; + + /** Used when a name sanitises to nothing, e.g. for an anonymous class. */ + private static final String FALLBACK = "tck"; + + private ReportNames() {} + + /** + * Derives a configuration name from a suite class, e.g. {@code MyProviderInProcessTest} becomes + * {@code my-provider-in-process}. + * + * @param suite the concrete suite class + * @return a hyphenated, lower-case configuration name + */ + static String configurationOf(Class suite) { + String simple = suite.getSimpleName(); + for (String suffix : SUITE_SUFFIXES) { + if (simple.length() > suffix.length() && simple.endsWith(suffix)) { + simple = simple.substring(0, simple.length() - suffix.length()); + break; + } + } + String hyphenated = simple.replaceAll("([a-z0-9])([A-Z])", "$1-$2") + .replaceAll("([A-Z]+)([A-Z][a-z])", "$1-$2") + .toLowerCase(Locale.ROOT); + return hyphenated.isEmpty() ? FALLBACK : hyphenated; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckState.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckState.java new file mode 100644 index 000000000..69fc60f40 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/TckState.java @@ -0,0 +1,52 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.Client; +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.FlagEvaluationDetails; +import dev.openfeature.sdk.MutableContext; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * Scenario-scoped mutable state, injected into step definitions by PicoContainer. Suite-wide state + * lives in {@link ProviderTck}. + */ +public class TckState { + + /** Client bound to the domain the provider under test is registered under. */ + public Client client; + + /** The provider under test. */ + public FeatureProvider provider; + + /** Scenario-scoped OpenFeature domain, so scenarios cannot see each other's providers. */ + public String domain; + + /** The flag the current scenario is exercising. */ + public FlagUnderTest flag; + + /** Evaluation context accumulated by the context steps. */ + public MutableContext context = new MutableContext(); + + /** Result of the most recent evaluation. */ + public FlagEvaluationDetails evaluation; + + /** A previously resolved value, so a later evaluation can be asserted to differ. */ + public Object rememberedValue; + + /** Exception from the most recent provider call, recorded rather than propagated. */ + public Exception thrown; + + /** The call {@link #thrown} came from; {@code null} when none. */ + public String thrownBy; + + /** Duration of the most recent {@code shutdown()} call, or {@code null} if none. */ + public Duration shutdownDuration; + + /** Events observed by handlers registered in this scenario. */ + public final ConcurrentLinkedQueue events = new ConcurrentLinkedQueue<>(); + + /** The event most recently matched by an await step. */ + public Optional lastEvent = Optional.empty(); +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/TckSuiteListener.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/TckSuiteListener.java new file mode 100644 index 000000000..f8e88d0ca --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/TckSuiteListener.java @@ -0,0 +1,72 @@ +package dev.openfeature.contrib.tools.tck.internal; + +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import java.lang.reflect.Modifier; +import java.util.Optional; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.support.descriptor.ClassSource; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tracks the currently executing TCK suite class so step definitions can find its harness. + * + *

Registered via {@code META-INF/services}; inert for non-TCK containers. + */ +public class TckSuiteListener implements TestExecutionListener { + + private static final Logger log = LoggerFactory.getLogger(TckSuiteListener.class); + + private static volatile Class current; + + @Override + public void executionStarted(TestIdentifier testIdentifier) { + harnessClassOf(testIdentifier).ifPresent(suite -> { + current = suite; + log.debug("TCK suite started: {}", suite.getName()); + }); + } + + @Override + public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult result) { + harnessClassOf(testIdentifier).ifPresent(suite -> { + if (suite.equals(current)) { + current = null; + } + }); + } + + /** + * Returns the suite class currently executing, if it is a TCK suite. + * + * @return the executing suite class, or empty when none is running or the listener was not + * registered + */ + public static Optional> currentSuite() { + return Optional.ofNullable(current); + } + + /** The TCK suite class an identifier stands for, if it is one. */ + private static Optional> harnessClassOf(TestIdentifier testIdentifier) { + return testIdentifier + .getSource() + .filter(ClassSource.class::isInstance) + .map(ClassSource.class::cast) + .flatMap(TckSuiteListener::loadClass) + .filter(ProviderTckHarness.class::isAssignableFrom) + .filter(candidate -> !Modifier.isAbstract(candidate.getModifiers())) + .map(candidate -> candidate.asSubclass(ProviderTckHarness.class)); + } + + private static Optional> loadClass(ClassSource source) { + try { + // routine for sources from other engines; must not fail the run + return Optional.of(source.getJavaClass()); + } catch (RuntimeException e) { + log.trace("Ignoring unloadable class source {}", source, e); + return Optional.empty(); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/TckValues.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/TckValues.java new file mode 100644 index 000000000..e9b6eb58b --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/TckValues.java @@ -0,0 +1,61 @@ +package dev.openfeature.contrib.tools.tck.internal; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.openfeature.sdk.Value; +import java.io.IOException; + +/** + * Converts feature-file strings into typed values; {@code Integer} yields {@link Integer} and + * {@code Float} yields {@link Double}, with no widening. + */ +public final class TckValues { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private TckValues() {} + + /** + * Converts a feature-file string to a typed value. + * + * @param value the raw string from the feature file; the literal {@code null} yields + * {@code null} + * @param type the declared type, one of {@code Boolean}, {@code String}, {@code Integer}, + * {@code Float} or {@code Object} + * @return the converted value + */ + public static Object convert(String value, String type) { + if ("null".equals(value)) { + return null; + } + switch (type) { + case "Boolean": + return Boolean.parseBoolean(value); + case "String": + return value; + case "Integer": + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "'" + value + "' is not a 32-bit Integer; check your extension feature file " + + "(Client.getIntegerDetails cannot request larger values).", + e); + } + case "Float": + return Double.parseDouble(value); + case "Object": + return toValue(value); + default: + throw new IllegalArgumentException("Unknown flag type '" + type + + "'. Supported types are Boolean, String, Integer, Float and Object."); + } + } + + private static Value toValue(String json) { + try { + return Value.objectToValue(MAPPER.readValue(json, Object.class)); + } catch (IOException e) { + throw new IllegalArgumentException("Could not parse '" + json + "' as an Object flag value", e); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/AbstractSteps.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/AbstractSteps.java new file mode 100644 index 000000000..641f953b5 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/AbstractSteps.java @@ -0,0 +1,42 @@ +package dev.openfeature.contrib.tools.tck.internal.steps; + +import dev.openfeature.contrib.tools.tck.BackendControl; +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import dev.openfeature.contrib.tools.tck.TckState; + +/** + * Base for TCK step definitions; exposes scenario state, the harness, and {@link BackendControl}. + */ +public abstract class AbstractSteps { + + /** Scenario-scoped state, shared across all step classes in a scenario. */ + protected final TckState state; + + private final ProviderTckHarness harness; + + private final BackendControl backend; + + protected AbstractSteps(TckState state, ProviderTckHarness harness, BackendControl backend) { + this.harness = harness; + this.backend = backend; + this.state = state; + } + + /** + * Returns the provider author's harness. + * + * @return the discovered harness + */ + protected ProviderTckHarness harness() { + return harness; + } + + /** + * Returns the seam through which the backend is manipulated. + * + * @return the backend control for this suite + */ + protected BackendControl backend() { + return backend; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/ContextSteps.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/ContextSteps.java new file mode 100644 index 000000000..d4fb1b85a --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/ContextSteps.java @@ -0,0 +1,66 @@ +package dev.openfeature.contrib.tools.tck.internal.steps; + +import dev.openfeature.contrib.tools.tck.BackendControl; +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import dev.openfeature.contrib.tools.tck.TckState; +import dev.openfeature.sdk.MutableStructure; +import io.cucumber.java.en.Given; + +/** + * Steps that build the evaluation context passed to every evaluation. + */ +public class ContextSteps extends AbstractSteps { + + public ContextSteps(TckState state, ProviderTckHarness harness, BackendControl backend) { + super(state, harness, backend); + } + + /** + * Adds a typed entry to the evaluation context. + * + * @param key the context key + * @param type one of {@code Boolean}, {@code String}, {@code Integer} or {@code Float} + * @param value the value, as written in the feature file + */ + @Given("a context containing a key {string}, with type {string} and with value {string}") + public void contextContainingKeyWithTypeAndValue(String key, String type, String value) { + switch (type) { + case "Boolean": + state.context.add(key, Boolean.parseBoolean(value)); + break; + case "Integer": + state.context.add(key, Integer.parseInt(value)); + break; + case "Float": + state.context.add(key, Double.parseDouble(value)); + break; + case "String": + state.context.add(key, value); + break; + default: + throw new IllegalArgumentException("Unknown context value type '" + type + "'"); + } + } + + /** + * Sets the targeting key on the evaluation context. + * + * @param targetingKey the targeting key + */ + @Given("a context containing a targeting key with value {string}") + public void contextContainingTargetingKey(String targetingKey) { + state.context.setTargetingKey(targetingKey); + } + + /** + * Adds a nested structure entry to the evaluation context. + * + * @param outerKey the outer key + * @param innerKey the key inside the nested structure + * @param value the string value stored under the inner key + */ + @Given("a context containing a nested property with outer key {string} and inner key {string}, with value {string}") + public void contextContainingNestedProperty(String outerKey, String innerKey, String value) { + state.context.add(outerKey, new MutableStructure().add(innerKey, value)); + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/EventSteps.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/EventSteps.java new file mode 100644 index 000000000..ed5a51cf6 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/EventSteps.java @@ -0,0 +1,109 @@ +package dev.openfeature.contrib.tools.tck.internal.steps; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.awaitility.Awaitility.await; + +import dev.openfeature.contrib.tools.tck.BackendControl; +import dev.openfeature.contrib.tools.tck.ProviderEventRecord; +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import dev.openfeature.contrib.tools.tck.TckState; +import dev.openfeature.sdk.ProviderEvent; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Registers provider event handlers and awaits events, defaulting to + * {@link ProviderTckHarness#eventTimeout()}. + */ +public class EventSteps extends AbstractSteps { + + private static final Logger log = LoggerFactory.getLogger(EventSteps.class); + + public EventSteps(TckState state, ProviderTckHarness harness, BackendControl backend) { + super(state, harness, backend); + } + + /** + * Registers a handler for one kind of provider event. + * + * @param eventType one of {@code ready}, {@code error}, {@code stale} or {@code change} + */ + @Given("a {} event handler") + public void registerEventHandler(String eventType) { + state.client.on(mapEventType(eventType), details -> { + log.info("{} event observed", eventType); + state.events.add(new ProviderEventRecord(eventType, details)); + }); + } + + /** + * Awaits an event of the given kind, using the provider's configured timeout. + * + * @param eventType the event kind + */ + @When("a {} event was fired") + public void eventWasFired(String eventType) { + awaitEvent(eventType, harness().eventTimeout().toMillis()); + } + + /** + * Awaits an event of the given kind, using the provider's configured timeout. + * + * @param eventType the event kind + */ + @Then("the {} event handler should have been executed") + public void theEventHandlerShouldHaveBeenExecuted(String eventType) { + awaitEvent(eventType, harness().eventTimeout().toMillis()); + } + + /** + * Awaits an event of the given kind within an explicit deadline. + * + * @param eventType the event kind + * @param milliseconds the deadline + */ + @Then("the {} event handler should have been executed within {int}ms") + public void theEventHandlerShouldHaveBeenExecutedWithin(String eventType, int milliseconds) { + awaitEvent(eventType, milliseconds); + } + + private void awaitEvent(String eventType, long milliseconds) { + log.info("Awaiting {} event (timeout {}ms)", eventType, milliseconds); + await().alias("provider event " + eventType) + .atMost(milliseconds, MILLISECONDS) + .pollInterval(10, MILLISECONDS) + .until(() -> + state.events.stream().anyMatch(event -> event.type().equals(eventType))); + + // drain through the first match so stale events can't satisfy later awaits + ProviderEventRecord matched = null; + while (!state.events.isEmpty()) { + ProviderEventRecord head = state.events.poll(); + if (head != null && head.type().equals(eventType)) { + matched = head; + break; + } + } + state.lastEvent = Optional.ofNullable(matched); + } + + private static ProviderEvent mapEventType(String eventType) { + switch (eventType) { + case "ready": + return ProviderEvent.PROVIDER_READY; + case "error": + return ProviderEvent.PROVIDER_ERROR; + case "stale": + return ProviderEvent.PROVIDER_STALE; + case "change": + return ProviderEvent.PROVIDER_CONFIGURATION_CHANGED; + default: + throw new IllegalArgumentException( + "Unknown event type '" + eventType + "'. The TCK recognises ready, error, stale and change."); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/FlagSteps.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/FlagSteps.java new file mode 100644 index 000000000..d7726727d --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/FlagSteps.java @@ -0,0 +1,232 @@ +package dev.openfeature.contrib.tools.tck.internal.steps; + +import static org.assertj.core.api.Assertions.assertThat; + +import dev.openfeature.contrib.tools.tck.BackendControl; +import dev.openfeature.contrib.tools.tck.FlagUnderTest; +import dev.openfeature.contrib.tools.tck.ProviderEventRecord; +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import dev.openfeature.contrib.tools.tck.TckState; +import dev.openfeature.contrib.tools.tck.internal.TckValues; +import dev.openfeature.sdk.ErrorCode; +import dev.openfeature.sdk.Structure; +import dev.openfeature.sdk.Value; +import io.cucumber.datatable.DataTable; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Flag evaluation steps and assertions on resolution details. + */ +public class FlagSteps extends AbstractSteps { + + private static final Logger log = LoggerFactory.getLogger(FlagSteps.class); + + public FlagSteps(TckState state, ProviderTckHarness harness, BackendControl backend) { + super(state, harness, backend); + } + + /** + * Declares the flag the scenario will evaluate. + * + * @param type the declared type: {@code Boolean}, {@code String}, {@code Integer}, + * {@code Float} or {@code Object} + * @param key the flag key + * @param defaultValue the code default, as written in the feature file + */ + @Given("a {}-flag with key {string} and a default value {string}") + public void flagWithKeyAndDefaultValue(String type, String key, String defaultValue) { + state.flag = new FlagUnderTest(key, type, TckValues.convert(defaultValue, type)); + } + + /** + * Evaluates the declared flag via the matching typed API, recording any exception. + */ + @When("the flag was evaluated with details") + public void theFlagWasEvaluatedWithDetails() { + FlagUnderTest flag = state.flag; + try { + switch (flag.type()) { + case "Boolean": + state.evaluation = + state.client.getBooleanDetails(flag.key(), (Boolean) flag.defaultValue(), state.context); + break; + case "String": + state.evaluation = + state.client.getStringDetails(flag.key(), (String) flag.defaultValue(), state.context); + break; + case "Integer": + state.evaluation = + state.client.getIntegerDetails(flag.key(), (Integer) flag.defaultValue(), state.context); + break; + case "Float": + state.evaluation = + state.client.getDoubleDetails(flag.key(), (Double) flag.defaultValue(), state.context); + break; + case "Object": + state.evaluation = + state.client.getObjectDetails(flag.key(), (Value) flag.defaultValue(), state.context); + break; + default: + throw new IllegalArgumentException("Unknown flag type '" + flag.type() + "'"); + } + } catch (RuntimeException e) { + log.warn("Evaluation of '{}' threw, which violates the SDK contract", flag.key(), e); + state.thrown = e; + state.thrownBy = "evaluation of '" + flag.key() + "'"; + } + } + + /** + * Asserts the resolved value, converted according to the flag's declared type. + * + * @param value the expected value, as written in the feature file + */ + @Then("the resolved details value should be \"{}\"") + public void theResolvedDetailsValueShouldBe(String value) { + requireEvaluation(); + if (state.evaluation.getErrorCode() != null) { + log.info( + "Evaluation of '{}' carries error code {}: {}", + state.flag.key(), + state.evaluation.getErrorCode(), + state.evaluation.getErrorMessage()); + } + assertThat(state.evaluation.getValue()).isEqualTo(TckValues.convert(value, state.flag.type())); + } + + /** + * Asserts the resolution reason. + * + * @param reason the expected reason + */ + @Then("the reason should be {string}") + public void theReasonShouldBe(String reason) { + requireEvaluation(); + assertThat(state.evaluation.getReason()).isEqualTo(reason); + } + + /** + * Asserts the resolved variant. + * + * @param variant the expected variant + */ + @Then("the variant should be {string}") + public void theVariantShouldBe(String variant) { + requireEvaluation(); + assertThat(state.evaluation.getVariant()).isEqualTo(variant); + } + + /** + * Asserts the error code, where an empty string means no error. + * + * @param errorCode the expected {@link ErrorCode} name, or an empty string + */ + @Then("the error-code should be {string}") + public void theErrorCodeShouldBe(String errorCode) { + requireEvaluation(); + if (errorCode == null || errorCode.isEmpty()) { + assertThat(state.evaluation.getErrorCode()).isNull(); + } else { + assertThat(state.evaluation.getErrorCode()).isEqualTo(ErrorCode.valueOf(errorCode)); + } + } + + /** + * Captures the current resolved value so a later evaluation can be asserted to differ. + */ + @When("the resolved value is remembered") + public void theResolvedValueIsRemembered() { + requireEvaluation(); + state.rememberedValue = state.evaluation.getValue(); + } + + /** Asserts that re-evaluation produced a different value than the remembered one. */ + @Then("the resolved details value should have changed") + public void theResolvedDetailsValueShouldHaveChanged() { + requireEvaluation(); + assertThat(state.evaluation.getValue()) + .withFailMessage( + "Expected '%s' to change after the configuration change, but it is still %s.", + state.flag.key(), state.rememberedValue) + .isNotEqualTo(state.rememberedValue); + } + + /** + * Asserts that a resolved structure contains the given entries ({@code key}, {@code type}, + * {@code value} columns). + * + * @param expected a table of expected entries + */ + @Then("the resolved object value should contain") + public void theResolvedObjectValueShouldContain(DataTable expected) { + requireEvaluation(); + assertThat(state.evaluation.getValue()) + .as("resolved value of '%s' is a structure", state.flag.key()) + .isInstanceOf(Value.class); + Structure structure = ((Value) state.evaluation.getValue()).asStructure(); + assertThat(structure) + .as("resolved value of '%s' is a structure", state.flag.key()) + .isNotNull(); + + for (Map row : expected.asMaps()) { + String key = row.get("key"); + Value actual = structure.getValue(key); + assertThat(actual).as("structure entry '%s'", key).isNotNull(); + + Object expectedValue = TckValues.convert(row.get("value"), row.get("type")); + Object actualValue = actual.asObject(); + + // nested numbers compared by value; JSON has one number type + if (expectedValue instanceof Number && actualValue instanceof Number) { + assertThat(((Number) actualValue).doubleValue()) + .as("structure entry '%s'", key) + .isEqualTo(((Number) expectedValue).doubleValue()); + } else { + assertThat(actualValue).as("structure entry '%s'", key).isEqualTo(expectedValue); + } + } + } + + /** + * Asserts that no error message accompanies the evaluation (requirement 2.3.2). + */ + @Then("the error message should be empty") + public void theErrorMessageShouldBeEmpty() { + requireEvaluation(); + assertThat(state.evaluation.getErrorMessage()) + .as("error message of a successful evaluation of '%s'", state.flag.key()) + .isNullOrEmpty(); + } + + /** + * Asserts that no recorded call (evaluation, shutdown, initialize) threw. + */ + @Then("no exception should have been thrown") + public void noExceptionShouldHaveBeenThrown() { + assertThat(state.thrown) + .withFailMessage( + "%s threw %s, but the scenario expected it to return normally.", state.thrownBy, state.thrown) + .isNull(); + } + + /** Asserts the flag under test appears in the payload of the most recently matched event. */ + @Then("the flag should be part of the event payload") + public void theFlagShouldBePartOfTheEventPayload() { + ProviderEventRecord event = state.lastEvent.orElseThrow( + () -> new AssertionError("No event has been matched yet; await an event before asserting its payload")); + assertThat(event.details().getFlagsChanged()).contains(state.flag.key()); + } + + private void requireEvaluation() { + if (state.evaluation == null) { + throw new AssertionError("No evaluation has been performed. " + + "Did the scenario forget 'When the flag was evaluated with details'?" + + (state.thrown == null ? "" : " " + state.thrownBy + " threw: " + state.thrown)); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/ProviderSteps.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/ProviderSteps.java new file mode 100644 index 000000000..a4101c5cf --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/ProviderSteps.java @@ -0,0 +1,232 @@ +package dev.openfeature.contrib.tools.tck.internal.steps; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import dev.openfeature.contrib.tools.tck.BackendControl; +import dev.openfeature.contrib.tools.tck.Capability; +import dev.openfeature.contrib.tools.tck.ProviderTck; +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import dev.openfeature.contrib.tools.tck.TckState; +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.Metadata; +import dev.openfeature.sdk.NoOpProvider; +import dev.openfeature.sdk.OpenFeatureAPI; +import dev.openfeature.sdk.ProviderState; +import io.cucumber.java.After; +import io.cucumber.java.AfterAll; +import io.cucumber.java.Before; +import io.cucumber.java.BeforeAll; +import io.cucumber.java.Scenario; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import java.time.Duration; +import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Lifecycle, capability gating, provider registration, and backend-control steps. + */ +public class ProviderSteps extends AbstractSteps { + + private static final Logger log = LoggerFactory.getLogger(ProviderSteps.class); + + public ProviderSteps(TckState state, ProviderTckHarness harness, BackendControl backend) { + super(state, harness, backend); + } + + /** Opens the suite and rejects an undeclarable capability set. */ + @BeforeAll + public static void beforeAll() { + TckLifecycle.beginSuite(); + } + + /** Stops every backend the suite started. */ + @AfterAll + public static void afterAll() { + TckLifecycle.endSuite(); + } + + /** + * Skips scenarios needing an undeclared {@link Capability}; see + * {@link Capability#requireDeclared(java.net.URI, java.util.Collection, java.util.Set)}. Tags + * under {@link ProviderTck#EXTENSIONS} are exempt from the vocabulary check. + * + * @param scenario the scenario about to run + */ + @Before(order = 0) + public void gateOnCapabilities(Scenario scenario) { + Capability.requireDeclared( + scenario.getUri(), scenario.getSourceTagNames(), harness().capabilities()); + } + + /** + * Restores the backend to the state every scenario starts from. + */ + @Before(order = 10) + public void prepareBackend() { + backend().prepareScenario(); + } + + /** + * Replaces the provider with a {@link NoOpProvider} via the SDK so its emitter is shut down. + */ + @After + public void tearDown() { + if (state.domain != null) { + OpenFeatureAPI.getInstance().setProvider(state.domain, new NoOpProvider()); + } + } + + /** + * Creates the provider under test and registers it under a scenario-scoped domain; only + * {@code stable} waits for readiness. + * + * @param flavour either {@code stable} or {@code unavailable} + */ + @Given("a {} provider") + public void createProvider(String flavour) { + ProviderTckHarness harness = harness(); + FeatureProvider provider; + boolean waitForReady; + + switch (flavour) { + case "stable": + provider = harness.createProvider(); + waitForReady = true; + break; + case "unavailable": + provider = harness.createUnavailableProvider(); + waitForReady = false; + break; + default: + throw new IllegalArgumentException( + "Unknown provider flavour '" + flavour + "'. The TCK recognises 'stable' and 'unavailable'."); + } + + String domain = "tck-" + UUID.randomUUID(); + OpenFeatureAPI api = OpenFeatureAPI.getInstance(); + if (waitForReady) { + api.setProviderAndWait(domain, provider); + } else { + api.setProvider(domain, provider); + } + + state.provider = provider; + state.domain = domain; + state.client = api.getClient(domain); + log.info( + "Registered {} provider {} under domain {}", + flavour, + provider.getMetadata().getName(), + domain); + } + + /** + * Asserts that the provider metadata name is not empty (requirement 2.1.1). + */ + @Then("the provider metadata name should not be empty") + public void theProviderMetadataNameShouldNotBeEmpty() { + Metadata metadata = requireProvider().getMetadata(); + assertThat(metadata).as("provider metadata").isNotNull(); + assertThat(metadata.getName()).as("provider metadata name").isNotBlank(); + } + + /** + * Calls the provider's {@code shutdown()} directly, timing it and recording any exception. + */ + @When("the provider is shut down") + public void theProviderIsShutDown() { + FeatureProvider provider = requireProvider(); + long started = System.nanoTime(); + try { + provider.shutdown(); + } catch (RuntimeException e) { + log.warn("shutdown() of provider {} threw", provider.getMetadata().getName(), e); + state.thrown = e; + state.thrownBy = "shutdown()"; + } finally { + state.shutdownDuration = Duration.ofNanos(System.nanoTime() - started); + } + } + + /** + * Calls the provider's {@code initialize()} directly, recording any exception. + */ + @When("the provider is initialized again") + public void theProviderIsInitializedAgain() { + FeatureProvider provider = requireProvider(); + try { + provider.initialize(state.context); + } catch (Exception e) { + log.warn( + "initialize() of provider {} threw after shutdown", + provider.getMetadata().getName(), + e); + state.thrown = e; + state.thrownBy = "initialize()"; + } + } + + /** + * Asserts that the last shutdown returned within a bound. + * + * @param milliseconds the bound + */ + @Then("the shutdown should have completed within {int}ms") + public void theShutdownShouldHaveCompletedWithin(int milliseconds) { + assertThat(state.shutdownDuration) + .as("a shutdown was recorded; did the scenario forget 'When the provider is shut down'?") + .isNotNull(); + assertThat(state.shutdownDuration.toMillis()) + .withFailMessage( + "shutdown() took %dms, over the %dms bound; it must not block on an unresponsive backend.", + state.shutdownDuration.toMillis(), milliseconds) + .isLessThanOrEqualTo(milliseconds); + } + + /** Makes the backend unreachable for the rest of the scenario. */ + @When("the connection is lost") + public void theConnectionIsLost() { + backend().disconnect(); + } + + /** + * Brings the backend back after {@code the connection is lost}. + */ + @When("the connection is restored") + public void theConnectionIsRestored() { + backend().reconnect(); + } + + /** Mutates flag configuration so a conforming provider observes a configuration change. */ + @When("the flag was modified") + public void theFlagWasModified() { + backend().changeFlag(); + } + + /** + * Awaits the expected provider state within {@link ProviderTckHarness#readyTimeout()}. + * + * @param expected the expected {@link ProviderState}, case-insensitive + */ + @Then("the client should be in {} state") + public void theClientShouldBeInState(String expected) { + ProviderState target = ProviderState.valueOf(expected.toUpperCase()); + await().alias("provider state " + target) + .atMost(harness().readyTimeout().toMillis(), MILLISECONDS) + .pollInterval(10, MILLISECONDS) + .until(() -> state.client.getProviderState() == target); + } + + private FeatureProvider requireProvider() { + if (state.provider == null) { + throw new AssertionError("No provider has been created. " + + "Did the scenario forget 'Given a stable provider' or 'Given a unavailable provider'?"); + } + return state.provider; + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckLifecycle.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckLifecycle.java new file mode 100644 index 000000000..18b3b74ae --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckLifecycle.java @@ -0,0 +1,163 @@ +package dev.openfeature.contrib.tools.tck.internal.steps; + +import dev.openfeature.contrib.tools.tck.BackendControl; +import dev.openfeature.contrib.tools.tck.Capability; +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import dev.openfeature.contrib.tools.tck.internal.TckSuiteListener; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.ServiceLoader; +import java.util.Set; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * One harness and the backend it brings up, owned by a {@link TckObjectFactory}. + * + *

Statics exist only because Cucumber's suite hooks are static. + */ +final class TckLifecycle { + + /** System property selecting a harness by simple class name when several are registered. */ + static final String HARNESS_SELECTOR_PROPERTY = "openfeature.tck.harness"; + + private static final Logger log = LoggerFactory.getLogger(TckLifecycle.class); + + private static final Set started = Collections.synchronizedSet(new LinkedHashSet<>()); + + private static volatile boolean suiteRunning; + + private final ProviderTckHarness harness; + private final BackendControl backendControl; + + private TckLifecycle(ProviderTckHarness harness, BackendControl backendControl) { + this.harness = harness; + this.backendControl = backendControl; + } + + /** Marks a suite as running and rejects an undeclarable capability set before anything starts. */ + static void beginSuite() { + ProviderTckHarness harness = discoverHarness(); + log.info("Provider TCK harness: {}", harness.getClass().getName()); + Capability.requireDeclarable(harness.capabilities()); + suiteRunning = true; + } + + /** Stops every lifecycle this suite started. */ + static void endSuite() { + suiteRunning = false; + List toStop; + synchronized (started) { + toStop = new ArrayList<>(started); + started.clear(); + } + RuntimeException failure = null; + for (TckLifecycle lifecycle : toStop) { + try { + lifecycle.harness.stopSuite(); + } catch (RuntimeException e) { + // keep stopping the rest + log.warn( + "Suite teardown failed for {}", + lifecycle.harness.getClass().getName(), + e); + failure = failure == null ? e : failure; + } + } + if (failure != null) { + throw failure; + } + } + + /** + * Starts a lifecycle for the caller to own, or empty when no suite is running. + * + * @return a started lifecycle, or empty when there is no suite to start one for + */ + static Optional startForExecutingSuite() { + if (!suiteRunning) { + return Optional.empty(); + } + return Optional.of(start(discoverHarness())); + } + + /** Starts the given harness and registers the result for {@link #endSuite()}. */ + static TckLifecycle start(ProviderTckHarness harness) { + try { + harness.startSuite(); + BackendControl control = harness.backendControl(); + if (control == null) { + throw new IllegalStateException(harness.getClass().getName() + + ".backendControl() returned null; supply overHttp(..) or inProcess()."); + } + log.info("Backend control: {}", control.description()); + TckLifecycle lifecycle = new TckLifecycle(harness, control); + started.add(lifecycle); + return lifecycle; + } catch (RuntimeException e) { + // release anything a partial start allocated, e.g. containers + harness.stopSuite(); + throw e; + } + } + + /** How many lifecycles are currently started, for tests that assert on isolation. */ + static int startedCount() { + return started.size(); + } + + /** The harness this lifecycle owns. */ + ProviderTckHarness harness() { + return harness; + } + + /** The seam through which the TCK manipulates this lifecycle's backend. */ + BackendControl backendControl() { + return backendControl; + } + + /** Finds a fresh harness for the executing suite, falling back to {@link ServiceLoader}. */ + private static ProviderTckHarness discoverHarness() { + Optional> suite = TckSuiteListener.currentSuite(); + if (suite.isPresent()) { + return instantiate(suite.get()); + } + + List found = new ArrayList<>(); + ServiceLoader.load(ProviderTckHarness.class).forEach(found::add); + + if (found.isEmpty()) { + throw new IllegalStateException( + "No ProviderTckHarness found; extend ContainerizedProviderTckTest or ProviderTckTest."); + } + if (found.size() == 1) { + return found.get(0); + } + + String selector = System.getProperty(HARNESS_SELECTOR_PROPERTY); + if (selector == null) { + throw new IllegalStateException("Several ProviderTckHarness implementations are registered (" + + found.stream().map(h -> h.getClass().getSimpleName()).collect(Collectors.joining(", ")) + + "); select one with -D" + HARNESS_SELECTOR_PROPERTY + "=."); + } + return found.stream() + .filter(h -> h.getClass().getSimpleName().equals(selector)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No registered ProviderTckHarness named '" + selector + + "'. Registered: " + + found.stream().map(h -> h.getClass().getSimpleName()).collect(Collectors.joining(", ")))); + } + + private static ProviderTckHarness instantiate(Class suite) { + try { + return suite.getDeclaredConstructor().newInstance(); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException( + suite.getName() + " could not be instantiated; it needs a public no-arg constructor.", e); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckObjectFactory.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckObjectFactory.java new file mode 100644 index 000000000..046e31037 --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckObjectFactory.java @@ -0,0 +1,90 @@ +package dev.openfeature.contrib.tools.tck.internal.steps; + +import dev.openfeature.contrib.tools.tck.BackendControl; +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import io.cucumber.core.backend.ObjectFactory; +import java.lang.reflect.Constructor; +import java.lang.reflect.Modifier; +import java.util.LinkedHashSet; +import java.util.Set; +import org.picocontainer.MutablePicoContainer; +import org.picocontainer.PicoBuilder; + +/** + * Instantiates step classes, with the harness and backend control available for injection. + * + *

The container is per scenario; the {@link TckLifecycle} is per factory, started on first use. + */ +public final class TckObjectFactory implements ObjectFactory { + + private final Set> glue = new LinkedHashSet<>(); + + private MutablePicoContainer pico; + + private TckLifecycle lifecycle; + + @Override + public boolean addClass(Class glueClass) { + if (isInstantiable(glueClass) && glue.add(glueClass)) { + addConstructorDependencies(glueClass); + } + return true; + } + + /** + * Registers the constructor parameter types a glue class needs, recursively. + * + * @param glueClass the class whose constructor parameters to register + */ + private void addConstructorDependencies(Class glueClass) { + for (Constructor constructor : glueClass.getConstructors()) { + for (Class parameter : constructor.getParameterTypes()) { + addClass(parameter); + } + } + } + + /** + * Cucumber's own rule for a component: public, concrete, not an inner class. + * + * @param candidate the type to test + * @return whether the container can instantiate it + */ + private static boolean isInstantiable(Class candidate) { + boolean nonStaticInner = !Modifier.isStatic(candidate.getModifiers()) && candidate.getEnclosingClass() != null; + return Modifier.isPublic(candidate.getModifiers()) + && !Modifier.isAbstract(candidate.getModifiers()) + && !nonStaticInner; + } + + @Override + public void start() { + pico = new PicoBuilder().withCaching().build(); + if (lifecycle == null) { + // empty when a launcher selects only the extension glue and never reaches @BeforeAll + lifecycle = TckLifecycle.startForExecutingSuite().orElse(null); + } + if (lifecycle != null) { + pico.addComponent(ProviderTckHarness.class, lifecycle.harness()); + pico.addComponent(BackendControl.class, lifecycle.backendControl()); + } + for (Class glueClass : glue) { + pico.addComponent(glueClass); + } + pico.start(); + } + + @Override + public void stop() { + if (pico != null) { + pico.stop(); + pico.dispose(); + pico = null; + } + } + + @Override + public T getInstance(Class glueClass) { + return pico.getComponent(glueClass); + } +} diff --git a/tools/tck/src/main/resources/META-INF/services/io.cucumber.core.backend.ObjectFactory b/tools/tck/src/main/resources/META-INF/services/io.cucumber.core.backend.ObjectFactory new file mode 100644 index 000000000..6ccd9791b --- /dev/null +++ b/tools/tck/src/main/resources/META-INF/services/io.cucumber.core.backend.ObjectFactory @@ -0,0 +1 @@ +dev.openfeature.contrib.tools.tck.internal.steps.TckObjectFactory diff --git a/tools/tck/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener b/tools/tck/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener new file mode 100644 index 000000000..3a537e817 --- /dev/null +++ b/tools/tck/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener @@ -0,0 +1 @@ +dev.openfeature.contrib.tools.tck.internal.TckSuiteListener diff --git a/tools/tck/src/main/resources/extensions/README.md b/tools/tck/src/main/resources/extensions/README.md new file mode 100644 index 000000000..0f8a53194 --- /dev/null +++ b/tools/tck/src/main/resources/extensions/README.md @@ -0,0 +1,33 @@ +# Provider TCK extension point + +Feature files on the classpath under `extensions/` run in the TCK suite alongside the canonical +scenarios. This README keeps the directory present on the classpath; Cucumber ignores non-`.feature` +files. + +## Adding scenarios + +``` +src/test/resources/extensions/fractional.feature +src/test/java/openfeature/tck/extensions/FractionalSteps.java // package openfeature.tck.extensions +``` + +Steps in the `openfeature.tck.extensions` glue package are picked up automatically and share the +suite's backend lifecycle. + +Step classes declare what they need as constructor arguments; any of `TckState`, +`ProviderTckHarness` and `BackendControl`: + +```java +public FractionalSteps(TckState state, ProviderTckHarness harness, BackendControl backend) { ... } +``` + +The built-in step definitions live in `dev.openfeature.contrib.tools.tck.internal.steps`; they are +internal, not API. + +Both the step class and each step method must be `public`; otherwise the scenario reports +*undefined*, which a non-strict run reads as a pass. + +## Why `extensions/` and not `gherkin/` + +A file in `gherkin/` with a canonical name would silently replace the canonical file. A separate +directory avoids that and keeps extension scenarios distinguishable from conformance results. diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalAssetDigestTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalAssetDigestTest.java new file mode 100644 index 000000000..84035ceba --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalAssetDigestTest.java @@ -0,0 +1,134 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.CodeSource; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Asserts the packaged canonical assets match the pinned spec revision, by digest. To move the pin, + * update the {@code spec} gitlink, {@link #PINNED_REVISION} and {@link #PINNED_DIGEST} in one commit. + */ +class CanonicalAssetDigestTest { + + /** The open-feature/spec commit the packaged assets come from; must match the submodule HEAD. */ + static final String PINNED_REVISION = "10b1a610ed0afe6bf0e38b441d7362f5a76cb47d"; + + /** SHA-256 of the three asset trees at {@link #PINNED_REVISION}, as {@link #digest} computes it. */ + static final String PINNED_DIGEST = "7a753b5f5f60248336d93f565bc12ad844af9f04de40867916b98f283363354b"; + + /** The generated resource directories, in the order they are digested. */ + private static final List ASSET_DIRECTORIES = Arrays.asList("flags", "gherkin", "openapi"); + + @Test + @DisplayName("the packaged canonical assets are the pinned revision's, byte for byte") + void thePackagedAssetsAreThePinnedRevisions() { + String actual = digest(); + assertThat(actual) + .as( + "Packaged canonical assets do not match open-feature/spec %s.%n" + + " expected %s%n" + + " actual %s%n" + + "Run `git submodule update --init tools/tck/spec` (HEAD must be %s) and rebuild " + + "with `clean`. If moving the pin, update PINNED_REVISION and PINNED_DIGEST.", + PINNED_REVISION, PINNED_DIGEST, actual, PINNED_REVISION) + .isEqualTo(PINNED_DIGEST); + } + + /** SHA-256 over each asset's relative path and CR-stripped content, sorted by path, zero-separated. */ + private static String digest() { + MessageDigest sha256; + try { + sha256 = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("This JVM does not provide SHA-256", e); + } + Path root = codeSourceRoot(); + for (Path file : assetFiles(root)) { + sha256.update(relativePath(root, file).getBytes(StandardCharsets.UTF_8)); + sha256.update((byte) 0); + sha256.update(withoutCarriageReturns(read(file))); + sha256.update((byte) 0); + } + StringBuilder hex = new StringBuilder(); + for (byte b : sha256.digest()) { + hex.append(String.format("%02x", b)); + } + return hex.toString(); + } + + private static List assetFiles(Path root) { + List files = new ArrayList<>(); + for (String directory : ASSET_DIRECTORIES) { + Path assets = root.resolve(directory); + if (!Files.isDirectory(assets)) { + throw new IllegalStateException("No " + directory + "/ directory in the tck code source " + root + + ". The canonical assets were not copied from the spec submodule; run the build " + + "through Maven rather than compiling the sources alone."); + } + try (Stream walk = Files.walk(assets)) { + files.addAll(walk.filter(Files::isRegularFile).collect(Collectors.toList())); + } catch (IOException e) { + throw new UncheckedIOException("Could not list the canonical assets under " + assets, e); + } + } + files.sort(Comparator.comparing(file -> relativePath(root, file))); + return files; + } + + private static String relativePath(Path root, Path file) { + return root.relativize(file).toString().replace('\\', '/'); + } + + private static byte[] read(Path file) { + try { + return Files.readAllBytes(file); + } catch (IOException e) { + throw new UncheckedIOException("Could not read the canonical asset " + file, e); + } + } + + private static byte[] withoutCarriageReturns(byte[] content) { + byte[] stripped = new byte[content.length]; + int length = 0; + for (byte b : content) { + if (b != '\r') { + stripped[length++] = b; + } + } + byte[] exact = new byte[length]; + System.arraycopy(stripped, 0, exact, 0, length); + return exact; + } + + /** The artifact's code source root; avoids the classloader so shadowing copies are ignored. */ + private static Path codeSourceRoot() { + CodeSource codeSource = ProviderTck.class.getProtectionDomain().getCodeSource(); + if (codeSource == null || codeSource.getLocation() == null) { + throw new IllegalStateException( + "The tck code source is not visible to this JVM, so the packaged canonical assets " + + "cannot be read."); + } + try { + return Paths.get(codeSource.getLocation().toURI()); + } catch (URISyntaxException | IllegalArgumentException e) { + throw new IllegalStateException("The tck code source is not a file: " + codeSource.getLocation(), e); + } + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalFlagsTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalFlagsTest.java new file mode 100644 index 000000000..8feb24983 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalFlagsTest.java @@ -0,0 +1,176 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.openfeature.sdk.ImmutableContext; +import dev.openfeature.sdk.Value; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Pins the in-process flag set to the packaged definition, reading the JSON independently of the + * decoder and resolving each flag through the accessor its JSON type calls for. + */ +class CanonicalFlagsTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + /** The {@code flags} object of the packaged definition, read without going through the decoder. */ + private static Map packaged; + + @BeforeAll + static void readThePackagedDefinition() throws IOException { + try (InputStream in = CanonicalFlagsTest.class.getClassLoader().getResourceAsStream(CanonicalFlags.RESOURCE)) { + assertThat(in) + .as( + "%s must be packaged on the classpath by the copy-provider-tck-flags execution", + CanonicalFlags.RESOURCE) + .isNotNull(); + + JsonNode flags = MAPPER.readTree(in).path("flags"); + assertThat(flags.isObject()).isTrue(); + + packaged = new LinkedHashMap<>(); + for (Iterator> it = flags.fields(); it.hasNext(); ) { + Map.Entry entry = it.next(); + if (!"$comment".equals(entry.getKey())) { + packaged.put(entry.getKey(), entry.getValue()); + } + } + } + assertThat(packaged).as("the packaged definition defines flags").isNotEmpty(); + } + + @Test + @DisplayName("the decoded flag set defines exactly the keys the packaged definition defines") + void theDecodedSetHasExactlyThePackagedKeys() { + assertThat(CanonicalFlags.flagSet().keySet()) + .as("decoded keys match packaged keys") + .containsExactlyInAnyOrderElementsOf(packaged.keySet()) + .as("$comment is documentation, not a flag") + .doesNotContain("$comment"); + } + + @Test + @DisplayName("every flag resolves as its packaged state and default variant say it should") + void everyFlagResolvesToItsPackagedDefaultVariant() throws Exception { + InMemoryProvider provider = BackendControl.inProcess().createProvider(); + provider.initialize(new ImmutableContext()); + ImmutableContext context = new ImmutableContext(); + + List checked = new ArrayList<>(); + List disabled = new ArrayList<>(); + for (Map.Entry entry : packaged.entrySet()) { + String key = entry.getKey(); + String defaultVariant = entry.getValue().path("defaultVariant").asText(null); + assertThat(defaultVariant) + .as("%s names no defaultVariant, so the definition itself is broken", key) + .isNotNull(); + + JsonNode expected = entry.getValue().path("variants").path(defaultVariant); + assertThat(expected.isMissingNode()) + .as("%s resolves to variant '%s', which the definition does not define", key, defaultVariant) + .isFalse(); + + String state = entry.getValue().path("state").asText(null); + assertThat(state) + .as("%s names no state, so the definition itself is broken", key) + .isIn("ENABLED", "DISABLED"); + + boolean enabled = "ENABLED".equals(state); + assertResolves(provider, context, key, defaultVariant, expected, enabled); + checked.add(key); + if (!enabled) { + disabled.add(key); + } + } + + assertThat(checked) + .as("the loop must actually have run over the packaged flags") + .hasSameSizeAs(packaged.keySet()); + + assertThat(disabled).as("at least one disabled flag was checked").isNotEmpty(); + } + + /** Resolves one flag via its JSON type's accessor; the fallback never equals the expected value. */ + private static void assertResolves( + InMemoryProvider provider, + ImmutableContext context, + String key, + String variant, + JsonNode expected, + boolean enabled) { + + String where = key + " variant '" + variant + "'" + (enabled ? "" : ", disabled so the default stands in"); + switch (expected.getNodeType()) { + case BOOLEAN: + boolean bool = expected.booleanValue(); + assertThat(provider.getBooleanEvaluation(key, !bool, context).getValue()) + .as(where) + .isEqualTo(enabled ? bool : !bool); + break; + case STRING: + String string = expected.textValue(); + String stringFallback = string + "-fallback"; + assertThat(provider.getStringEvaluation(key, stringFallback, context) + .getValue()) + .as(where) + .isEqualTo(enabled ? string : stringFallback); + break; + case NUMBER: + assertNumberResolves(provider, context, key, where, expected, enabled); + break; + case OBJECT: + case ARRAY: + Value structure = Value.objectToValue(MAPPER.convertValue(expected, Object.class)); + Value objectFallback = new Value("fallback"); + assertThat(provider.getObjectEvaluation(key, objectFallback, context) + .getValue()) + .as(where) + .isEqualTo(enabled ? structure : objectFallback); + break; + default: + fail("%s is a %s, which is not a flag value", where, expected.getNodeType()); + break; + } + } + + /** Resolves a numeric flag via the int, long or double accessor its literal calls for. */ + private static void assertNumberResolves( + InMemoryProvider provider, + ImmutableContext context, + String key, + String where, + JsonNode expected, + boolean enabled) { + + if (!expected.isIntegralNumber()) { + double value = expected.doubleValue(); + assertThat(provider.getDoubleEvaluation(key, value + 1, context).getValue()) + .as(where) + .isEqualTo(enabled ? value : value + 1); + } else if (expected.canConvertToInt()) { + int value = expected.intValue(); + assertThat(provider.getIntegerEvaluation(key, value + 1, context).getValue()) + .as(where) + .isEqualTo(enabled ? value : value + 1); + } else { + long value = expected.longValue(); + assertThat(provider.getLongEvaluation(key, value + 1, context).getValue()) + .as(where) + .isEqualTo(enabled ? value : value + 1); + } + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalTagCoverageTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalTagCoverageTest.java new file mode 100644 index 000000000..04aa13ce6 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/CanonicalTagCoverageTest.java @@ -0,0 +1,181 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.cucumber.gherkin.GherkinParser; +import io.cucumber.messages.types.Envelope; +import io.cucumber.messages.types.Examples; +import io.cucumber.messages.types.Feature; +import io.cucumber.messages.types.FeatureChild; +import io.cucumber.messages.types.GherkinDocument; +import io.cucumber.messages.types.Rule; +import io.cucumber.messages.types.RuleChild; +import io.cucumber.messages.types.Scenario; +import io.cucumber.messages.types.Tag; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.CodeSource; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Every non-reserved capability is carried by a canonical scenario, no reserved one is, and every + * carried tag is a known capability. Tags are parsed, not scanned as text. + */ +class CanonicalTagCoverageTest { + + /** Tags carried by the canonical feature files this artifact ships, as Gherkin parses them. */ + private static final Set CARRIED = readCarriedTags(); + + @Test + @DisplayName("every capability that is not reserved is carried by at least one canonical scenario") + void everyUnreservedCapabilityGatesSomething() { + for (Capability capability : Capability.values()) { + if (capability.reserved()) { + continue; + } + assertThat(CARRIED) + .as( + "%s (%s) is not reserved but no canonical scenario carries its tag; packaged " + + "gherkin/ may be stale, or the capability should be reserved", + capability.name(), capability.tag()) + .contains(capability.tag()); + } + } + + @Test + @DisplayName("no reserved capability's tag is carried by a canonical scenario") + void noReservedCapabilityGatesAnything() { + for (Capability capability : Capability.values()) { + if (!capability.reserved()) { + continue; + } + assertThat(CARRIED) + .as( + "%s (%s) is reserved but a canonical scenario carries its tag; drop the " + "reservation", + capability.name(), capability.tag()) + .doesNotContain(capability.tag()); + } + } + + @Test + @DisplayName("every tag a canonical scenario carries resolves to a capability") + void everyCarriedTagIsInTheVocabulary() { + // reverse of the first test; catches a re-pin that adds a tag without a Capability + for (String tag : CARRIED) { + assertThat(Capability.fromTag(tag)) + .as( + "a canonical scenario carries %s, which Capability.fromTag does not resolve; " + + "add it to the Capability enum", + tag) + .isPresent(); + } + } + + @Test + @DisplayName("a tag named only in a Gherkin comment is prose, not a tag") + void aTagInACommentIsNotCarried() { + // events.feature names @caching in a comment; a text scan would misreport it + assertThat(rawCanonicalText()) + .as("events.feature still names @caching in prose") + .contains("@caching"); + assertThat(CARRIED).as("but nothing carries it as a tag").doesNotContain(Capability.CACHING.tag()); + } + + private static Set readCarriedTags() { + Set tags = new LinkedHashSet<>(); + forEachCanonicalFeature((name, content) -> { + GherkinParser parser = GherkinParser.builder() + .includeSource(false) + .includePickles(false) + .includeGherkinDocument(true) + .build(); + try (Stream envelopes = parser.parse(name, content)) { + envelopes.forEach(envelope -> envelope.getGherkinDocument() + .flatMap(GherkinDocument::getFeature) + .ifPresent(feature -> collectFeature(feature, tags))); + } + }); + if (tags.isEmpty()) { + throw new IllegalStateException("No tags found in the packaged canonical feature files. The " + + "artifact is not intact, or gherkin/ was not copied from the spec submodule."); + } + return tags; + } + + private static String rawCanonicalText() { + StringBuilder all = new StringBuilder(); + forEachCanonicalFeature((name, content) -> all.append(new String(content, StandardCharsets.UTF_8))); + return all.toString(); + } + + private static void collectFeature(Feature feature, Set tags) { + addAll(feature.getTags(), tags); + for (FeatureChild child : feature.getChildren()) { + child.getScenario().ifPresent(scenario -> collectScenario(scenario, tags)); + child.getRule().ifPresent(rule -> collectRule(rule, tags)); + } + } + + private static void collectRule(Rule rule, Set tags) { + addAll(rule.getTags(), tags); + for (RuleChild child : rule.getChildren()) { + child.getScenario().ifPresent(scenario -> collectScenario(scenario, tags)); + } + } + + private static void collectScenario(Scenario scenario, Set tags) { + addAll(scenario.getTags(), tags); + for (Examples examples : scenario.getExamples()) { + addAll(examples.getTags(), tags); + } + } + + private static void addAll(List from, Set tags) { + for (Tag tag : from) { + tags.add(tag.getName()); + } + } + + /** Reads {@code gherkin/} from the artifact's code source, not the classloader, to avoid shadowing. */ + private static void forEachCanonicalFeature(FeatureConsumer consumer) { + CodeSource codeSource = ProviderTck.class.getProtectionDomain().getCodeSource(); + if (codeSource == null || codeSource.getLocation() == null) { + throw new IllegalStateException("The tck code source is not visible to this JVM, so the packaged " + + "canonical feature files cannot be read."); + } + Path root; + try { + root = Paths.get(codeSource.getLocation().toURI()); + } catch (URISyntaxException | IllegalArgumentException e) { + throw new IllegalStateException("The tck code source is not a file: " + codeSource.getLocation(), e); + } + + Path features = root.resolve(ProviderTck.FEATURES); + if (!Files.isDirectory(features)) { + throw new IllegalStateException("No " + ProviderTck.FEATURES + "/ directory in the tck code source " + root + + ". The canonical assets were not copied from the spec submodule."); + } + try (DirectoryStream entries = Files.newDirectoryStream(features, "*.feature")) { + for (Path entry : entries) { + consumer.accept(ProviderTck.FEATURES + "/" + entry.getFileName(), Files.readAllBytes(entry)); + } + } catch (IOException e) { + throw new IllegalStateException("Could not read the canonical feature files from " + features, e); + } + } + + @FunctionalInterface + private interface FeatureConsumer { + void accept(String name, byte[] content); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableBackendControl.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableBackendControl.java new file mode 100644 index 000000000..9c03df4dd --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableBackendControl.java @@ -0,0 +1,112 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.exceptions.GeneralError; +import dev.openfeature.sdk.providers.memory.Flag; +import java.util.HashMap; +import java.util.Map; + +/** + * In-process control for {@link ControllableProvider}, with a backend that can refuse to answer. + * Unlocks the {@code @lifecycle} and {@code @unavailable} scenarios without Docker. + */ +final class ControllableBackendControl implements BackendControl { + + /** The flag {@link #changeFlag()} mutates, per the control API's {@code POST /change}. */ + private static final String CHANGING_FLAG = "changing-flag"; + + /** The canonical flag set, never mutated after construction. */ + private final Map> baseline = CanonicalFlags.flagSet(); + + /** {@code changing-flag} as the definition ships it, the source of its variants. */ + private final Flag changingBaseline = requireChangingFlag(); + + /** The provider serving the current scenario, or {@code null} between scenarios. */ + private ControllableProvider current; + + /** Which variant {@code changing-flag} currently resolves to. */ + private String changingVariant = changingBaseline.getDefaultVariant(); + + /** Creates the provider for the next scenario; the store is read at {@code initialize()} time. */ + ControllableProvider createProvider() { + changingVariant = changingBaseline.getDefaultVariant(); + current = new ControllableProvider(this::snapshot); + return current; + } + + /** Creates a provider whose {@code initialize()} throws; not registered as {@link #current}. */ + ControllableProvider createUnavailableProvider() { + return new ControllableProvider(ControllableBackendControl::unreachable); + } + + @Override + public String description() { + return "in-process control of " + ControllableProvider.class.getSimpleName(); + } + + @Override + public BackendType backendType() { + return BackendType.IN_PROCESS; + } + + /** Drops the previous scenario's provider; the baseline is never mutated. */ + @Override + public void prepareScenario() { + current = null; + } + + /** Flips {@code changing-flag} between its two variants. */ + @Override + public void changeFlag() { + changingVariant = otherVariant(changingVariant); + requireProvider().changeFlag(CHANGING_FLAG, changingFlag(changingVariant)); + } + + /** A fresh copy of the baseline, which is what initialisation acquires. */ + private Map> snapshot() { + return new HashMap<>(baseline); + } + + /** The unreachable backend: every attempt to read it fails the way a dead socket would. */ + private static Map> unreachable() { + throw new GeneralError("the TCK's in-process backend is unreachable for this provider"); + } + + /** A variant of {@code changing-flag} other than the given one, read from the definition. */ + private String otherVariant(String resolved) { + for (String variant : changingBaseline.getVariants().keySet()) { + if (!variant.equals(resolved)) { + return variant; + } + } + throw new IllegalStateException("The canonical definition of '" + CHANGING_FLAG + "' has only the variant '" + + resolved + "'. changeFlag() has to switch to a different one, so the flag needs at least two."); + } + + /** Rebuilds {@code changing-flag} with a different variant as the one it resolves to. */ + private Flag changingFlag(String defaultVariant) { + return Flag.builder() + .variants(changingBaseline.getVariants()) + .defaultVariant(defaultVariant) + .disabled(changingBaseline.isDisabled()) + .build(); + } + + /** The canonical definition of {@code changing-flag}, which the suite cannot do without. */ + private Flag requireChangingFlag() { + Flag flag = baseline.get(CHANGING_FLAG); + if (flag == null) { + throw new IllegalStateException("The canonical flag definition does not define '" + CHANGING_FLAG + + "', which is the flag POST /change mutates and the @configuration-change scenarios evaluate."); + } + return flag; + } + + private ControllableProvider requireProvider() { + if (current == null) { + throw new IllegalStateException("No controllable provider exists for this scenario. In-process backend " + + "control manipulates the provider itself, so the scenario must create one, with " + + "'Given a stable provider', before any step that changes flag state."); + } + return current; + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableProvider.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableProvider.java new file mode 100644 index 000000000..014b203b7 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableProvider.java @@ -0,0 +1,103 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.EvaluationContext; +import dev.openfeature.sdk.EventProvider; +import dev.openfeature.sdk.Metadata; +import dev.openfeature.sdk.ProviderEvaluation; +import dev.openfeature.sdk.ProviderEventDetails; +import dev.openfeature.sdk.ProviderState; +import dev.openfeature.sdk.Value; +import dev.openfeature.sdk.exceptions.GeneralError; +import dev.openfeature.sdk.providers.memory.Flag; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.util.Collections; +import java.util.Map; +import java.util.function.Supplier; + +/** + * An in-JVM provider with a real initialisation that can fail, for Docker-free lifecycle coverage. + * Wraps rather than extends {@link InMemoryProvider} to avoid spurious configuration-change events. + */ +final class ControllableProvider extends EventProvider { + + /** What this provider calls itself. Asserted only as "not empty", by the metadata feature. */ + private static final String NAME = "TckControllableProvider"; + + /** Supplies the flag set, or throws when the backend is unreachable. */ + private final Supplier>> backend; + + /** The flag store serving the current session, or {@code null} before init and after shutdown. */ + private volatile InMemoryProvider delegate; + + ControllableProvider(Supplier>> backend) { + this.backend = backend; + } + + @Override + public Metadata getMetadata() { + return () -> NAME; + } + + /** Acquires the flag set from the backend; throws if it is unreachable. */ + @Override + public void initialize(EvaluationContext context) throws Exception { + InMemoryProvider acquired = new InMemoryProvider(backend.get()); + acquired.initialize(context); + delegate = acquired; + } + + /** Releases the flag set; skips {@code super.shutdown()} so the provider can be reinitialised. */ + @Override + public void shutdown() { + delegate = null; + } + + @Override + public ProviderState getState() { + InMemoryProvider current = delegate; + return current == null ? ProviderState.NOT_READY : current.getState(); + } + + /** Changes a flag and emits the event from this provider, since the delegate is unregistered. */ + void changeFlag(String key, Flag flag) { + requireDelegate().updateFlag(key, flag); + emitProviderConfigurationChanged(ProviderEventDetails.builder() + .flagsChanged(Collections.singletonList(key)) + .message("flag changed by the TCK's in-process control") + .build()); + } + + @Override + public ProviderEvaluation getBooleanEvaluation(String key, Boolean fallback, EvaluationContext ctx) { + return requireDelegate().getBooleanEvaluation(key, fallback, ctx); + } + + @Override + public ProviderEvaluation getStringEvaluation(String key, String fallback, EvaluationContext ctx) { + return requireDelegate().getStringEvaluation(key, fallback, ctx); + } + + @Override + public ProviderEvaluation getIntegerEvaluation(String key, Integer fallback, EvaluationContext ctx) { + return requireDelegate().getIntegerEvaluation(key, fallback, ctx); + } + + @Override + public ProviderEvaluation getDoubleEvaluation(String key, Double fallback, EvaluationContext ctx) { + return requireDelegate().getDoubleEvaluation(key, fallback, ctx); + } + + @Override + public ProviderEvaluation getObjectEvaluation(String key, Value fallback, EvaluationContext ctx) { + return requireDelegate().getObjectEvaluation(key, fallback, ctx); + } + + /** The store, or an error if not initialised or shut down. */ + private InMemoryProvider requireDelegate() { + InMemoryProvider current = delegate; + if (current == null) { + throw new GeneralError(NAME + " has no flag store: initialize() has not run, or shutdown() released it."); + } + return current; + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableProviderTckTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableProviderTckTest.java new file mode 100644 index 000000000..7e289c31c --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ControllableProviderTckTest.java @@ -0,0 +1,47 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import java.util.EnumSet; +import java.util.Set; + +/** + * Runs the suite against a provider with a real, failable initialisation, covering the lifecycle + * scenarios without Docker. + */ +public class ControllableProviderTckTest extends ProviderTckTest { + + private final ControllableBackendControl control = new ControllableBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public FeatureProvider createProvider() { + return control.createProvider(); + } + + /** A provider over an in-JVM store that refuses to answer. */ + @Override + public FeatureProvider createUnavailableProvider() { + return control.createUnavailableProvider(); + } + + /** {@link InMemoryProviderTckTest}'s set plus LIFECYCLE, REINITIALIZATION and UNAVAILABLE_INIT. */ + @Override + public Set capabilities() { + return EnumSet.of( + Capability.EVENTS, + Capability.LIFECYCLE, + Capability.REINITIALIZATION, + Capability.UNAVAILABLE_INIT, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.VARIANTS, + Capability.DISABLED_FLAGS, + Capability.STRING_TYPING, + Capability.FULLY_TYPED_VALUES, + Capability.STANDARD_REASONS); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/DeclarationApiTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/DeclarationApiTest.java new file mode 100644 index 000000000..4b080f7aa --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/DeclarationApiTest.java @@ -0,0 +1,296 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opentest4j.TestAbortedException; + +/** Tests the declaration vocabulary: capabilities, deviations, configuration names and control APIs. */ +class DeclarationApiTest { + + @Test + @DisplayName("a reserved capability is not declarable and declaring one fails the run") + void reservedCapabilitiesAreNotDeclarable() { + assertThat(Capability.declarable()) + .as("declarable() is every capability some scenario gates") + .doesNotContain(Capability.CACHING) + .contains(Capability.EVENTS, Capability.OBJECT); + + assertThat(Capability.declarableExcept(Capability.STALE)) + .doesNotContain(Capability.STALE, Capability.CACHING) + .contains(Capability.EVENTS); + + assertThatThrownBy(() -> Capability.requireDeclarable(EnumSet.allOf(Capability.class))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("CACHING") + .hasMessageContaining("declares reserved"); + } + + @Test + @DisplayName("targeting is a declarable capability, not a reserved one") + void targetingIsDeclarable() { + // targeting-key-flag scenarios carry the tag, so it gates something + assertThat(Capability.fromTag("@targeting")).contains(Capability.TARGETING); + assertThat(Capability.TARGETING.reserved()).isFalse(); + assertThat(Capability.declarable()).contains(Capability.TARGETING); + Capability.requireDeclarable(EnumSet.of(Capability.TARGETING)); + + // "declare everything" still excludes @caching, the only reserved tag + assertThat(Capability.CACHING.reserved()).isTrue(); + assertThat(Capability.declarable()).doesNotContain(Capability.CACHING); + assertThat(Capability.declarableExcept(Capability.TARGETING)) + .doesNotContain(Capability.TARGETING, Capability.CACHING); + } + + @Test + @DisplayName("variants is declarable, because 2.2.4 is a SHOULD and the field is optional") + void variantsIsADeclarableChoice() { + // 2.2.4 is a SHOULD, so a backend without variants withholds the tag + assertThat(Capability.fromTag("@variants")).contains(Capability.VARIANTS); + assertThat(Capability.VARIANTS.reserved()).isFalse(); + assertThat(Capability.declarable()).contains(Capability.VARIANTS); + + TestAbortedException aborted = catchThrowableOfType( + () -> Capability.requireDeclared( + Arrays.asList("@variants"), Capability.declarableExcept(Capability.VARIANTS)), + TestAbortedException.class); + assertThat(aborted).isNotNull(); + assertThat(aborted) + .hasMessageContaining("VARIANTS") + .hasMessageContaining("@variants") + .hasMessageContaining("does not declare"); + } + + @Test + @DisplayName("a capability the Java SDK cannot express is refused here, not left to every adopter") + void aCapabilityTheSdkCannotExpressIsRefusedCentrally() { + // 2^53 - 1 does not fit the SDK's 32-bit Integer accessor + assertThat(Capability.LARGE_INTEGERS.inexpressible()).isTrue(); + assertThat(Capability.declarable()) + .as("no Java provider may claim it, so \"everything\" does not include it") + .doesNotContain(Capability.LARGE_INTEGERS); + assertThat(Capability.declarableExcept(Capability.EVENTS)) + .as("nor does \"everything except\", which is what the adoptions call") + .doesNotContain(Capability.LARGE_INTEGERS); + + assertThatThrownBy(() -> Capability.requireDeclarable(EnumSet.of(Capability.EVENTS, Capability.LARGE_INTEGERS))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("LARGE_INTEGERS") + .hasMessageContaining("@large-integers") + .as("the message names the SDK limitation") + .hasMessageContaining("Client.getIntegerDetails") + .hasMessageContaining("cannot express"); + } + + @Test + @DisplayName("the two refusals are different facts, and neither message could be mistaken for the other") + void reservedAndInexpressibleAreToldApart() { + // "no scenario carries this yet" vs "this SDK cannot ask it"; neither blames the provider + assertThat(Capability.CACHING.reserved()).isTrue(); + assertThat(Capability.CACHING.inexpressible()) + .as("a reservation is global and expires; it is not a language's limit") + .isFalse(); + assertThat(Capability.LARGE_INTEGERS.reserved()) + .as("scenarios do carry @large-integers, which is what makes it not a reservation") + .isFalse(); + + String reserved = catchThrowableOfType( + () -> Capability.requireDeclarable(EnumSet.of(Capability.CACHING)), + IllegalArgumentException.class) + .getMessage(); + String inexpressible = catchThrowableOfType( + () -> Capability.requireDeclarable(EnumSet.of(Capability.LARGE_INTEGERS)), + IllegalArgumentException.class) + .getMessage(); + + assertThat(reserved) + .as("the reserved refusal says there is nothing to gate yet") + .contains("no scenario in the suite carries") + .as("and never blames the SDK, because a reservation is every language's") + .doesNotContain("SDK"); + assertThat(inexpressible) + .as("the inexpressible refusal says the opposite: the scenarios exist elsewhere") + .contains("the scenarios exist and are asked in languages whose API is wide enough") + .contains("This is not a reserved capability") + .doesNotContain("no scenario in the suite carries"); + + // same distinction in the skip message + TestAbortedException undeclared = catchThrowableOfType( + () -> Capability.requireDeclared( + Arrays.asList("@variants"), Capability.declarableExcept(Capability.VARIANTS)), + TestAbortedException.class); + TestAbortedException unaskable = catchThrowableOfType( + () -> Capability.requireDeclared(Arrays.asList("@large-integers"), Capability.declarable()), + TestAbortedException.class); + + assertThat(undeclared).hasMessageContaining("provider does not declare capability"); + assertThat(unaskable) + .hasMessageContaining("the Java SDK cannot express capability LARGE_INTEGERS") + .hasMessageContaining("Client.getIntegerDetails") + .as("not the provider's decision, and the reason has to say so") + .hasMessageContaining("not the provider under test declining"); + assertThat(unaskable.getMessage()).doesNotContain("provider does not declare"); + + // inexpressible reason wins regardless of declaration + TestAbortedException evenIfSomehowDeclared = catchThrowableOfType( + () -> Capability.requireDeclared( + Arrays.asList("@large-integers"), EnumSet.of(Capability.LARGE_INTEGERS)), + TestAbortedException.class); + assertThat(evenIfSomehowDeclared).hasMessageContaining("the Java SDK cannot express capability"); + } + + @Test + @DisplayName("a tag maps back to the capability it gates") + void tagsMapBackToCapabilities() { + assertThat(Capability.fromTag("@numeric-coercion")).contains(Capability.NUMERIC_COERCION); + assertThat(Capability.fromTag("@not-a-capability")).isEmpty(); + } + + @Test + @DisplayName("reinitialisation is a declarable choice, and withholding it skips only its own scenario") + void reinitialisationIsADeclarableChoice() { + // 2.5.2 permits but does not require reuse after shutdown + assertThat(Capability.fromTag("@reinitialization")).contains(Capability.REINITIALIZATION); + assertThat(Capability.REINITIALIZATION.reserved()).isFalse(); + assertThat(Capability.declarable()).contains(Capability.REINITIALIZATION); + + // withholding it skips only the @reinitialization scenario, not all of @lifecycle + Set reachesBackendButNoReuse = EnumSet.of(Capability.LIFECYCLE); + Capability.requireDeclared(Arrays.asList("@lifecycle"), reachesBackendButNoReuse); + + TestAbortedException aborted = catchThrowableOfType( + () -> Capability.requireDeclared( + Arrays.asList("@lifecycle", "@reinitialization"), reachesBackendButNoReuse), + TestAbortedException.class); + assertThat(aborted).isNotNull(); + assertThat(aborted) + .hasMessageContaining("REINITIALIZATION") + .hasMessageContaining("@reinitialization") + .hasMessageContaining("does not declare"); + } + + @Test + @DisplayName("the gate skips an undeclared capability and lets an untagged scenario run") + void theGateSkipsUndeclaredCapabilities() { + Set declared = EnumSet.of(Capability.EVENTS); + + TestAbortedException aborted = catchThrowableOfType( + () -> Capability.requireDeclared(Arrays.asList("@object"), declared), TestAbortedException.class); + assertThat(aborted) + .as("an undeclared capability aborts, which is what the JUnit Platform reports as skipped") + .isNotNull(); + assertThat(aborted).hasMessageContaining("OBJECT").hasMessageContaining("@object"); + + Capability.requireDeclared(Arrays.asList("@events", "@not-a-capability"), declared); + Capability.requireDeclared(Collections.emptyList(), declared); + } + + @Test + @DisplayName("a deviation records the capability the gap is about, tracked or not") + void deviationsRecordTheCapabilityTheGapIsAbout() { + KnownDeviation tracked = KnownDeviation.tracked( + Capability.NUMERIC_COERCION, "https://example.invalid/1234", "0.5 as an integer returns 0"); + assertThat(tracked.capability).isEqualTo("@numeric-coercion"); + assertThat(tracked.issue).isEqualTo("https://example.invalid/1234"); + assertThat(tracked.summary).isEqualTo("0.5 as an integer returns 0"); + + KnownDeviation untracked = KnownDeviation.untracked(null, "a gap against a mandatory scenario"); + assertThat(untracked.capability).isNull(); + assertThat(untracked.issue).isNull(); + } + + @Test + @DisplayName("a deviation must say what the gap is") + void deviationsMustCarryASummary() { + // a deviation without a summary cannot be acted on + assertThatThrownBy(() -> KnownDeviation.untracked(Capability.OBJECT, null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("no summary"); + + assertThatThrownBy(() -> KnownDeviation.tracked(Capability.OBJECT, "https://example.invalid/1", " ")) + .as("blank is the same omission with whitespace in it") + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("no summary"); + } + + @Test + @DisplayName("a deviation cannot name a capability whose scenarios were never put to the provider") + void deviationsCannotNameAnUnaskedCapability() { + // reserved and inexpressible capabilities are never asked, so cannot deviate + assertThatThrownBy(() -> KnownDeviation.untracked(Capability.CACHING, "no scenario carries it")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("reserved CACHING") + .hasMessageContaining("no scenario in the suite carries") + .hasMessageNotContaining("SDK"); + + assertThatThrownBy(() -> KnownDeviation.tracked( + Capability.LARGE_INTEGERS, "https://example.invalid/1", "cannot be asked")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("LARGE_INTEGERS") + .hasMessageContaining("the Java SDK cannot express") + .hasMessageContaining("Client.getIntegerDetails") + .as("it is not a reservation, and the message must not read as one") + .hasMessageNotContaining("no scenario in the suite carries"); + + // null means a mandatory-scenario gap; ordinary capabilities still allowed + assertThat(KnownDeviation.untracked(null, "a mandatory gap").capability).isNull(); + assertThat(KnownDeviation.untracked(Capability.DISABLED_FLAGS, "withheld and skipped").capability) + .isEqualTo("@disabled-flags"); + } + + @Test + @DisplayName("a configuration name is derived from the suite class, and is overridable") + void configurationNamesAreDerivedFromTheSuiteClass() { + assertThat(new TckSuiteFixture().configuration()).isEqualTo("tck-suite-fixture"); + assertThat(new NamedConfiguration().configuration()).isEqualTo("a-name-of-my-own"); + } + + @Test + @DisplayName("an adopter declares nothing by default, which is silence rather than a claim") + void theDefaultsAreSilence() { + assertThat(new TckSuiteFixture().knownDeviations()).isEmpty(); + assertThat(new TckSuiteFixture().capabilities()).isEqualTo(Capability.declarable()); + } + + @Test + @DisplayName("a backend says what kind it is, and an unfamiliar one is custom rather than wrong") + void backendsSayHowTheyWereDriven() { + assertThat(new InProcessBackendControl().backendType()) + .as("a provider with no backend is controlled in-process, the narrow allowance") + .isEqualTo(BackendType.IN_PROCESS); + + assertThat(BackendType.HTTP.wireValue()).isEqualTo("http"); + assertThat(BackendType.IN_PROCESS.wireValue()).isEqualTo("in-process"); + assertThat(BackendType.CUSTOM.wireValue()).isEqualTo("custom"); + + // closed set + assertThat(BackendType.values()).containsExactly(BackendType.HTTP, BackendType.IN_PROCESS, BackendType.CUSTOM); + + // custom controls default to CUSTOM + assertThat( + new BackendControl() { + @Override + public void prepareScenario() {} + + @Override + public void changeFlag() {} + }.backendType()) + .as("the default is the weaker claim, so silence cannot overstate conformance") + .isEqualTo(BackendType.CUSTOM); + } + + /** A suite that names its configuration rather than taking the derived name. */ + private static final class NamedConfiguration extends TckSuiteFixture { + @Override + public String configuration() { + return "a-name-of-my-own"; + } + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ExtensionPointTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ExtensionPointTest.java new file mode 100644 index 000000000..9aa6b1a1f --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ExtensionPointTest.java @@ -0,0 +1,172 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.cucumber.junit.platform.engine.Constants; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.platform.engine.TestSource; +import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.engine.support.descriptor.ClassSource; +import org.junit.platform.engine.support.descriptor.ClasspathResourceSource; +import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; +import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; +import org.junit.platform.launcher.core.LauncherFactory; +import org.junit.platform.launcher.listeners.SummaryGeneratingListener; +import org.junit.platform.suite.api.ConfigurationParameter; + +/** + * An adopter extends the suite by convention: features under {@code extensions/} and steps in + * {@code openfeature.tck.extensions} are picked up with no annotations. + */ +class ExtensionPointTest { + + private static final String EXTENSION_FEATURE = ProviderTck.EXTENSIONS + "/extension-selftest.feature"; + + @Test + @DisplayName("an extension feature is discovered into the same suite and engine as the canonical set") + void extensionsJoinTheCanonicalSuite() { + TestPlan plan = discover(LauncherDiscoveryRequestBuilder.request() + .selectors(DiscoverySelectors.selectClass(TckSuiteFixture.class))); + + TestIdentifier suite = only(plan, ExtensionPointTest::isTckSuite); + TestIdentifier cucumber = only( + plan, identifier -> "cucumber".equals(engineIdOf(identifier)) && isDescendant(plan, identifier, suite)); + + List resources = new ArrayList<>(); + for (TestIdentifier test : testsUnder(plan, cucumber)) { + resourceOf(test).ifPresent(resources::add); + } + + assertThat(resources) + .as("extension and canonical scenarios share one Cucumber engine under one suite") + .contains(EXTENSION_FEATURE) + .contains(ProviderTck.FEATURES + "/errors.feature"); + } + + @Test + @DisplayName("the extension glue package is resolved, so an adopter's steps need no registration") + void theExtensionGluePackageIsResolved() { + // executed, since unresolved glue only shows at run time; canonical glue omitted to avoid Docker + SummaryGeneratingListener summary = new SummaryGeneratingListener(); + LauncherFactory.create() + .execute( + LauncherDiscoveryRequestBuilder.request() + .selectors(DiscoverySelectors.selectClasspathResource(EXTENSION_FEATURE)) + .configurationParameter(Constants.GLUE_PROPERTY_NAME, ProviderTck.EXTENSION_GLUE) + .configurationParameter(Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, "false") + .configurationParameter( + Constants.OBJECT_FACTORY_PROPERTY_NAME, ProviderTck.OBJECT_FACTORY) + .build(), + summary); + + assertThat(summary.getSummary().getTestsSucceededCount()) + .as("the fixture scenario ran with its steps resolved from %s", ProviderTck.EXTENSION_GLUE) + .isEqualTo(1); + assertThat(summary.getSummary().getTotalFailureCount()).isZero(); + } + + @Test + @DisplayName("the extension directory ships in this artifact, so the selector resolves with no adopter files") + void theExtensionDirectoryIsShipped() { + // a selector on a missing resource is a hard discovery error + assertThat(getClass().getClassLoader().getResource(ProviderTck.EXTENSIONS + "/README.md")) + .as( + "%s/README.md keeps the extension selector resolvable for an adopter who adds nothing", + ProviderTck.EXTENSIONS) + .isNotNull(); + } + + @Test + @DisplayName("the glue constant composes in an annotation value") + void theGlueConstantComposesInAnAnnotationValue() { + // VendorGlue compiling is the real evidence; annotation values must be constants + ConfigurationParameter parameter = VendorGlue.class.getAnnotation(ConfigurationParameter.class); + + assertThat(parameter.value()) + .isEqualTo("dev.openfeature.contrib.tools.tck.internal.steps," + + "openfeature.tck.extensions,com.vendor.steps"); + } + + /** An adopter who wants a third glue package writes this, and does not restate our package. */ + @ConfigurationParameter(key = Constants.GLUE_PROPERTY_NAME, value = ProviderTck.ALL_GLUE + ",com.vendor.steps") + private static final class VendorGlue {} + + private static TestPlan discover(LauncherDiscoveryRequestBuilder request) { + return LauncherFactory.create().discover(request.build()); + } + + private static Optional resourceOf(TestIdentifier identifier) { + Optional source = identifier.getSource(); + if (!source.isPresent() || !(source.get() instanceof ClasspathResourceSource)) { + return Optional.empty(); + } + return Optional.of(((ClasspathResourceSource) source.get()).getClasspathResourceName()); + } + + private static String engineIdOf(TestIdentifier identifier) { + return identifier.getUniqueIdObject().getLastSegment().getType().equals("engine") + ? identifier.getUniqueIdObject().getLastSegment().getValue() + : null; + } + + private static boolean isDescendant(TestPlan plan, TestIdentifier identifier, TestIdentifier ancestor) { + Optional parent = plan.getParent(identifier); + while (parent.isPresent()) { + if (parent.get().equals(ancestor)) { + return true; + } + parent = plan.getParent(parent.get()); + } + return false; + } + + private static List testsUnder(TestPlan plan, TestIdentifier root) { + List tests = new ArrayList<>(); + collectTests(plan, root, tests); + return tests; + } + + private static void collectTests(TestPlan plan, TestIdentifier identifier, List into) { + if (identifier.isTest()) { + into.add(identifier); + } + for (TestIdentifier child : plan.getChildren(identifier)) { + collectTests(plan, child, into); + } + } + + private static TestIdentifier only(TestPlan plan, Predicate predicate) { + List matching = new ArrayList<>(); + for (TestIdentifier root : plan.getRoots()) { + collectMatching(plan, root, predicate, matching); + } + assertThat(matching).hasSize(1); + return matching.get(0); + } + + private static void collectMatching( + TestPlan plan, TestIdentifier identifier, Predicate predicate, List into) { + if (predicate.test(identifier)) { + into.add(identifier); + } + for (TestIdentifier child : plan.getChildren(identifier)) { + collectMatching(plan, child, predicate, into); + } + } + + /** The suite node for {@link TckSuiteFixture}. */ + private static boolean isTckSuite(TestIdentifier identifier) { + return identifier + .getSource() + .filter(ClassSource.class::isInstance) + .map(source -> ((ClassSource) source).getJavaClass()) + .filter(TckSuiteFixture.class::equals) + .isPresent(); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/GlueVisibilityTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/GlueVisibilityTest.java new file mode 100644 index 000000000..dfd810bcd --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/GlueVisibilityTest.java @@ -0,0 +1,104 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.net.URISyntaxException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.CodeSource; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** Verifies glue classes are public; Cucumber silently skips steps it cannot see. */ +class GlueVisibilityTest { + + private static final List> STEP_ANNOTATIONS = + List.of(Given.class, When.class, Then.class); + + @Test + @DisplayName("every canonical step class is public, or Cucumber silently finds no steps") + void stepClassesArePublic() throws Exception { + List> glue = canonicalGlue(); + + assertThat(glue).as("the canonical glue package holds step classes").isNotEmpty(); + + assertThat(glue) + .allSatisfy(type -> assertThat(Modifier.isPublic(type.getModifiers())) + .as("%s must be public, or its scenarios report undefined", type.getSimpleName()) + .isTrue()); + } + + @Test + @DisplayName("every annotated step method is public, or Cucumber silently finds no steps") + void stepMethodsArePublic() throws Exception { + List steps = new ArrayList<>(); + for (Class type : canonicalGlue()) { + for (Method method : type.getDeclaredMethods()) { + if (STEP_ANNOTATIONS.stream().anyMatch(method::isAnnotationPresent)) { + steps.add(method); + } + } + } + + assertThat(steps).as("the canonical glue package defines steps").isNotEmpty(); + + assertThat(steps) + .allSatisfy(method -> assertThat(Modifier.isPublic(method.getModifiers())) + .as( + "%s.%s must be public: getMethods() sees nothing else", + method.getDeclaringClass().getSimpleName(), method.getName()) + .isTrue()); + } + + /** + * Loads the step classes in {@link ProviderTck#GLUE} from the code source. + * + * @return the classes in the canonical glue package that define steps + */ + private static List> canonicalGlue() throws IOException, URISyntaxException, ClassNotFoundException { + CodeSource codeSource = ProviderTck.class.getProtectionDomain().getCodeSource(); + assertThat(codeSource) + .as("the suite has a code source to read the glue package from") + .isNotNull(); + + Path root = Paths.get(codeSource.getLocation().toURI()); + Path pkg = root.resolve(ProviderTck.GLUE.replace('.', '/')); + assertThat(Files.isDirectory(pkg)) + .as("the glue package %s exists at %s", ProviderTck.GLUE, pkg) + .isTrue(); + + List> glue = new ArrayList<>(); + try (DirectoryStream entries = Files.newDirectoryStream(pkg, "*.class")) { + for (Path entry : entries) { + String file = entry.getFileName().toString(); + if (file.contains("$")) { + continue; + } + String name = ProviderTck.GLUE + "." + file.substring(0, file.length() - ".class".length()); + Class type = Class.forName(name); + boolean definesSteps = false; + for (Method method : type.getDeclaredMethods()) { + if (STEP_ANNOTATIONS.stream().anyMatch(method::isAnnotationPresent)) { + definesSteps = true; + break; + } + } + if (definesSteps) { + glue.add(type); + } + } + } + return glue; + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/HttpBackendControlTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/HttpBackendControlTest.java new file mode 100644 index 000000000..98284ce17 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/HttpBackendControlTest.java @@ -0,0 +1,349 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Asserts the request sequence {@link HttpBackendControl} sends, against a JDK {@link HttpServer} + * stub, per the rules in {@code openapi/control-api.yaml}. + */ +class HttpBackendControlTest { + + private static final Duration PROBE_BUDGET = Duration.ofSeconds(5); + + @Test + @DisplayName("prepareScenario uses /reset for every scenario when the backend implements it") + void prefersReset() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + + control.prepareScenario(); + control.prepareScenario(); + + // preferred; /start would cause an availability blip + assertThat(stub.paths()).containsExactly("/reset", "/reset"); + } + } + + @Test + @DisplayName("an unimplemented /reset is probed once, then every scenario falls back to /start") + void fallsBackToStartAndCachesTheAnswer() throws Exception { + try (StubControlApi stub = StubControlApi.start().answering("/reset", 404)) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + + control.prepareScenario(); + control.prepareScenario(); + control.prepareScenario(); + + // probed once per suite; flagd-testbed has no /reset + assertThat(stub.paths()).containsExactly("/reset", "/start", "/start", "/start"); + } + } + + @ParameterizedTest + @ValueSource(ints = {404, 501}) + @DisplayName("both documented not-implemented statuses trigger the fallback") + void bothNotImplementedStatusesFallBack(int status) throws Exception { + try (StubControlApi stub = StubControlApi.start().answering("/reset", status)) { + // control-api.yaml permits either + new HttpBackendControl(stub.baseUrl(), "default").prepareScenario(); + + assertThat(stub.paths()).containsExactly("/reset", "/start"); + } + } + + @Test + @DisplayName("the scenario after a disconnect starts the backend rather than resetting it") + void aDisconnectForcesStart() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + control.prepareScenario(); // settles on /reset, which this stub implements + stub.forget(); + + control.disconnect(); + control.prepareScenario(); + + // /reset is not specified to start a stopped backend + assertThat(stub.paths()).containsExactly("/stop", "/start"); + } + } + + @Test + @DisplayName("reconnecting clears the disconnect, so the next scenario resets again") + void reconnectClearsTheDisconnect() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + control.prepareScenario(); + control.disconnect(); + control.reconnect(); + stub.forget(); + + control.prepareScenario(); + + // backend is up again, so /reset suffices + assertThat(stub.paths()).containsExactly("/reset"); + } + } + + @Test + @DisplayName("the fallback and the reconnect both name the configuration under test") + void startNamesTheBackendConfiguration() throws Exception { + try (StubControlApi stub = StubControlApi.start().answering("/reset", 404)) { + new HttpBackendControl(stub.baseUrl(), "ssl").prepareScenario(); + + assertThat(stub.requests()).containsExactly("POST /reset", "POST /start?config=ssl"); + } + } + + @Test + @DisplayName("no operation ever reaches /restart") + void nothingCallsRestart() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + + control.prepareScenario(); + control.changeFlag(); + control.disconnect(); + control.reconnect(); + control.prepareScenario(); + + // /restart is optional and unused; asserted over the wire + assertThat(stub.paths()).doesNotContain("/restart"); + } + } + + @Test + @DisplayName("changeFlag posts /change") + void changeFlagPostsChange() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + new HttpBackendControl(stub.baseUrl(), "default").changeFlag(); + + assertThat(stub.paths()).containsExactly("/change"); + } + } + + @Test + @DisplayName("an unexpected status fails loudly instead of passing silently") + void anUnexpectedStatusThrows() throws Exception { + try (StubControlApi stub = StubControlApi.start().answering("/change", 500)) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + + // a silent no-op would leave the scenario in an unknown state + assertThatThrownBy(control::changeFlag) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("HTTP 500") + .hasMessageContaining("control-api.yaml"); + } + } + + @Test + @DisplayName("an unreachable control API says which call failed and where") + void anUnreachableControlApiThrows() throws Exception { + String baseUrl = StubControlApi.addressOfAClosedServer(); + HttpBackendControl control = new HttpBackendControl(baseUrl, "default"); + + // an unreachable control API is a broken stack, not an outage + assertThatThrownBy(control::changeFlag) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("POST " + baseUrl + "/change"); + } + + @Test + @DisplayName("baseUrl is reportable without rebuilding it") + void baseUrlIsReportable() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + + assertThat(control.baseUrl()).isEqualTo(stub.baseUrl()); + assertThat(control.description()).contains(stub.baseUrl()); + assertThat(control.backendType()).isEqualTo(BackendType.HTTP); + } + } + + // awaitReady + + @Test + @DisplayName("awaitReady returns as soon as /healthz answers, without sleeping first") + void awaitReadyReturnsOnTheFirstAnswer() throws Exception { + try (StubControlApi stub = StubControlApi.start()) { + new HttpBackendControl(stub.baseUrl(), "default").awaitReady(PROBE_BUDGET); + + assertThat(stub.requests()).containsExactly("GET /healthz"); + } + } + + @Test + @DisplayName("awaitReady treats an unimplemented /healthz as ready") + void awaitReadyAcceptsNotImplemented() throws Exception { + try (StubControlApi stub = StubControlApi.start().answering("/healthz", 404)) { + // 404 means not implemented, which control-api.yaml defines as ready + new HttpBackendControl(stub.baseUrl(), "default").awaitReady(PROBE_BUDGET); + + assertThat(stub.paths()).containsExactly("/healthz"); + } + } + + @Test + @DisplayName("awaitReady keeps probing while the control API says not yet") + void awaitReadyRetriesNotReady() throws Exception { + try (StubControlApi stub = StubControlApi.start().scripting("/healthz", 503, 503, 200)) { + new HttpBackendControl(stub.baseUrl(), "default").awaitReady(Duration.ofSeconds(10)); + + // 503 means not ready, so retry + assertThat(stub.paths()).containsExactly("/healthz", "/healthz", "/healthz"); + } + } + + @Test + @DisplayName("awaitReady gives up reporting what the last probe actually saw") + void awaitReadyReportsTheLastProbe() throws Exception { + try (StubControlApi stub = StubControlApi.start().answering("/healthz", 503)) { + HttpBackendControl control = new HttpBackendControl(stub.baseUrl(), "default"); + + // distinguish refused connection from 503 + assertThatThrownBy(() -> control.awaitReady(Duration.ofMillis(300))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("did not become ready") + .hasRootCauseMessage("control API not ready, HTTP 503"); + } + } + + @Test + @DisplayName("awaitReady reports a control API that is not there at all") + void awaitReadyReportsAnAbsentControlApi() throws Exception { + String baseUrl = StubControlApi.addressOfAClosedServer(); + HttpBackendControl control = new HttpBackendControl(baseUrl, "default"); + + assertThatThrownBy(() -> control.awaitReady(Duration.ofMillis(300))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("did not become ready") + .hasRootCauseInstanceOf(IOException.class); + } + + /** A loopback control API stub that records requests and answers scripted statuses. */ + private static final class StubControlApi implements AutoCloseable { + + private final HttpServer server; + private final List requests = Collections.synchronizedList(new ArrayList<>()); + private final Map statuses = new HashMap<>(); + private final Map> scripted = new HashMap<>(); + + private StubControlApi(HttpServer server) { + this.server = server; + } + + static StubControlApi start() throws IOException { + HttpServer server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + StubControlApi stub = new StubControlApi(server); + server.createContext("/", stub::answer); + server.start(); + return stub; + } + + /** An address that is almost certainly not listening. */ + static String addressOfAClosedServer() throws IOException { + try (StubControlApi stub = start()) { + return stub.baseUrl(); + } + } + + /** Answers {@code status} for every request to {@code path}. */ + StubControlApi answering(String path, int status) { + statuses.put(path, status); + return this; + } + + /** Answers the given statuses in order for {@code path}; the last one repeats. */ + StubControlApi scripting(String path, int... sequence) { + Deque queue = new ArrayDeque<>(); + for (int status : sequence) { + queue.add(status); + } + scripted.put(path, queue); + return this; + } + + String baseUrl() { + return "http://" + server.getAddress().getHostString() + ":" + + server.getAddress().getPort(); + } + + /** Every request as {@code METHOD path[?query]}, in the order received. */ + List requests() { + synchronized (requests) { + return new ArrayList<>(requests); + } + } + + /** Every request's path, dropping the method and the query string. */ + List paths() { + return requests().stream() + .map(request -> request.substring(request.indexOf(' ') + 1)) + .map(target -> target.contains("?") ? target.substring(0, target.indexOf('?')) : target) + .collect(Collectors.toList()); + } + + /** Drops what has been recorded, so a test can assert only the part it set up. */ + void forget() { + requests.clear(); + } + + private void answer(HttpExchange exchange) { + try { + String path = exchange.getRequestURI().getPath(); + String query = exchange.getRequestURI().getQuery(); + requests.add(exchange.getRequestMethod() + " " + path + (query == null ? "" : "?" + query)); + + int status = nextStatus(path); + + byte[] body = "{\"status\":\"stub\"}".getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + exchange.close(); + } + } + + /** The scripted status for this path, or the fixed one, defaulting to 200. */ + private synchronized int nextStatus(String path) { + Deque script = scripted.get(path); + if (script == null) { + return statuses.getOrDefault(path, 200); + } + // last entry repeats so the script never runs dry + return script.size() > 1 ? script.poll() : script.peek(); + } + + @Override + public void close() { + server.stop(0); + } + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/InMemoryProviderTckTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/InMemoryProviderTckTest.java new file mode 100644 index 000000000..2b0228586 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/InMemoryProviderTckTest.java @@ -0,0 +1,40 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.util.EnumSet; +import java.util.Set; + +/** + * Runs the TCK against the SDK's {@link InMemoryProvider}; the reference adoption for a provider with + * no backend, and a fast Docker-free canary. + */ +public class InMemoryProviderTckTest extends ProviderTckTest { + + /** Both the flag store and the provider factory, since changeFlag() must reach the live provider. */ + private final InProcessBackendControl control = new InProcessBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public FeatureProvider createProvider() { + return control.createProvider(); + } + + /** Omissions reflect InMemoryProvider properties (strict typing, no rules, no connection), not defects. */ + @Override + public Set capabilities() { + return EnumSet.of( + Capability.EVENTS, + Capability.CONFIGURATION_CHANGE, + Capability.OBJECT, + Capability.VARIANTS, + Capability.DISABLED_FLAGS, + Capability.STRING_TYPING, + Capability.FULLY_TYPED_VALUES, + Capability.STANDARD_REASONS); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/InProcessBackendControlTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/InProcessBackendControlTest.java new file mode 100644 index 000000000..0d4563de1 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/InProcessBackendControlTest.java @@ -0,0 +1,117 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.openfeature.sdk.ImmutableContext; +import dev.openfeature.sdk.ProviderEvaluation; +import dev.openfeature.sdk.exceptions.TypeMismatchError; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** Asserts {@link InProcessBackendControl} fails loudly on unsupported operations and isolates scenarios. */ +class InProcessBackendControlTest { + + @Test + @DisplayName("connection operations throw rather than silently doing nothing") + void connectionOperationsThrow() { + InProcessBackendControl control = new InProcessBackendControl(); + + // message must name the fix + assertThatThrownBy(control::disconnect) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support 'disconnect'") + .hasMessageContaining("test-configuration bug") + .hasMessageContaining("Capability.STALE"); + + assertThatThrownBy(control::reconnect) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support 'reconnect'"); + } + + @Test + @DisplayName("changing a flag without a provider fails instead of being lost") + void changeFlagWithoutProviderThrows() { + InProcessBackendControl control = new InProcessBackendControl(); + control.prepareScenario(); + + assertThatThrownBy(control::changeFlag) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Given a stable provider"); + } + + @Test + @DisplayName("changeFlag changes the resolved value and the next scenario starts from baseline") + void changeFlagIsVisibleAndDoesNotLeak() throws Exception { + InProcessBackendControl control = new InProcessBackendControl(); + + control.prepareScenario(); + InMemoryProvider first = control.createProvider(); + first.initialize(new ImmutableContext()); + assertThat(resolveChangingFlag(first)).isEqualTo("foo"); + + control.changeFlag(); + assertThat(resolveChangingFlag(first)) + .as("changeFlag must actually change what the provider resolves, not merely emit an event") + .isEqualTo("bar"); + + // next scenario must not inherit the change + control.prepareScenario(); + InMemoryProvider second = control.createProvider(); + second.initialize(new ImmutableContext()); + assertThat(resolveChangingFlag(second)) + .as("each scenario starts from the canonical baseline") + .isEqualTo("foo"); + } + + @Test + @DisplayName("the canonical flag set omits missing-flag") + void missingFlagIsAbsent() throws Exception { + InProcessBackendControl control = new InProcessBackendControl(); + InMemoryProvider provider = control.createProvider(); + provider.initialize(new ImmutableContext()); + + // FLAG_NOT_FOUND scenario relies on its absence + assertThatThrownBy(() -> provider.getStringEvaluation("missing-flag", "fallback", new ImmutableContext())) + .hasMessageContaining("missing-flag"); + } + + @Test + @DisplayName("the canonical flag set keeps the values and types the scenarios depend on") + void canonicalFlagsKeepTheirValuesAndTypes() throws Exception { + InMemoryProvider provider = new InProcessBackendControl().createProvider(); + provider.initialize(new ImmutableContext()); + ImmutableContext context = new ImmutableContext(); + + // must be a double, or the lossless-coercion scenario passes trivially + assertThat(provider.getDoubleEvaluation("integral-float-flag", 0.1, context) + .getValue()) + .as("integral-float-flag is a Double") + .isEqualTo(10.0); + // why the self-tests withhold NUMERIC_COERCION + assertThatThrownBy(() -> provider.getIntegerEvaluation("integral-float-flag", 1, context)) + .isInstanceOf(TypeMismatchError.class); + + assertThat(provider.getIntegerEvaluation("large-integer-flag", 1, context) + .getValue()) + .isEqualTo(2147483647); + + // each default differs from the resolved value + assertThat(provider.getBooleanEvaluation("boolean-zero-flag", true, context) + .getValue()) + .isFalse(); + assertThat(provider.getIntegerEvaluation("integer-zero-flag", 1, context) + .getValue()) + .isZero(); + assertThat(provider.getStringEvaluation("string-zero-flag", "fallback", context) + .getValue()) + .isEmpty(); + } + + private static String resolveChangingFlag(InMemoryProvider provider) { + ProviderEvaluation evaluation = + provider.getStringEvaluation("changing-flag", "unset", new ImmutableContext()); + return evaluation.getValue(); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/MultiProviderTckTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/MultiProviderTckTest.java new file mode 100644 index 000000000..4aeb13ffc --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/MultiProviderTckTest.java @@ -0,0 +1,43 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.multiprovider.MultiProvider; +import dev.openfeature.sdk.providers.memory.InMemoryProvider; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +/** + * Runs the TCK against {@link MultiProvider} wrapping a single {@link InMemoryProvider}, to verify + * delegation is transparent. + */ +public class MultiProviderTckTest extends ProviderTckTest { + + private final InProcessBackendControl control = new InProcessBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public FeatureProvider createProvider() { + return new MultiProvider(Collections.singletonList(control.createProvider())); + } + + /** + * Omits CONFIGURATION_CHANGE since MultiProvider swallows child events; remove once + * java-sdk#1882 is fixed. + */ + @Override + public Set capabilities() { + return EnumSet.of( + Capability.EVENTS, + Capability.OBJECT, + Capability.VARIANTS, + Capability.DISABLED_FLAGS, + Capability.STRING_TYPING, + Capability.FULLY_TYPED_VALUES, + Capability.STANDARD_REASONS); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ReservedTagExpiryTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ReservedTagExpiryTest.java new file mode 100644 index 000000000..e859e26cc --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ReservedTagExpiryTest.java @@ -0,0 +1,80 @@ +package dev.openfeature.contrib.tools.tck; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; +import org.junit.platform.launcher.core.LauncherFactory; +import org.junit.platform.launcher.listeners.SummaryGeneratingListener; +import org.junit.platform.launcher.listeners.TestExecutionSummary; +import org.opentest4j.TestAbortedException; + +/** + * A scenario carrying a {@linkplain Capability#reserved() reserved} tag fails the run rather than + * being silently skipped. + */ +class ReservedTagExpiryTest { + + @Test + @DisplayName("a reserved tag fails its scenario, and a comment naming one does not") + void aReservedTagFailsTheRun() { + SummaryGeneratingListener listener = new SummaryGeneratingListener(); + LauncherFactory.create() + .execute( + LauncherDiscoveryRequestBuilder.request() + .selectors(DiscoverySelectors.selectClass(ReservedTagSuiteFixture.class)) + .build(), + listener); + TestExecutionSummary summary = listener.getSummary(); + + assertThat(summary.getTestsFailedCount()) + .as("the scenario tagged %s fails the run rather than being quietly skipped", Capability.CACHING.tag()) + .isEqualTo(1); + assertThat(summary.getTestsAbortedCount()).as("a failure, not an abort").isZero(); + assertThat(summary.getTestsSkippedCount()).isZero(); + + TestExecutionSummary.Failure failure = summary.getFailures().get(0); + assertThat(failure.getException()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(Capability.CACHING.name()) + .hasMessageContaining(Capability.CACHING.tag()); + assertThat(failure.getTestIdentifier().getDisplayName()) + .as("the tagged scenario is the one that failed") + .contains("still calls reserved"); + + assertThat(summary.getTestsSucceededCount()) + .as("the untagged scenario passes, though a Gherkin comment in it names %s", Capability.CACHING.tag()) + .isEqualTo(1); + } + + @Test + @DisplayName("the reserved tag is reported even when another tag on the scenario is undeclared") + void theExpiryIsReportedBeforeTheSkip() { + assertThatThrownBy(() -> Capability.requireDeclared( + Arrays.asList(Capability.EVENTS.tag(), Capability.CACHING.tag()), + EnumSet.noneOf(Capability.class))) + .as("the expired reservation is reported before the undeclared skip") + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(Capability.CACHING.tag()); + } + + @Test + @DisplayName("a tag that gates nothing, and a declared capability, still pass the gate") + void ordinaryTagsAreUnaffected() { + assertThatCode(() -> Capability.requireDeclared( + Arrays.asList("@some-adopter-tag", Capability.EVENTS.tag()), EnumSet.of(Capability.EVENTS))) + .doesNotThrowAnyException(); + + assertThatThrownBy(() -> Capability.requireDeclared( + Collections.singletonList(Capability.EVENTS.tag()), EnumSet.noneOf(Capability.class))) + .as("an undeclared capability is still a skip, not a failure") + .isInstanceOf(TestAbortedException.class); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ReservedTagSuiteFixture.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ReservedTagSuiteFixture.java new file mode 100644 index 000000000..355bc8742 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/ReservedTagSuiteFixture.java @@ -0,0 +1,48 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import io.cucumber.junit.platform.engine.Constants; +import java.util.EnumSet; +import java.util.Set; +import org.junit.platform.suite.api.ConfigurationParameter; +import org.junit.platform.suite.api.IncludeEngines; +import org.junit.platform.suite.api.SelectClasspathResource; +import org.junit.platform.suite.api.Suite; + +/** + * A real TCK suite over {@code reserved-selftest/}, run by {@link ReservedTagExpiryTest}. Not named + * {@code *Test} so Surefire skips it; does not extend {@link ProviderTckTest} to avoid inheriting its selectors. + */ +@Suite +@IncludeEngines("cucumber") +@SelectClasspathResource(ReservedTagSuiteFixture.FEATURES) +@ConfigurationParameter(key = Constants.PLUGIN_PROPERTY_NAME, value = ProviderTck.PLUGINS) +@ConfigurationParameter(key = Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, value = "false") +@ConfigurationParameter( + key = Constants.EXECUTION_MODE_FEATURE_PROPERTY_NAME, + value = ProviderTck.FEATURE_EXECUTION_MODE) +@ConfigurationParameter(key = Constants.GLUE_PROPERTY_NAME, value = ProviderTck.ALL_GLUE) +@ConfigurationParameter(key = Constants.OBJECT_FACTORY_PROPERTY_NAME, value = ProviderTck.OBJECT_FACTORY) +public class ReservedTagSuiteFixture implements ProviderTckHarness { + + /** The classpath directory holding this fixture's feature file. */ + public static final String FEATURES = "reserved-selftest"; + + private final InProcessBackendControl control = new InProcessBackendControl(); + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public FeatureProvider createProvider() { + return control.createProvider(); + } + + /** Empty; reserved tags cannot be declared. */ + @Override + public Set capabilities() { + return EnumSet.noneOf(Capability.class); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/TckSuiteFixture.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/TckSuiteFixture.java new file mode 100644 index 000000000..ad196880c --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/TckSuiteFixture.java @@ -0,0 +1,34 @@ +package dev.openfeature.contrib.tools.tck; + +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.NoOpProvider; +import java.io.File; +import java.util.Collections; +import java.util.List; + +/** + * A concrete containerised suite, only ever discovered (never run) by these tests. Not named + * {@code *Test} so Surefire skips it. + */ +public class TckSuiteFixture extends ContainerizedProviderTckTest { + + @Override + public File composeFile() { + return new File("src/test/resources/there-is-no-stack.yaml"); + } + + @Override + public List backendPorts() { + return Collections.singletonList(8013); + } + + @Override + public FeatureProvider createProvider(BackendEndpoint endpoint) { + return new NoOpProvider(); + } + + @Override + public FeatureProvider createUnavailableProvider() { + return new NoOpProvider(); + } +} diff --git a/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/internal/steps/TckLifecycleTest.java b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/internal/steps/TckLifecycleTest.java new file mode 100644 index 000000000..eeb047ca9 --- /dev/null +++ b/tools/tck/src/test/java/dev/openfeature/contrib/tools/tck/internal/steps/TckLifecycleTest.java @@ -0,0 +1,127 @@ +package dev.openfeature.contrib.tools.tck.internal.steps; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.openfeature.contrib.tools.tck.BackendControl; +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.NoOpProvider; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** Verifies each lifecycle owns its own harness and backend. */ +class TckLifecycleTest { + + /** A harness that records its own lifecycle and brings nothing up. */ + private static final class RecordingHarness implements ProviderTckHarness { + + private final BackendControl control = BackendControl.inProcess(); + private int starts; + private int stops; + + @Override + public FeatureProvider createProvider() { + return new NoOpProvider(); + } + + @Override + public BackendControl backendControl() { + return control; + } + + @Override + public void startSuite() { + starts++; + } + + @Override + public void stopSuite() { + stops++; + } + } + + @AfterEach + void releaseWhateverIsLeft() { + TckLifecycle.endSuite(); + } + + @Test + @DisplayName("two lifecycles own separate harnesses and separate backends") + void lifecyclesAreIndependent() { + RecordingHarness first = new RecordingHarness(); + RecordingHarness second = new RecordingHarness(); + + TckLifecycle one = TckLifecycle.start(first); + TckLifecycle two = TckLifecycle.start(second); + + assertThat(one).isNotSameAs(two); + assertThat(one.harness()).isSameAs(first); + assertThat(two.harness()).isSameAs(second); + assertThat(one.backendControl()) + .as("each lifecycle has its own control") + .isNotSameAs(two.backendControl()); + assertThat(first.starts).isEqualTo(1); + assertThat(second.starts).isEqualTo(1); + assertThat(TckLifecycle.startedCount()).isEqualTo(2); + } + + @Test + @DisplayName("endSuite stops every lifecycle, not only the one the suite thread started") + void endSuiteStopsAll() { + RecordingHarness onThisThread = new RecordingHarness(); + RecordingHarness onAnother = new RecordingHarness(); + TckLifecycle.start(onThisThread); + TckLifecycle.start(onAnother); + + TckLifecycle.endSuite(); + + assertThat(onThisThread.stops).isEqualTo(1); + assertThat(onAnother.stops) + .as("@AfterAll runs on one thread, so the registry is what stops the others") + .isEqualTo(1); + assertThat(TckLifecycle.startedCount()).isZero(); + } + + @Test + @DisplayName("a failed start releases what it allocated instead of leaving it registered") + void aFailedStartLeavesNothingBehind() { + class NullControlHarness implements ProviderTckHarness { + private int stops; + + @Override + public FeatureProvider createProvider() { + return new NoOpProvider(); + } + + @Override + public BackendControl backendControl() { + return null; + } + + @Override + public void stopSuite() { + stops++; + } + } + NullControlHarness harness = new NullControlHarness(); + + assertThatThrownBy(() -> TckLifecycle.start(harness)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("backendControl() returned null"); + + assertThat(harness.stops) + .as("a partial start is torn down by the starter") + .isEqualTo(1); + assertThat(TckLifecycle.startedCount()) + .as("and nothing half-started is left for endSuite to find") + .isZero(); + } + + @Test + @DisplayName("no suite running means no lifecycle, so extension-only glue registers nothing") + void noSuiteMeansNoLifecycle() { + assertThat(TckLifecycle.startForExecutingSuite()).isEmpty(); + } +} diff --git a/tools/tck/src/test/java/openfeature/tck/extensions/ExtensionSelfTestSteps.java b/tools/tck/src/test/java/openfeature/tck/extensions/ExtensionSelfTestSteps.java new file mode 100644 index 000000000..fc4334db6 --- /dev/null +++ b/tools/tck/src/test/java/openfeature/tck/extensions/ExtensionSelfTestSteps.java @@ -0,0 +1,29 @@ +package openfeature.tck.extensions; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; + +/** + * Steps for the TCK's own extension fixture, proving {@code openfeature.tck.extensions} is on the + * glue path with no configuration. + */ +public class ExtensionSelfTestSteps { + + private boolean ran; + + /** A step Cucumber can only find if the extension glue package is on the glue path. */ + @Given("a step defined in the extension glue package") + public void aStepDefinedInTheExtensionGluePackage() { + ran = true; + } + + /** Asserts the step above ran, so that a missing glue package fails rather than passes. */ + @Then("the extension step ran") + public void theExtensionStepRan() { + assertThat(ran) + .as("the extension glue package was resolved and its steps executed") + .isTrue(); + } +} diff --git a/tools/tck/src/test/resources/extensions/extension-selftest.feature b/tools/tck/src/test/resources/extensions/extension-selftest.feature new file mode 100644 index 000000000..182ede7ed --- /dev/null +++ b/tools/tck/src/test/resources/extensions/extension-selftest.feature @@ -0,0 +1,8 @@ +@extension-selftest +Feature: An adopter's own scenarios run inside the TCK suite + + The TCK's own proof of its extension point; test-scoped, so not in the released JAR. + + Scenario: A step class in the adopter's own glue package is on the suite's glue path + Given a step defined in the extension glue package + Then the extension step ran diff --git a/tools/tck/src/test/resources/reserved-selftest/reserved-tag.feature b/tools/tck/src/test/resources/reserved-selftest/reserved-tag.feature new file mode 100644 index 000000000..963cbccd4 --- /dev/null +++ b/tools/tck/src/test/resources/reserved-selftest/reserved-tag.feature @@ -0,0 +1,12 @@ +Feature: A reserved capability tag on a scenario fails the run + + Proof of the Capability.reserved() expiry check, run by ReservedTagExpiryTest. Kept outside + extensions/ since the second scenario is meant to fail. + + Scenario: A Gherkin comment naming a reserved tag is prose, not a tag + # untagged; this comment mentions @caching, which must not count as a tag + Given a stable provider + + @caching + Scenario: A tag this implementation still calls reserved fails the run + Given a stable provider diff --git a/tools/tck/version.txt b/tools/tck/version.txt new file mode 100644 index 000000000..6e8bf73aa --- /dev/null +++ b/tools/tck/version.txt @@ -0,0 +1 @@ +0.1.0