From a04784d207d8b75462b3a3d1d38b98e18e7e9ea1 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 15:07:21 +0200 Subject: [PATCH 01/27] test(flagd): add a seam for flagd-specific TCK extension scenarios The TCK builds every provider with createProvider() and no arguments. The flagd-testbed scenarios configure the provider per scenario, so the adoption now records per-scenario option changes and applies them in createProvider, and exposes the launchpad URL. FlagdSteps holds what every extension feature shares: gating on @rpc and @in-process, the testbed's option step, launchpad configuration and restart, a context table, and a negative event assertion. Signed-off-by: Simon Schrottner --- .../flagd/tck/AbstractResolverTest.java | 39 +++- .../providers/flagd/tck/FlagdTckHarness.java | 43 ++++ .../providers/flagd/tck/InProcessTest.java | 2 +- .../contrib/providers/flagd/tck/RpcTest.java | 2 +- .../tck/extensions/FlagdSteps.java | 219 ++++++++++++++++++ 5 files changed, 299 insertions(+), 6 deletions(-) create mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java create mode 100644 providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java index f8a9b1b545..40c3aed95c 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java @@ -11,6 +11,8 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; /** * Shared configuration for running the OpenFeature Provider TCK against the flagd provider. @@ -20,7 +22,7 @@ * the ruleset off the disk. All three share a backend stack; the two gRPC modes differ only in * resolver and port, and {@link FileTest} says what the third needs instead. */ -abstract class AbstractResolverTest extends ContainerizedProviderTckTest { +abstract class AbstractResolverTest extends ContainerizedProviderTckTest implements FlagdTckHarness { /** A port nothing listens on; an outage on the Compose stack belongs to the control API. */ private static final int UNAVAILABLE_PORT = 9999; @@ -36,8 +38,12 @@ abstract class AbstractResolverTest extends ContainerizedProviderTckTest { /** gRPC deadline for a dead port; these scenarios assert that the failure is reported promptly. */ private static final int UNAVAILABLE_DEADLINE_MS = 1000; + /** Per-scenario option changes recorded by the extension steps; see {@link FlagdTckHarness}. */ + private final List> customizations = new CopyOnWriteArrayList<>(); + /** The resolver under test. */ - protected abstract Config.Resolver resolver(); + @Override + public abstract Config.Resolver resolver(); /** * The container-internal port this mode waits on before the first scenario. @@ -71,13 +77,13 @@ public List backendPorts() { @Override public FeatureProvider createProvider(BackendEndpoint endpoint) { - return new FlagdProvider(flagSource(baseOptions().deadline(CONNECTED_DEADLINE_MS), endpoint) + return new FlagdProvider(customized(flagSource(baseOptions().deadline(CONNECTED_DEADLINE_MS), endpoint)) .build()); } @Override public FeatureProvider createUnavailableProvider() { - return new FlagdProvider(missingFlagSource(baseOptions().deadline(UNAVAILABLE_DEADLINE_MS)) + return new FlagdProvider(customized(missingFlagSource(baseOptions().deadline(UNAVAILABLE_DEADLINE_MS))) .build()); } @@ -103,6 +109,26 @@ protected FlagdOptions.FlagdOptionsBuilder missingFlagSource(FlagdOptions.FlagdO return options.host("localhost").port(UNAVAILABLE_PORT); } + @Override + public void customize(Consumer customization) { + customizations.add(customization); + } + + @Override + public void resetCustomizations() { + customizations.clear(); + } + + @Override + public BackendEndpoint backendEndpoint() { + return endpoint(); + } + + @Override + public String launchpadUrl() { + return "http://" + endpoint().host() + ":" + endpoint().port(controlPort()); + } + /** * {@inheritDoc} * @@ -120,6 +146,11 @@ public Set capabilities() { return Capability.declarableExcept(Capability.REINITIALIZATION); } + private FlagdOptions.FlagdOptionsBuilder customized(FlagdOptions.FlagdOptionsBuilder builder) { + customizations.forEach(customization -> customization.accept(builder)); + return builder; + } + private FlagdOptions.FlagdOptionsBuilder baseOptions() { return FlagdOptions.builder() .resolverType(resolver()) diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java new file mode 100644 index 0000000000..11a6707f18 --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java @@ -0,0 +1,43 @@ +package dev.openfeature.contrib.providers.flagd.tck; + +import dev.openfeature.contrib.providers.flagd.Config; +import dev.openfeature.contrib.providers.flagd.FlagdOptions; +import dev.openfeature.contrib.tools.tck.BackendEndpoint; +import java.util.function.Consumer; + +/** + * What the flagd extension scenarios under {@code src/test/resources/extensions} need from the + * adoption beyond the TCK's own harness. + * + *

The TCK builds every provider through {@code createProvider()} with no arguments, which suits + * the canonical scenarios: they ask the same question of every provider. The extensions port the + * flagd-testbed scenarios, which configure the provider per scenario (a cache, a selector, a grace + * period). Rather than a second way of creating a provider, a scenario records its configuration + * here and the TCK's own {@code Given a stable provider} applies it. + * + *

Step classes receive the harness by injection and cast it to this interface. + */ +public interface FlagdTckHarness { + + /** The resolver this suite runs, for gating {@code @rpc} and {@code @in-process} scenarios. */ + Config.Resolver resolver(); + + /** + * Records a change to the options of the next provider this scenario creates. + * + *

Applied after the adoption's defaults, in the order recorded, so a scenario can override + * any of them. Cleared before every scenario. + * + * @param customization a change to the options builder + */ + void customize(Consumer customization); + + /** Discards every recorded customization. */ + void resetCustomizations(); + + /** The running Compose stack, for steps that need a port the adoption does not use itself. */ + BackendEndpoint backendEndpoint(); + + /** Base URL of the flagd-testbed launchpad control API, without a trailing slash. */ + String launchpadUrl(); +} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/InProcessTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/InProcessTest.java index 3dd2341506..b3f1dcc71d 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/InProcessTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/InProcessTest.java @@ -16,7 +16,7 @@ public String configuration() { } @Override - protected Config.Resolver resolver() { + public Config.Resolver resolver() { return Config.Resolver.IN_PROCESS; } diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/RpcTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/RpcTest.java index 2a3ebe12ef..4283d6058c 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/RpcTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/RpcTest.java @@ -18,7 +18,7 @@ public String configuration() { } @Override - protected Config.Resolver resolver() { + public Config.Resolver resolver() { return Config.Resolver.RPC; } diff --git a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java new file mode 100644 index 0000000000..c0f01f4fe9 --- /dev/null +++ b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java @@ -0,0 +1,219 @@ +package openfeature.tck.extensions; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.openfeature.contrib.providers.flagd.Config; +import dev.openfeature.contrib.providers.flagd.FlagdOptions; +import dev.openfeature.contrib.providers.flagd.resolver.rpc.cache.CacheType; +import dev.openfeature.contrib.providers.flagd.tck.FlagdTckHarness; +import dev.openfeature.contrib.tools.tck.ProviderTckHarness; +import dev.openfeature.contrib.tools.tck.TckState; +import dev.openfeature.sdk.MutableContext; +import dev.openfeature.sdk.Value; +import io.cucumber.datatable.DataTable; +import io.cucumber.java.After; +import io.cucumber.java.Before; +import io.cucumber.java.Scenario; +import io.cucumber.java.en.Given; +import io.cucumber.java.en.Then; +import io.cucumber.java.en.When; +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import org.opentest4j.TestAbortedException; + +/** + * Shared steps for the flagd extension scenarios: resolver gating, provider options and the + * launchpad endpoints the canonical {@code BackendControl} does not cover. + * + *

The features port flagd-testbed scenarios the TCK does not ask. They keep the testbed's + * wording wherever it does not collide with a TCK step. It collides on the provider: the TCK's + * {@code a {} provider} also matches {@code a stable flagd provider}, so a scenario records its + * options with {@code an option ...} and then uses the TCK's {@code Given a stable provider}. + */ +public class FlagdSteps { + + private static final HttpClient HTTP = HttpClient.newHttpClient(); + + /** Option names the testbed spells differently from {@link FlagdOptions.FlagdOptionsBuilder}. */ + private static final Map OPTION_NAMES; + + static { + Map names = new HashMap<>(); + names.put("resolver", "resolverType"); + names.put("deadlineMs", "deadline"); + names.put("keepAliveTime", "keepAlive"); + names.put("cache", "cacheType"); + OPTION_NAMES = Collections.unmodifiableMap(names); + } + + private final TckState state; + private final FlagdTckHarness flagd; + + public FlagdSteps(TckState state, ProviderTckHarness harness) { + this.state = state; + this.flagd = (FlagdTckHarness) harness; + } + + /** + * Skips a scenario tagged for the other resolver, and starts every scenario with the adoption's + * own options. + * + *

Runs after the TCK's capability gate (order 0) and before it prepares the backend (order + * 10). A scenario tagged with neither {@code @rpc} nor {@code @in-process} runs in both. + * + * @param scenario the scenario about to run + */ + @Before(order = 1) + public void gateOnResolver(Scenario scenario) { + flagd.resetCustomizations(); + Collection tags = scenario.getSourceTagNames(); + boolean rpc = tags.contains("@rpc"); + boolean inProcess = tags.contains("@in-process"); + if (!rpc && !inProcess) { + return; + } + boolean matches = flagd.resolver() == Config.Resolver.RPC ? rpc : inProcess; + if (!matches) { + throw new TestAbortedException("Skipped: scenario is not tagged for the " + flagd.resolver() + " resolver"); + } + } + + /** Leaves no customization behind for a canonical scenario that follows. */ + @After + public void resetCustomizations() { + flagd.resetCustomizations(); + } + + /** + * Sets an option on the next provider this scenario creates, by the testbed's option name. + * + * @param option the option name, as config.feature spells it + * @param type the value's type + * @param value the value + */ + @Given("an option {string} of type {string} with value {string}") + public void anOption(String option, String type, String value) throws IOException { + Object converted = convert(value, type); + String name = OPTION_NAMES.getOrDefault(option, option); + Method setter = Arrays.stream(FlagdOptions.FlagdOptionsBuilder.class.getMethods()) + .filter(method -> method.getName().equals(name) && method.getParameterCount() == 1) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("FlagdOptions has no option '" + name + "'")); + flagd.customize(builder -> { + try { + setter.invoke(builder, converted); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Cannot set option '" + name + "' to " + converted, e); + } + }); + } + + /** + * Restarts flagd on one of the launchpad's flag configurations ({@code default}, + * {@code metadata}, {@code ssl}, {@code sync-payload}). + * + *

Comes before {@code Given a stable provider}, so the provider connects to it. The TCK's + * {@code the connection is restored} returns to {@code default}. + * + * @param configuration the launchpad configuration name + */ + @Given("the flagd backend serves the {string} configuration") + public void theBackendServes(String configuration) { + launchpad("/start?config=" + configuration); + } + + /** + * Stops flagd and starts it again after the given delay, without waiting for it. + * + * @param seconds how long flagd stays down + */ + @When("the connection is lost for {int}s") + public void theConnectionIsLostFor(int seconds) { + launchpad("/restart?seconds=" + seconds); + } + + /** + * Adds typed entries to the evaluation context. + * + * @param table rows with {@code key}, {@code type} and {@code value} columns + */ + @Given("a context with the following keys:") + public void aContextWithTheFollowingKeys(DataTable table) throws IOException { + Map entries = new HashMap<>(state.context.asMap()); + for (Map row : table.asMaps(String.class, String.class)) { + entries.put(row.get("key"), Value.objectToValue(convert(row.get("value"), row.get("type")))); + } + state.context = new MutableContext(state.context.getTargetingKey(), entries); + } + + /** + * Asserts that no event of the given kind was recorded during the scenario. + * + * @param eventType the event kind + */ + @Then("the {} event handler should not have been executed") + public void theEventHandlerShouldNotHaveBeenExecuted(String eventType) { + assertThat(state.events.stream().anyMatch(event -> event.type().equals(eventType))) + .as("no %s event should have fired", eventType) + .isFalse(); + } + + private void launchpad(String path) { + HttpRequest request = HttpRequest.newBuilder(URI.create(flagd.launchpadUrl() + path)) + .POST(HttpRequest.BodyPublishers.noBody()) + .build(); + try { + HttpResponse response = HTTP.send(request, HttpResponse.BodyHandlers.ofString()); + assertThat(response.statusCode()).as("launchpad POST %s", path).isEqualTo(200); + } catch (IOException e) { + throw new IllegalStateException("launchpad POST " + path + " failed", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted during launchpad POST " + path, e); + } + } + + /** The testbed's value conversion, for the types the extension features use. */ + static Object convert(String value, String type) throws IOException { + if ("null".equals(value) && !"String".equals(type)) { + return null; + } + switch (type) { + case "Boolean": + return Boolean.parseBoolean(value); + case "String": + return value; + case "Integer": + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + return Long.parseLong(value); + } + case "Long": + return Long.parseLong(value); + case "Float": + return Double.parseDouble(value); + case "CacheType": + return CacheType.valueOf(value.toUpperCase()).getValue(); + case "StringList": + return value.isEmpty() + ? Collections.emptyList() + : Arrays.stream(value.split(",")).map(String::trim).collect(Collectors.toList()); + case "Object": + return Value.objectToValue(new ObjectMapper().readValue(value, Object.class)); + default: + throw new IllegalArgumentException("Unknown value type '" + type + "'"); + } + } +} From f5ca143bc57f2c0d9e228f42b6fe30d926906932 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 15:12:53 +0200 Subject: [PATCH 02/27] test(flagd): restore the default launchpad configuration after an extension scenario Signed-off-by: Simon Schrottner --- .../openfeature/tck/extensions/FlagdSteps.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java index c0f01f4fe9..b0bc41f2b6 100644 --- a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java +++ b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java @@ -60,6 +60,9 @@ public class FlagdSteps { private final TckState state; private final FlagdTckHarness flagd; + /** The launchpad configuration this scenario switched to, if any. */ + private String servedConfiguration; + public FlagdSteps(TckState state, ProviderTckHarness harness) { this.state = state; this.flagd = (FlagdTckHarness) harness; @@ -89,10 +92,17 @@ public void gateOnResolver(Scenario scenario) { } } - /** Leaves no customization behind for a canonical scenario that follows. */ + /** + * Leaves nothing behind for a canonical scenario that follows: no customization, and flagd back + * on the {@code default} configuration. The TCK's own preparation may only reset flag state, + * which would keep a scenario's {@code ssl} or {@code metadata} configuration running. + */ @After - public void resetCustomizations() { + public void restoreDefaults() { flagd.resetCustomizations(); + if (servedConfiguration != null && !"default".equals(servedConfiguration)) { + launchpad("/start?config=default"); + } } /** @@ -131,6 +141,7 @@ public void anOption(String option, String type, String value) throws IOExceptio @Given("the flagd backend serves the {string} configuration") public void theBackendServes(String configuration) { launchpad("/start?config=" + configuration); + servedConfiguration = configuration; } /** From 9dd94b34ea390298d7d9311c98ac2b7a9c5b4cac Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 15:34:01 +0200 Subject: [PATCH 03/27] test(flagd): select the fractional bucketing version by tag Scenarios tagged @fractional-vN run only when N is the version the backend implements: v3 by default, -Dflagd.tck.fractional=v2 to run the v2 expectations instead. Untagged scenarios run regardless. Signed-off-by: Simon Schrottner --- .../providers/flagd/tck/FlagdTckHarness.java | 13 +++++++++ .../tck/extensions/FlagdSteps.java | 27 +++++++++++++++---- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java index 11a6707f18..a3d35ad496 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java @@ -22,6 +22,19 @@ public interface FlagdTckHarness { /** The resolver this suite runs, for gating {@code @rpc} and {@code @in-process} scenarios. */ Config.Resolver resolver(); + /** + * The fractional bucketing version the backend implements, as the testbed tags it + * ({@code v1}, {@code v2}, {@code v3}). A scenario tagged {@code @fractional-vN} for another + * version is skipped. + * + *

{@code v3} by default: the Java in-process evaluator and the flagd shipped with + * flagd-testbed v3.10.4 both implement it. Override with {@code -Dflagd.tck.fractional=v2} to + * run the v2 expectations, against an older image for instance. + */ + default String fractionalVersion() { + return System.getProperty("flagd.tck.fractional", "v3"); + } + /** * Records a change to the options of the next provider this scenario creates. * diff --git a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java index b0bc41f2b6..7fdac6d863 100644 --- a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java +++ b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java @@ -28,6 +28,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.opentest4j.TestAbortedException; @@ -83,12 +84,28 @@ public void gateOnResolver(Scenario scenario) { Collection tags = scenario.getSourceTagNames(); boolean rpc = tags.contains("@rpc"); boolean inProcess = tags.contains("@in-process"); - if (!rpc && !inProcess) { - return; + if (rpc || inProcess) { + boolean matches = flagd.resolver() == Config.Resolver.RPC ? rpc : inProcess; + if (!matches) { + throw new TestAbortedException( + "Skipped: scenario is not tagged for the " + flagd.resolver() + " resolver"); + } } - boolean matches = flagd.resolver() == Config.Resolver.RPC ? rpc : inProcess; - if (!matches) { - throw new TestAbortedException("Skipped: scenario is not tagged for the " + flagd.resolver() + " resolver"); + gateOnFractionalVersion(tags); + } + + /** + * Skips a scenario whose {@code @fractional-vN} tags name only other bucketing versions than + * {@link FlagdTckHarness#fractionalVersion()}. A scenario with no version tag runs regardless. + */ + private void gateOnFractionalVersion(Collection tags) { + List versions = tags.stream() + .filter(tag -> tag.matches("@fractional-v\\d+")) + .collect(Collectors.toList()); + String wanted = "@fractional-" + flagd.fractionalVersion(); + if (!versions.isEmpty() && !versions.contains(wanted)) { + throw new TestAbortedException("Skipped: scenario expects fractional " + versions + + ", the backend implements " + wanted + " (override with -Dflagd.tck.fractional)"); } } From 4dfa21a5804385bb55f4945b3b7ee75c16470ba3 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 15:22:23 +0200 Subject: [PATCH 04/27] test(flagd): port the testbed's evaluation, disabled and selector scenarios as TCK extensions Signed-off-by: Simon Schrottner --- .../resources/extensions/disabled.feature | 39 ++++++++++ .../resources/extensions/evaluation.feature | 74 +++++++++++++++++++ .../resources/extensions/selector.feature | 27 +++++++ 3 files changed, 140 insertions(+) create mode 100644 providers/flagd/src/test/resources/extensions/disabled.feature create mode 100644 providers/flagd/src/test/resources/extensions/evaluation.feature create mode 100644 providers/flagd/src/test/resources/extensions/selector.feature diff --git a/providers/flagd/src/test/resources/extensions/disabled.feature b/providers/flagd/src/test/resources/extensions/disabled.feature new file mode 100644 index 0000000000..44a29337e3 --- /dev/null +++ b/providers/flagd/src/test/resources/extensions/disabled.feature @@ -0,0 +1,39 @@ +@ext-evaluation @disabled @in-process @rpc +Feature: Disabled flag evaluation + + # Ported from flagd-testbed gherkin/disabled.feature. A flag with state=DISABLED resolves + # successfully with reason=DISABLED; the evaluator substitutes the caller-provided default value + # and omits the variant. Relates to: https://github.com/open-feature/flagd/issues/1965 + # + # The TCK's @disabled-flags scenarios ask the value for the four scalar flags and the reason for + # disabled-boolean-flag only, so the boolean row is left out here: the remaining rows add the + # DISABLED reason for each type and the structured flag. + + Scenario Outline: Evaluating a disabled flag returns reason DISABLED and code default + Given an option "cache" of type "CacheType" with value "disabled" + And a stable provider + And a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "DISABLED" + + Examples: + | key | type | default | + | disabled-string-flag | String | bye | + | disabled-integer-flag | Integer | 1 | + | disabled-float-flag | Float | 0.1 | + | disabled-object-flag | Object | {} | + + Scenario Outline: Flag disabled in one flag set, enabled in another + Given an option "cache" of type "CacheType" with value "disabled" + And an option "selector" of type "String" with value "" + And a stable provider + And a Boolean-flag with key "cross-flagset-flag" and a default value "false" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "" + + Examples: + | selector | resolved_value | reason | + | flags/allFlags.json | false | DISABLED | + | rawflags/selector-flags.json | true | STATIC | diff --git a/providers/flagd/src/test/resources/extensions/evaluation.feature b/providers/flagd/src/test/resources/extensions/evaluation.feature new file mode 100644 index 0000000000..bf05fcb5c3 --- /dev/null +++ b/providers/flagd/src/test/resources/extensions/evaluation.feature @@ -0,0 +1,74 @@ +@ext-evaluation @rpc @in-process +Feature: flagd evaluations + + # Ported from flagd-testbed gherkin/evaluation.feature. Asks what the TCK's evaluation.feature + # and reason.feature do not: the reason flagd reports for a zero value, a float zero value, + # zero values served by a targeting rule, and flags with no default variant. + # + # Not ported: "Resolve values" (the TCK's evaluation.feature asks it) and the @deprecated + # @no-default outline, which @no-default-variant replaces. + + Background: + Given an option "cache" of type "CacheType" with value "disabled" + And a stable provider + + Scenario Outline: Resolves zero value + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "STATIC" + + Examples: + | key | type | default | resolved_value | + | boolean-zero-flag | Boolean | true | false | + | string-zero-flag | String | hi | | + | integer-zero-flag | Integer | 1 | 0 | + | float-zero-flag | Float | 0.1 | 0.0 | + + @targeting + Scenario Outline: Resolves zero value with targeting + Given a -flag with key "" and a default value "" + And a context containing a key "email", with type "String" and with value "ballmer@macrosoft.com" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "TARGETING_MATCH" + + Examples: + | key | type | default | resolved_value | + | boolean-targeted-zero-flag | Boolean | true | false | + | string-targeted-zero-flag | String | hi | | + | integer-targeted-zero-flag | Integer | 1 | 0 | + | float-targeted-zero-flag | Float | 0.1 | 0.0 | + + @targeting + Scenario Outline: Resolves zero value with targeting using default + Given a -flag with key "" and a default value "" + And a context containing a key "email", with type "String" and with value "ballmer@none.com" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "DEFAULT" + + Examples: + | key | type | default | resolved_value | + | boolean-targeted-zero-flag | Boolean | true | false | + | string-targeted-zero-flag | String | hi | | + | integer-targeted-zero-flag | Integer | 1 | 0 | + | float-targeted-zero-flag | Float | 0.1 | 0.0 | + + @no-default-variant + Scenario Outline: Resolves flag with no default variant correctly + Given a -flag with key "" and a default value "" + And a context containing a key "email", with type "String" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "" + + Examples: + | key | type | email | code_default | resolved_value | reason | + | null-default-flag | Boolean | | true | true | DEFAULT | + | null-default-flag | Boolean | | false | false | DEFAULT | + | undefined-default-flag | Integer | | 100 | 100 | DEFAULT | + | no-default-flag-null-targeting-variant | String | wozniak@orange.com | Inventor | Inventor | DEFAULT | + | no-default-flag-null-targeting-variant | String | wozniak@orange.com | Founder | Founder | DEFAULT | + | no-default-flag-null-targeting-variant | String | jobs@orange.com | CEO | CEO | TARGETING_MATCH | + | no-default-flag-undefined-targeting-variant | String | wozniak@orange.com | Retired | Retired | DEFAULT | diff --git a/providers/flagd/src/test/resources/extensions/selector.feature b/providers/flagd/src/test/resources/extensions/selector.feature new file mode 100644 index 0000000000..8351c56ff7 --- /dev/null +++ b/providers/flagd/src/test/resources/extensions/selector.feature @@ -0,0 +1,27 @@ +@ext-evaluation @in-process +Feature: flagd selector + + # Ported from flagd-testbed gherkin/selector.feature. The launchpad's default configuration + # serves both flags/allFlags.json and rawflags/selector-flags.json, so the selector narrows the + # sync to the second source. + + Background: + Given an option "selector" of type "String" with value "rawflags/selector-flags.json" + And a stable provider + + Scenario Outline: Flags not found + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the reason should be "ERROR" + + Examples: + | key | type | default | + | boolean-flag | Boolean | false | + | string-flag | String | bye | + | integer-flag | Integer | 1 | + | float-flag | Float | 0.1 | + + Scenario: Resolve values + Given a String-flag with key "selector-flag" and a default value "foo" + When the flag was evaluated with details + Then the reason should be "STATIC" From 151d011fe2bf7b7959e946bfb06878ffa80c9e18 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 15:22:46 +0200 Subject: [PATCH 05/27] test(flagd): port the testbed's non-fractional targeting scenarios as a TCK extension Evaluator reuse and refs, string and sem_ver operators, time-based rules, error edge cases and operator errors, run in both resolvers under @ext-targeting. Signed-off-by: Simon Schrottner --- .../resources/extensions/targeting.feature | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 providers/flagd/src/test/resources/extensions/targeting.feature diff --git a/providers/flagd/src/test/resources/extensions/targeting.feature b/providers/flagd/src/test/resources/extensions/targeting.feature new file mode 100644 index 0000000000..4480c61eb5 --- /dev/null +++ b/providers/flagd/src/test/resources/extensions/targeting.feature @@ -0,0 +1,222 @@ +@ext-targeting @rpc @in-process +Feature: Targeting rules + + # Ported from flagd-testbed's gherkin/targeting.feature: the json-evaluation scenarios the Provider + # TCK does not ask. The @fractional scenarios live in their own extension feature. + # + # Not ported: + # - "Targeting by targeting key": the TCK's evaluation.feature (@targeting) asserts both values + # and reason.feature asserts TARGETING_MATCH and DEFAULT for the same flag and keys. + # - "fractional operator with missing bucket key": a fractional scenario, despite its tags. + # + # Deviations from the testbed's wording: + # - "a stable flagd provider" collides with the TCK's "a {} provider", so the provider step is the + # TCK's "a stable provider". + # - "Time-based operations" sets the context with "a context with the following keys:", because the + # TCK's context step parses Integer as 32 bits and 4133980802 does not fit. + + Background: + Given an option "cache" of type "CacheType" with value "disabled" + And a stable provider + + # evaluator refs + Scenario Outline: Evaluator reuse + Given a String-flag with key "" and a default value "fallback" + And a context containing a key "email", with type "String" and with value "ballmer@macrosoft.com" + When the flag was evaluated with details + Then the resolved details value should be "" + Examples: + | key | value | + | some-email-targeted-flag | hi | + | some-other-email-targeted-flag | yes | + + @evaluator-refs @evaluator-refs-whitespace + Scenario Outline: Evaluator $ref resolves regardless of whitespace around the colon + Given a String-flag with key "" and a default value "fallback" + And a context containing a key "email", with type "String" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "TARGETING_MATCH" + Examples: + | key | email | value | + | ref-whitespace-compact-flag | ballmer@macrosoft.com | hi | + | ref-whitespace-space-after-colon-flag | ballmer@macrosoft.com | hi | + | ref-whitespace-space-around-colon-flag | ballmer@macrosoft.com | hi | + | ref-whitespace-compact-flag | user@example.com | bye | + | ref-whitespace-space-after-colon-flag | user@example.com | bye | + | ref-whitespace-space-around-colon-flag | user@example.com | bye | + + @evaluator-refs @non-existent-evaluator-ref + Scenario: Ref to nonexistent evaluator yields parse error + Given a String-flag with key "ref-to-nonexistent-evaluator-flag" and a default value "fallback" + When the flag was evaluated with details + Then the resolved details value should be "fallback" + And the error-code should be "PARSE_ERROR" + + @string + Scenario Outline: Substring operators + Given a String-flag with key "starts-ends-flag" and a default value "fallback" + And a context containing a key "id", with type "String" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + Examples: + | id | value | + | abcdef | prefix | + | uvwxyz | postfix | + | abcxyz | prefix | + | lmnopq | none | + | 3 | none | + + @semver + Scenario Outline: Semantic version operator numeric comparison + Given a String-flag with key "equal-greater-lesser-version-flag" and a default value "fallback" + And a context containing a key "version", with type "String" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + Examples: + | version | value | + | 2.0.0 | equal | + | 2.1.0 | greater | + | 1.9.0 | lesser | + | 2.0.0-alpha | lesser | + | 2.0.0.0 | none | + + @semver + Scenario Outline: Semantic version operator semantic comparison + Given a String-flag with key "major-minor-version-flag" and a default value "fallback" + And a context containing a key "version", with type "String" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + Examples: + | version | value | + | 3.0.1 | minor | + | 3.1.0 | major | + | 4.0.0 | none | + + @semver @semver-edge-cases @semver-v-prefix + Scenario Outline: sem_ver v-prefix handling + Given a String-flag with key "semver-v-prefix-flag" and a default value "fallback" + And a context containing a key "version", with type "String" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + Examples: + | version | value | + | 1.0.0 | match | + | v1.0.0 | match | + | V1.0.0 | match | + | 2.0.0 | no-match | + + @semver @semver-edge-cases @semver-partial-version + Scenario Outline: sem_ver partial version string handling + Given a String-flag with key "semver-partial-version-flag" and a default value "fallback" + And a context containing a key "version", with type "" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + Examples: + | version | type | value | + | 1.5.0 | String | match | + | 1.0.0 | String | match | + | 1.0 | String | match | + | 1 | String | match | + | 1 | Integer | match | + | 1 | Float | match | + | 1.2 | Float | match | + | 2.0.0 | String | no-match | + + @semver @semver-edge-cases @semver-numeric-context + Scenario Outline: sem_ver numeric context value coercion + Given a String-flag with key "semver-numeric-context-flag" and a default value "fallback" + And a context containing a key "version", with type "" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + Examples: + | version | type | value | + | 1.2 | Float | match | + | 1.1 | Float | no-match | + | 2 | Integer | match | + | 1 | Integer | no-match | + | 1.2 | String | match | + + @semver @semver-edge-cases @semver-build-metadata + Scenario Outline: sem_ver build metadata ignored + Given a String-flag with key "semver-build-metadata-flag" and a default value "fallback" + And a context containing a key "version", with type "String" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + Examples: + | version | value | + | 1.0.0 | match | + | 1.0.0+other | match | + | 2.0.0 | no-match | + + Scenario Outline: Time-based operations + Given a Integer-flag with key "timestamp-flag" and a default value "0" + And a context with the following keys: + | key | type | value | + | time | Integer |

Event ordering needs nothing more than that. The TCK's {@code the {} event handler should have + * been executed} drains the recorded events through the one it matched, so each assertion in a + * chain waits for an event newer than the previous one. What draining cannot express is an + * absence between two events: once {@code ready} is matched, an {@code error} recorded before it + * has been discarded, and {@code the error event handler should not have been executed} passes + * vacuously. {@link #executedWithNoneBefore} asserts the absence before it drains. + */ +public class ConnectionSteps { + + /** A port nothing listens on, so a provider given a target URI cannot fall back to host and port. */ + private static final int UNAVAILABLE_PORT = 9999; + + private final TckState state; + private final ProviderTckHarness harness; + private final FlagdTckHarness flagd; + + public ConnectionSteps(TckState state, ProviderTckHarness harness) { + this.state = state; + this.harness = harness; + this.flagd = (FlagdTckHarness) harness; + } + + /** + * Connects the next provider through envoy, by a target URI whose {@code } is envoy's + * routing listener. The host and port options point at a closed port, as the testbed does, so + * the scenario passes only if the target URI is what the provider used. + * + * @param targetUri the testbed's target URI, with {@code localhost:} standing for envoy + */ + @Given("a target URI {string} routed through envoy") + public void aTargetUriRoutedThroughEnvoy(String targetUri) { + BackendEndpoint endpoint = flagd.backendEndpoint(); + String resolved = targetUri + .replace("localhost", endpoint.host(FlagdTckHarness.ENVOY_SERVICE)) + .replace( + "", + String.valueOf( + endpoint.port(FlagdTckHarness.ENVOY_SERVICE, FlagdTckHarness.ENVOY_ROUTING_PORT))); + flagd.customize(builder -> builder.targetUri(resolved).port(UNAVAILABLE_PORT)); + } + + /** + * Points the next provider at envoy's listener that answers every call with HTTP 403, which + * gRPC reports as {@code PERMISSION_DENIED}. + * + *

Followed by the TCK's {@code Given a unavailable provider}, the flavour that does not wait + * for readiness; a forbidden provider never becomes ready. + */ + @Given("the provider connects to envoy's forbidden listener") + public void theProviderConnectsToTheForbiddenListener() { + BackendEndpoint endpoint = flagd.backendEndpoint(); + String host = endpoint.host(FlagdTckHarness.ENVOY_SERVICE); + int port = endpoint.port(FlagdTckHarness.ENVOY_SERVICE, FlagdTckHarness.ENVOY_FORBIDDEN_PORT); + flagd.customize(builder -> builder.host(host).port(port)); + } + + /** + * Awaits an event of one kind, asserts that no event of another kind was recorded before it, + * and then drains through it as the TCK's own await does. + * + * @param eventType the event kind to await + * @param forbiddenType the event kind that must not precede it + */ + @Then("the {} event handler should have been executed with no {} event before it") + public void executedWithNoneBefore(String eventType, String forbiddenType) { + await().alias("provider event " + eventType) + .atMost(harness.eventTimeout().toMillis(), MILLISECONDS) + .pollInterval(10, MILLISECONDS) + .until(() -> + state.events.stream().anyMatch(event -> event.type().equals(eventType))); + + List before = new ArrayList<>(); + ProviderEventRecord matched = null; + while (!state.events.isEmpty()) { + ProviderEventRecord head = state.events.poll(); + if (head.type().equals(eventType)) { + matched = head; + break; + } + before.add(head); + } + state.lastEvent = Optional.ofNullable(matched); + + assertThat(before.stream().map(ProviderEventRecord::type).collect(Collectors.toList())) + .as("events recorded before the %s event", eventType) + .doesNotContain(forbiddenType); + } +} diff --git a/providers/flagd/src/test/resources/extensions/connection.feature b/providers/flagd/src/test/resources/extensions/connection.feature new file mode 100644 index 0000000000..5932c9d437 --- /dev/null +++ b/providers/flagd/src/test/resources/extensions/connection.feature @@ -0,0 +1,116 @@ +@ext-connection +Feature: flagd provider disconnect and reconnect + + # Ported from flagd-testbed gherkin/connection.feature. + # + # Not ported: the "stable" row of the Connection outline, which every canonical evaluation + # scenario asks, and the @unixsocket rows, which need a socket shared between host and container + # that the TCK stack does not mount. The testbed's "Provider unavailable" is kept although the + # canonical lifecycle.feature asks the same, because it holds flagd to a 3000ms bound where the + # canonical scenario allows 10000ms. + # + # The outlines over provider flavours are spelled out, one scenario per flavour, because a flavour + # here is a set of steps rather than a value. Reconnect timing is the testbed's "stable flagd + # provider", as in events.feature. + # + # @targetURI and @forbidden connect through the envoy sidecar that + # tools/flagd-testbed/docker-compose.flagd.yaml adds to the shared stack. + + @customCert + Scenario: Connection with a custom certificate + Given the flagd backend serves the "ssl" configuration + And an option "tls" of type "Boolean" with value "true" + And an option "certPath" of type "String" with value "test-harness/ssl/custom-root-cert.crt" + And a stable provider + And a Boolean-flag with key "boolean-flag" and a default value "false" + When the flag was evaluated with details + Then the resolved details value should be "true" + + @reconnect + Scenario: Provider reconnection + Given an option "retryGracePeriod" of type "Integer" with value "5" + And an option "retryBackoffMaxMs" of type "Integer" with value "500" + And a stable provider + And a ready event handler + And a error event handler + When a ready event was fired + When the connection is lost for 8s + Then the error event handler should have been executed + Then the ready event handler should have been executed + + @reconnect @customCert + Scenario: Provider reconnection with a custom certificate + Given the flagd backend serves the "ssl" configuration + And an option "tls" of type "Boolean" with value "true" + And an option "certPath" of type "String" with value "test-harness/ssl/custom-root-cert.crt" + And an option "retryGracePeriod" of type "Integer" with value "5" + And an option "retryBackoffMaxMs" of type "Integer" with value "500" + And a stable provider + And a ready event handler + And a error event handler + When a ready event was fired + When the connection is lost for 8s + Then the error event handler should have been executed + Then the ready event handler should have been executed + + @unavailable + Scenario: Provider unavailable + Given an option "deadlineMs" of type "Integer" with value "1000" + And a unavailable provider + And a error event handler + Then the error event handler should have been executed within 3000ms + + # A forbidden response from flagd results in a fatal client state. "a unavailable provider" is the + # TCK's flavour that does not wait for readiness; the step before it points that provider at envoy. + @forbidden + Scenario: Provider forbidden + Given an option "fatalStatusCodes" of type "StringList" with value "PERMISSION_DENIED" + And the provider connects to envoy's forbidden listener + And a unavailable provider + And a error event handler + Then the error event handler should have been executed within 5000ms + And the client should be in fatal state + + @targetURI @rpc + Scenario: Connection via TargetUri rpc + Given a target URI "envoy://localhost:/rpc.service" routed through envoy + And a stable provider + And a Boolean-flag with key "boolean-flag" and a default value "false" + When the flag was evaluated with details + Then the resolved details value should be "true" + + @targetURI @in-process + Scenario: Connection via TargetUri in-process + Given a target URI "envoy://localhost:/sync.service" routed through envoy + And a stable provider + And a Boolean-flag with key "boolean-flag" and a default value "false" + When the flag was evaluated with details + Then the resolved details value should be "true" + + @targetURI @in-process @reconnect + Scenario: Re-Connection via TargetUri in-process + Given a target URI "envoy://localhost:/sync.service" routed through envoy + And an option "retryGracePeriod" of type "Integer" with value "5" + And an option "retryBackoffMaxMs" of type "Integer" with value "500" + And a stable provider + And a ready event handler + And a error event handler + When a ready event was fired + When the connection is lost for 8s + Then the error event handler should have been executed + Then the ready event handler should have been executed + + @targetURI @in-process @reconnect @grace + Scenario: Re-Connection via TargetUri in-process with grace period + Given a target URI "envoy://localhost:/sync.service" routed through envoy + And an option "retryGracePeriod" of type "Integer" with value "5" + And an option "retryBackoffMaxMs" of type "Integer" with value "500" + And a stable provider + And a ready event handler + And a stale event handler + And a error event handler + When a ready event was fired + When the connection is lost for 8s + Then the stale event handler should have been executed + Then the error event handler should have been executed + Then the ready event handler should have been executed diff --git a/providers/flagd/src/test/resources/extensions/events.feature b/providers/flagd/src/test/resources/extensions/events.feature new file mode 100644 index 0000000000..5602ed4e32 --- /dev/null +++ b/providers/flagd/src/test/resources/extensions/events.feature @@ -0,0 +1,52 @@ +@ext-connection @events +Feature: flagd provider state changes + + # Ported from flagd-testbed gherkin/events.feature. Not ported: "Flag change event", which the + # canonical events.feature asks in "A configuration change is signalled and applied", together + # with the changed evaluation it leaves out. + # + # The canonical @stale scenario stops flagd and starts it again on request. These chains instead + # take flagd down for a fixed time with the launchpad's /restart, so they ask what the retry grace + # period decides: an outage shorter than it ends stale -> ready, a longer one escalates to error. + # + # Each "should have been executed" drains the recorded events through the one it matched, so the + # next assertion waits for a newer event and a chain asserts its order. + # + # Timing is the testbed's "stable flagd provider" except for the deadline, which stays at the + # adoption's 15000ms for the in-process sync (see AbstractResolverTest): a 5s grace period, and a + # retry backoff capped at 500ms so reconnecting does not wait out an exponential backoff. + + Background: + Given an option "retryGracePeriod" of type "Integer" with value "5" + And an option "retryBackoffMaxMs" of type "Integer" with value "500" + And a stable provider + + Scenario: Provider events chain ready -> error -> ready + Given a ready event handler + And a error event handler + Then the ready event handler should have been executed + When the connection is lost for 8s + Then the error event handler should have been executed + Then the ready event handler should have been executed + + @grace + Scenario: Provider events chain ready -> stale -> error -> ready + Given a ready event handler + And a error event handler + And a stale event handler + Then the ready event handler should have been executed + When the connection is lost for 8s + Then the stale event handler should have been executed + Then the error event handler should have been executed + Then the ready event handler should have been executed + + @grace + Scenario: Provider events chain ready -> stale -> ready + Given a ready event handler + And a error event handler + And a stale event handler + Then the ready event handler should have been executed + When the connection is lost for 1s + Then the stale event handler should have been executed + Then the ready event handler should have been executed with no error event before it + And the error event handler should not have been executed diff --git a/providers/flagd/src/test/resources/extensions/rpc-caching.feature b/providers/flagd/src/test/resources/extensions/rpc-caching.feature new file mode 100644 index 0000000000..65beb871b6 --- /dev/null +++ b/providers/flagd/src/test/resources/extensions/rpc-caching.feature @@ -0,0 +1,67 @@ +@ext-connection @rpc @flagd-caching +Feature: Flag evaluation with caching + + # Ported from flagd-testbed gherkin/rpc-caching.feature (RPC only). The testbed's @caching tag is + # a reserved TCK capability, which the TCK refuses on any scenario, so this feature carries + # @flagd-caching instead. + # + # Reconnect timing is the testbed's "stable flagd provider", as in events.feature. + + Background: + Given an option "cache" of type "CacheType" with value "lru" + And an option "retryGracePeriod" of type "Integer" with value "5" + And an option "retryBackoffMaxMs" of type "Integer" with value "500" + And a stable provider + + Scenario Outline: Resolves details with caching + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the variant should be "" + And the reason should be "STATIC" + When the flag was evaluated with details + Then the resolved details value should be "" + And the variant should be "" + And the reason should be "CACHED" + + Examples: + | key | type | default | resolved_variant | resolved_value | + | boolean-flag | Boolean | false | on | true | + | string-flag | String | bye | greeting | hi | + | integer-flag | Integer | 1 | ten | 10 | + | float-flag | Float | 0.1 | half | 0.5 | + | object-flag | Object | {} | template | {"showImages": true, "title": "Check out these pics!", "imagesPerPage": 100.0 } | + + Scenario: Flag change event with caching + Given a String-flag with key "changing-flag" and a default value "false" + And a change event handler + When the flag was modified + And a change event was fired + And the flag was evaluated with details + Then the reason should be "STATIC" + When the flag was evaluated with details + Then the reason should be "CACHED" + When the flag was modified + And a change event was fired + And the flag was evaluated with details + Then the reason should be "STATIC" + When the flag was evaluated with details + Then the reason should be "CACHED" + + Scenario: Stale and Error stage + Given a ready event handler + And a stale event handler + And a error event handler + And a Boolean-flag with key "boolean-flag" and a default value "false" + When the flag was evaluated with details + Then the reason should be "STATIC" + When the flag was evaluated with details + Then the reason should be "CACHED" + When the connection is lost for 6s + And a stale event was fired + And the flag was evaluated with details + Then the reason should be "CACHED" + When a error event was fired + And a ready event was fired + And the flag was evaluated with details + Then the reason should be "STATIC" From 526de6bc3afdeff31b857c98ffd1817dc5a64f14 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 15:24:25 +0200 Subject: [PATCH 08/27] test(flagd): port the testbed's fractional scenarios as a TCK extension Every @fractional scenario of flagd-testbed's targeting.feature, plus the fractional @operator-errors ones, without the v1/v2 bucketing variants the legacy e2e suites exclude. Object and beyond-32-bit integer context values use the table step, as the TCK's context step cannot convert them. Signed-off-by: Simon Schrottner --- .../resources/extensions/fractional.feature | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 providers/flagd/src/test/resources/extensions/fractional.feature diff --git a/providers/flagd/src/test/resources/extensions/fractional.feature b/providers/flagd/src/test/resources/extensions/fractional.feature new file mode 100644 index 0000000000..5540f636e8 --- /dev/null +++ b/providers/flagd/src/test/resources/extensions/fractional.feature @@ -0,0 +1,397 @@ +@ext-fractional @rpc @in-process +Feature: Fractional targeting + + # Ported from the flagd-testbed's gherkin/targeting.feature: every @fractional scenario, plus the + # fractional @operator-errors ones. The @fractional-v1 and @fractional-v2 bucketing variants are + # left out, as the legacy e2e suites exclude them. The TCK's own @targeting scenarios do not cover + # the fractional operator. + # + # Where the testbed's context values are Objects or integers outside 32 bits, the TCK's context + # step cannot convert them, so those outlines use the table step with the testbed's conversion. + + Background: + Given an option "cache" of type "CacheType" with value "disabled" + And a stable provider + + @fractional + Scenario Outline: Fractional operator + Given a String-flag with key "fractional-flag" and a default value "fallback" + And a context containing a nested property with outer key "user" and inner key "name", with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + + @fractional-v3 + Examples: v3 + | name | value | + | jack | diamonds | + | queen | diamonds | + | ten | clubs | + | nine | clubs | + | 3 | spades | + + @fractional + Scenario Outline: Fractional operator shorthand + Given a String-flag with key "fractional-flag-shorthand" and a default value "fallback" + And a context containing a targeting key with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + + @fractional-v3 + Examples: v3 + | targeting key | value | + | jon@company.com | tails | + | jane@company.com | tails | + + @fractional + Scenario Outline: Fractional operator with shared seed + Given a String-flag with key "fractional-flag-A-shared-seed" and a default value "fallback" + And a context containing a nested property with outer key "user" and inner key "name", with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + + @fractional-v3 + Examples: v3 + | name | value | + | seven | hearts | + | eight | hearts | + | nine | diamonds | + | two | diamonds | + + @fractional + Scenario Outline: Second fractional operator with shared seed + Given a String-flag with key "fractional-flag-B-shared-seed" and a default value "fallback" + And a context containing a nested property with outer key "user" and inner key "name", with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + + @fractional-v3 + Examples: v3 + | name | value | + | seven | ace-of-hearts | + | eight | ace-of-hearts | + | nine | ace-of-diamonds | + | two | ace-of-diamonds | + + # Nested JSON Logic expressions as bucket variant names. + # Requires providers to support the @fractional-nested feature. + # Use -t "not @fractional-nested" to exclude during transition. + @fractional @fractional-nested + Scenario Outline: Fractional operator with nested if expression as variant name + # fractional-nested-if-flag: seed=targetingKey, bucket0=[if(tier=="premium","premium","standard"),50], bucket1=["standard",50] + # jon@company.com bv(100)=36 → bucket0; user1 bv(100)=76 → bucket1 + Given a String-flag with key "fractional-nested-if-flag" and a default value "fallback" + And a context containing a targeting key with value "" + And a context containing a key "tier", with type "String" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + Examples: + | targetingKey | tier | value | + | jon@company.com | premium | premium | + | jon@company.com | basic | standard | + | user1 | premium | standard | + | user1 | basic | standard | + + @fractional @fractional-nested + Scenario Outline: Fractional operator with nested var expression as variant name + # fractional-nested-var-flag: seed=targetingKey, bucket0=[var("color"),50], bucket1=["blue",50] + # jon@company.com bv(100)=36 → bucket0 (resolves var "color"); user1 bv(100)=76 → bucket1 ("blue") + Given a String-flag with key "fractional-nested-var-flag" and a default value "fallback" + And a context containing a targeting key with value "" + And a context containing a key "color", with type "String" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + Examples: + | targetingKey | color | value | + | jon@company.com | red | red | + | jon@company.com | green | green | + | user1 | red | blue | + | jon@company.com | yellow | fallback | + | jon@company.com | | fallback | + + @fractional @fractional-nested + Scenario Outline: Fractional operator with nested if expression as weight + # fractional-nested-weight-flag: seed=targetingKey, bucket0=["red",if(tier=="premium",100,0)], bucket1=["blue",10] + Given a String-flag with key "fractional-nested-weight-flag" and a default value "fallback" + And a context containing a targeting key with value "" + And a context containing a key "tier", with type "String" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + Examples: + | targetingKey | tier | value | + | jon@company.com | premium | red | + | jon@company.com | basic | blue | + | user1 | premium | red | + | user1 | basic | blue | + + @fractional @fractional-nested + Scenario: Fractional as condition + Given a String-flag with key "fractional-as-condition-flag" and a default value "zero" + And a context containing a targeting key with value "some-targeting-key" + When the flag was evaluated with details + Then the resolved details value should be "hundreds" + + @fractional @fractional-nested + Scenario: Fractional as condition evaluates false path + Given a String-flag with key "fractional-as-condition-false-flag" and a default value "zero" + And a context containing a targeting key with value "some-targeting-key" + When the flag was evaluated with details + Then the resolved details value should be "ones" + + @operator-errors + Scenario: fractional operator with missing bucket key falls back to default variant + Given a String-flag with key "fractional-null-bucket-key-flag" and a default value "wrong" + When the flag was evaluated with details + Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" + + @operator-errors @fractional + Scenario: fractional with all-zero bucket weights falls back to default variant + Given a String-flag with key "fractional-zero-weights-flag" and a default value "wrong" + And a context containing a targeting key with value "any-user" + When the flag was evaluated with details + Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" + + @operator-errors @fractional + Scenario: fractional negative bucket weight is clamped to zero + # ["one", -50] is treated as ["one", 0]; "two" gets 100% of the weight + Given a String-flag with key "fractional-negative-weight-flag" and a default value "wrong" + And a context containing a targeting key with value "any-user" + When the flag was evaluated with details + Then the resolved details value should be "two" + + @fractional @fractional-v3 + Scenario Outline: Fractional operator with basic types + Given a String-flag with key "fractional-basic-flag" and a default value "fallback" + And a context containing a key "hashing_input", with type "" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + + Examples: v3 + | hashing_input | type | value | + | true | Boolean | bucket5 | + | false | Boolean | bucket1 | + | user1 | String | bucket22 | + | user2 | String | bucket16 | + | 123 | Integer | bucket20 | + | 456 | Integer | bucket12 | + | 1.23 | Float | bucket14 | + | 4.56 | Float | bucket18 | + | 123 | String | bucket22 | + | true | String | bucket24 | + | false | String | bucket1 | + | null | String | bucket8 | + | 1.23 | String | bucket4 | + | 0 | Integer | bucket8 | + + @fractional @fractional-v3 + Scenario Outline: Fractional operator with float mapping + Given a String-flag with key "fractional-basic-flag" and a default value "fallback" + And a context with the following keys: + | key | type | value | + | hashing_input | | | + When the flag was evaluated with details + Then the resolved details value should be "" + + Examples: v3 + | hashing_input | type | value | + | 1.0 | Float | bucket23 | + | 1 | Integer | bucket23 | + | 1.0000000000000001 | Float | bucket23 | + | -2.0 | Float | bucket12 | + | -2 | Integer | bucket12 | + | 9007199254740992.0 | Float | bucket10 | + | 9007199254740992 | Integer | bucket10 | + + @fractional @fractional-v3 + Scenario Outline: Fractional operator with zero values + Given a String-flag with key "fractional-basic-flag" and a default value "fallback" + And a context containing a key "hashing_input", with type "" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + + Examples: v3 + | hashing_input | type | value | + | 0.0 | Float | bucket8 | + | -0.0 | Float | bucket8 | + | 0 | Integer | bucket8 | + + @fractional @fractional-v3 + Scenario Outline: Fractional operator with integer limits + Given a String-flag with key "fractional-basic-flag" and a default value "fallback" + And a context with the following keys: + | key | type | value | + | hashing_input | Integer | | + When the flag was evaluated with details + Then the resolved details value should be "" + + Examples: v3 + | hashing_input | value | + | 2147483647 | bucket10 | + | 2147483648 | bucket25 | + | -2147483648 | bucket8 | + | -2147483649 | bucket18 | + | 9007199254740991 | bucket2 | + | 23 | bucket21 | + | 24 | bucket11 | + | 255 | bucket13 | + | 256 | bucket13 | + | 65535 | bucket4 | + | 65536 | bucket11 | + | 4294967295 | bucket14 | + | 4294967296 | bucket19 | + | -24 | bucket4 | + | -25 | bucket16 | + + @fractional @fractional-v3 + Scenario Outline: Fractional operator with floats limits + Given a String-flag with key "fractional-basic-flag" and a default value "fallback" + And a context containing a key "hashing_input", with type "Float" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + + Examples: v3 + | hashing_input | value | + | 1e20 | bucket19 | + | -1e20 | bucket25 | + | 65504.0 | bucket16 | + | -65504.0 | bucket9 | + | 0.00006103515625 | bucket22 | + | 0.000000059604644775390625 | bucket23 | + | 65505.0 | bucket17 | + | -65505.0 | bucket17 | + | 3.4028234663852886e+38 | bucket10 | + | -3.4028234663852886e+38 | bucket3 | + | 1.1754943508222875e-38 | bucket24 | + | 3.5e+38 | bucket1 | + | -3.5e+38 | bucket6 | + | 1.7976931348623157e+308 | bucket17 | + | -1.7976931348623157e+308 | bucket9 | + | 2.2250738585072014e-308 | bucket11 | + | 4.9406564584124654e-324 | bucket2 | + + @fractional @fractional-v3 + Scenario Outline: Fractional operator with map key ordering + Given a String-flag with key "fractional-basic-flag" and a default value "fallback" + And a context with the following keys: + | key | type | value | + | hashing_input | Object | | + When the flag was evaluated with details + Then the resolved details value should be "" + + Examples: v3 + | hashing_input | value | + | {"a": 1, "b": 2} | bucket23 | + | {"b": 2, "a": 1} | bucket23 | + | {"a": 1, "b": 2, "c": 3} | bucket18 | + | {"c": 3, "a": 1, "b": 2} | bucket18 | + | {"z": 1, "aa": 2} | bucket21 | + | {"aa": 2, "z": 1} | bucket21 | + | {"a": {"b": 1}, "b": 2} | bucket9 | + | {"b": 2, "a": {"b": 1}} | bucket9 | + | {"a": true, "bbb": 1, "c": "text", "dddd": 2.5} | bucket25 | + | {"c": "text", "bbb": 1, "a": true, "dddd": 2.5} | bucket25 | + | {"aaa": 1, "ÿ": 2} | bucket12 | + | {"ÿ": 2, "aaa": 1} | bucket12 | + | {"": 2, "abc": 1} | bucket25 | + | {"key": -0.0} | bucket10 | + | {"key": 0} | bucket10 | + | {"key": 0.0} | bucket10 | + | {"b": -0.0, "a": {"b": 1}, "c": 0.0} | bucket8 | + | {"c": 0.0, "b": -0.0, "a": {"b": 1}} | bucket8 | + | {"café": "façade", "b": {"résumé": {"ÿ": true}}} | bucket22 | + + @fractional @fractional-v3 + Scenario Outline: Fractional operator with advanced structures + Given a String-flag with key "fractional-basic-flag" and a default value "fallback" + And a context with the following keys: + | key | type | value | + | hashing_input | Object | | + When the flag was evaluated with details + Then the resolved details value should be "" + + Examples: v3 + | hashing_input | value | + | {} | bucket3 | + | {"a": {} } | bucket20 | + | {"b": 2, "a": {} } | bucket20 | + | {"a": {}, "b": {}} | bucket21 | + | {"a": {}, "b": []} | bucket7 | + | {"c": 0.0, "a": []} | bucket18 | + | {"c": 0.0, "b": -0.0, "a": {"b": []}} | bucket9 | + | {"a": [] } | bucket12 | + | {"a": [1, [], 3], "b": null} | bucket3 | + | {"a": ["a", "b", {}], "b": [1, null, 2]} | bucket23 | + | {"a": [1, "two", true, [], null]} | bucket23 | + | {"a": [false, 2.5, ""], "b": [], "c": [{}], "c": null} | bucket15 | + | {"x": [{"a": 1}, {"b": 2}, {"b": null}]} | bucket3 | + | {"x": [{"a": {"b": []}}] } | bucket16 | + | {"x": [{"a": {"b": null}}] } | bucket6 | + | {"x": [{"a": {"b": {}}}] } | bucket2 | + | {"": "", "": [], "": {}, "": null} | bucket25 | + | {"": {"": ""}, "": ["", {"": ""}], "": {"": {"": ""}}} | bucket11 | + | {"a": [[[[[]]]]] } | bucket15 | + | {"a": [[[[null]]]] } | bucket7 | + | {"a": {"b": {"c": {"d": {}}} } } | bucket21 | + | {"a": {"b": {"c": {"d": null}} } } | bucket19 | + + @fractional @fractional-v3 + Scenario Outline: Fractional operator with string length boundaries + Given a String-flag with key "fractional-basic-flag" and a default value "fallback" + And a context containing a key "hashing_input", with type "String" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + + Examples: v3 + | hashing_input | value | + | | bucket4 | + | a | bucket6 | + | 12345678901234567890123 | bucket4 | + | 123456789012345678901234 | bucket14 | + | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | bucket18 | + | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | bucket7 | + + @fractional @fractional-v3 + Scenario Outline: Valid well-formed UTF-8 String consistency across different languages + Given a String-flag with key "fractional-basic-flag" and a default value "fallback" + And a context containing a key "hashing_input", with type "String" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + + Examples: v3 + | hashing_input | value | + | あいうえおかき | bucket17 | + | あいうえおかきく | bucket16 | + | ééééééééééé | bucket16 | + | éééééééééééé | bucket19 | + | café façade résumé café | bucket9 | + | A\u0000B | bucket15 | + | e\u0301 | bucket7 | + | \uD83D\uDE00 | bucket23 | + + @fractional @fractional-v3 + Scenario Outline: Fractional operator using implicit targeting key + Given a String-flag with key "fractional-flag-shorthand" and a default value "draw" + And a context containing a targeting key with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + + Examples: v3 + | targeting_key | value | + | user-1 | heads | + | user-2 | tails | + | user-3 | heads | + + @fractional @fractional-v3 + Scenario Outline: Fractional operator invalid implicit inputs + Given a String-flag with key "fractional-flag-shorthand" and a default value "fallback" + And a context containing a key "", with type "" and with value "" + When the flag was evaluated with details + Then the resolved details value should be "draw" + Then the reason should be "DEFAULT" + + Examples: v3 + | key | type | value | + | irrelevant | String | none | + | targetingKey | Integer | 12345 | + | targetingKey | Boolean | true | From 4efde902271bc83e5a53beb99369f8aabe81592b Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 15:48:32 +0200 Subject: [PATCH 09/27] test(flagd): port the testbed's fractional v2 scenarios The @fractional-v2 examples, the single-entry and the hash edge-case scenarios, gated by their version tags. v1 stays out: no backend here implements it. Signed-off-by: Simon Schrottner --- .../resources/extensions/fractional.feature | 69 ++++++++++++++++++- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/providers/flagd/src/test/resources/extensions/fractional.feature b/providers/flagd/src/test/resources/extensions/fractional.feature index 5540f636e8..d42ac78152 100644 --- a/providers/flagd/src/test/resources/extensions/fractional.feature +++ b/providers/flagd/src/test/resources/extensions/fractional.feature @@ -2,9 +2,12 @@ Feature: Fractional targeting # Ported from the flagd-testbed's gherkin/targeting.feature: every @fractional scenario, plus the - # fractional @operator-errors ones. The @fractional-v1 and @fractional-v2 bucketing variants are - # left out, as the legacy e2e suites exclude them. The TCK's own @targeting scenarios do not cover - # the fractional operator. + # fractional @operator-errors ones. The TCK's own @targeting scenarios do not cover the + # fractional operator. + # + # @fractional-vN tags select the bucketing version (FlagdSteps skips the other versions; v3 by + # default, -Dflagd.tck.fractional=v2 to override). Untagged scenarios must hold for every version. + # The @fractional-v1 blocks are left out: no backend here implements v1 bucketing. # # Where the testbed's context values are Objects or integers outside 32 bits, the TCK's context # step cannot convert them, so those outlines use the table step with the testbed's conversion. @@ -20,6 +23,15 @@ Feature: Fractional targeting When the flag was evaluated with details Then the resolved details value should be "" + @fractional-v2 + Examples: v2 + | name | value | + | jack | hearts | + | queen | spades | + | ten | clubs | + | nine | diamonds | + | 3 | clubs | + @fractional-v3 Examples: v3 | name | value | @@ -36,12 +48,25 @@ Feature: Fractional targeting When the flag was evaluated with details Then the resolved details value should be "" + @fractional-v2 + Examples: v2 + | targeting key | value | + | jon@company.com | heads | + | jane@company.com | tails | + @fractional-v3 Examples: v3 | targeting key | value | | jon@company.com | tails | | jane@company.com | tails | + @fractional @fractional-v2 @fractional-single-entry + Scenario: Fractional operator with single entry always resolves to the only variant + Given a String-flag with key "fractional-single-entry-flag" and a default value "fallback" + And a context containing a targeting key with value "some-targeting-key" + When the flag was evaluated with details + Then the resolved details value should be "single" + @fractional Scenario Outline: Fractional operator with shared seed Given a String-flag with key "fractional-flag-A-shared-seed" and a default value "fallback" @@ -49,6 +74,14 @@ Feature: Fractional targeting When the flag was evaluated with details Then the resolved details value should be "" + @fractional-v2 + Examples: v2 + | name | value | + | seven | hearts | + | eight | diamonds | + | nine | clubs | + | two | spades | + @fractional-v3 Examples: v3 | name | value | @@ -64,6 +97,14 @@ Feature: Fractional targeting When the flag was evaluated with details Then the resolved details value should be "" + @fractional-v2 + Examples: v2 + | name | value | + | seven | ace-of-hearts | + | eight | ace-of-diamonds | + | nine | ace-of-clubs | + | two | ace-of-spades | + @fractional-v3 Examples: v3 | name | value | @@ -72,6 +113,28 @@ Feature: Fractional targeting | nine | ace-of-diamonds | | two | ace-of-diamonds | + # Hash edge-case vectors — keys chosen by brute-force search so their + # MurmurHash3-x86-32 (seed=0) falls at the six critical boundary values. + # All keys are exact 6-char MurmurHash3-x86-32 (seed=0) preimages found by exhaustive search. + # ejOoVL → hash=0 EXACT → bv(100)=0 → lower + # bY9fO- → hash=1 EXACT → bv(100)=0 → lower + # SI7p- → hash=2147483647 EXACT i32::MAX → bv(100)=49 → lower + # 6LvT0 → hash=2147483648 EXACT i32::MIN u32 → bv(100)=50 → upper + # ceQdGm → hash=4294967295 EXACT u32::MAX → bv(100)=99 → upper + @fractional @fractional-v2 + Scenario Outline: Fractional operator hash edge cases + Given a String-flag with key "fractional-hash-edge-flag" and a default value "fallback" + And a context containing a targeting key with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + Examples: + | key | value | + | ejOoVL | lower | + | bY9fO- | lower | + | SI7p- | lower | + | 6LvT0 | upper | + | ceQdGm | upper | + # Nested JSON Logic expressions as bucket variant names. # Requires providers to support the @fractional-nested feature. # Use -t "not @fractional-nested" to exclude during transition. From 4ad21403dfa5e9953952bf8750f8548ffd35636c Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 16:00:28 +0200 Subject: [PATCH 10/27] style(flagd): apply spotless to the fractional version gate Signed-off-by: Simon Schrottner --- .../src/test/java/openfeature/tck/extensions/FlagdSteps.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java index 7fdac6d863..fdbfebd2c7 100644 --- a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java +++ b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java @@ -99,9 +99,8 @@ public void gateOnResolver(Scenario scenario) { * {@link FlagdTckHarness#fractionalVersion()}. A scenario with no version tag runs regardless. */ private void gateOnFractionalVersion(Collection tags) { - List versions = tags.stream() - .filter(tag -> tag.matches("@fractional-v\\d+")) - .collect(Collectors.toList()); + List versions = + tags.stream().filter(tag -> tag.matches("@fractional-v\\d+")).collect(Collectors.toList()); String wanted = "@fractional-" + flagd.fractionalVersion(); if (!versions.isEmpty() && !versions.contains(wanted)) { throw new TestAbortedException("Skipped: scenario expects fractional " + versions From 57e45ab340d40329cd008e434a2e1ce65d423e2c Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 16:33:08 +0200 Subject: [PATCH 11/27] test(flagd): require a resolver tag on every extension scenario An extension scenario must carry @rpc, @in-process or @file, on the scenario or its feature; one with none fails instead of running in every suite. Restores @file where the flagd-testbed has it. Uncommitted work from the #1951 session, committed as is. Signed-off-by: Simon Schrottner --- .../tck/extensions/FlagdSteps.java | 42 ++++++++++++++----- .../resources/extensions/connection.feature | 10 ++--- .../resources/extensions/evaluation.feature | 2 +- .../test/resources/extensions/events.feature | 2 +- .../resources/extensions/fractional.feature | 2 +- .../resources/extensions/metadata.feature | 2 +- .../resources/extensions/targeting.feature | 2 +- 7 files changed, 41 insertions(+), 21 deletions(-) diff --git a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java index fdbfebd2c7..517f13f479 100644 --- a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java +++ b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java @@ -7,6 +7,7 @@ import dev.openfeature.contrib.providers.flagd.FlagdOptions; import dev.openfeature.contrib.providers.flagd.resolver.rpc.cache.CacheType; import dev.openfeature.contrib.providers.flagd.tck.FlagdTckHarness; +import dev.openfeature.contrib.tools.tck.ProviderTck; import dev.openfeature.contrib.tools.tck.ProviderTckHarness; import dev.openfeature.contrib.tools.tck.TckState; import dev.openfeature.sdk.MutableContext; @@ -27,6 +28,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumMap; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -69,27 +71,45 @@ public FlagdSteps(TckState state, ProviderTckHarness harness) { this.flagd = (FlagdTckHarness) harness; } + /** The tag that includes a scenario in each resolver's suite. */ + private static final Map RESOLVER_TAGS; + + static { + Map tags = new EnumMap<>(Config.Resolver.class); + tags.put(Config.Resolver.RPC, "@rpc"); + tags.put(Config.Resolver.IN_PROCESS, "@in-process"); + tags.put(Config.Resolver.FILE, "@file"); + RESOLVER_TAGS = Collections.unmodifiableMap(tags); + } + /** - * Skips a scenario tagged for the other resolver, and starts every scenario with the adoption's - * own options. + * Runs an extension scenario only in the suites whose resolver tag it carries, and starts every + * scenario with the adoption's own options. + * + *

Every extension scenario must carry at least one of {@code @rpc}, {@code @in-process} and + * {@code @file}, on the scenario or its feature. One with none of them fails rather than running + * everywhere: the resolver tag is the include, as in the flagd-testbed. Canonical scenarios carry + * no resolver tag and run in every suite. * *

Runs after the TCK's capability gate (order 0) and before it prepares the backend (order - * 10). A scenario tagged with neither {@code @rpc} nor {@code @in-process} runs in both. + * 10). * * @param scenario the scenario about to run */ @Before(order = 1) public void gateOnResolver(Scenario scenario) { flagd.resetCustomizations(); + if (!scenario.getUri().toString().contains("/" + ProviderTck.EXTENSIONS + "/")) { + return; + } Collection tags = scenario.getSourceTagNames(); - boolean rpc = tags.contains("@rpc"); - boolean inProcess = tags.contains("@in-process"); - if (rpc || inProcess) { - boolean matches = flagd.resolver() == Config.Resolver.RPC ? rpc : inProcess; - if (!matches) { - throw new TestAbortedException( - "Skipped: scenario is not tagged for the " + flagd.resolver() + " resolver"); - } + if (RESOLVER_TAGS.values().stream().noneMatch(tags::contains)) { + throw new IllegalStateException("Extension scenario '" + scenario.getName() + "' (" + scenario.getUri() + + ") carries none of " + RESOLVER_TAGS.values() + "; tag it or its feature"); + } + String wanted = RESOLVER_TAGS.get(flagd.resolver()); + if (!tags.contains(wanted)) { + throw new TestAbortedException("Skipped: scenario is not tagged " + wanted); } gateOnFractionalVersion(tags); } diff --git a/providers/flagd/src/test/resources/extensions/connection.feature b/providers/flagd/src/test/resources/extensions/connection.feature index 5932c9d437..14be1fa34d 100644 --- a/providers/flagd/src/test/resources/extensions/connection.feature +++ b/providers/flagd/src/test/resources/extensions/connection.feature @@ -16,7 +16,7 @@ Feature: flagd provider disconnect and reconnect # @targetURI and @forbidden connect through the envoy sidecar that # tools/flagd-testbed/docker-compose.flagd.yaml adds to the shared stack. - @customCert + @rpc @in-process @file @customCert Scenario: Connection with a custom certificate Given the flagd backend serves the "ssl" configuration And an option "tls" of type "Boolean" with value "true" @@ -26,7 +26,7 @@ Feature: flagd provider disconnect and reconnect When the flag was evaluated with details Then the resolved details value should be "true" - @reconnect + @rpc @in-process @file @reconnect Scenario: Provider reconnection Given an option "retryGracePeriod" of type "Integer" with value "5" And an option "retryBackoffMaxMs" of type "Integer" with value "500" @@ -38,7 +38,7 @@ Feature: flagd provider disconnect and reconnect Then the error event handler should have been executed Then the ready event handler should have been executed - @reconnect @customCert + @rpc @in-process @file @reconnect @customCert Scenario: Provider reconnection with a custom certificate Given the flagd backend serves the "ssl" configuration And an option "tls" of type "Boolean" with value "true" @@ -53,7 +53,7 @@ Feature: flagd provider disconnect and reconnect Then the error event handler should have been executed Then the ready event handler should have been executed - @unavailable + @rpc @in-process @file @unavailable Scenario: Provider unavailable Given an option "deadlineMs" of type "Integer" with value "1000" And a unavailable provider @@ -62,7 +62,7 @@ Feature: flagd provider disconnect and reconnect # A forbidden response from flagd results in a fatal client state. "a unavailable provider" is the # TCK's flavour that does not wait for readiness; the step before it points that provider at envoy. - @forbidden + @rpc @in-process @forbidden Scenario: Provider forbidden Given an option "fatalStatusCodes" of type "StringList" with value "PERMISSION_DENIED" And the provider connects to envoy's forbidden listener diff --git a/providers/flagd/src/test/resources/extensions/evaluation.feature b/providers/flagd/src/test/resources/extensions/evaluation.feature index bf05fcb5c3..418558ba5a 100644 --- a/providers/flagd/src/test/resources/extensions/evaluation.feature +++ b/providers/flagd/src/test/resources/extensions/evaluation.feature @@ -1,4 +1,4 @@ -@ext-evaluation @rpc @in-process +@ext-evaluation @rpc @in-process @file Feature: flagd evaluations # Ported from flagd-testbed gherkin/evaluation.feature. Asks what the TCK's evaluation.feature diff --git a/providers/flagd/src/test/resources/extensions/events.feature b/providers/flagd/src/test/resources/extensions/events.feature index 5602ed4e32..9c59f83977 100644 --- a/providers/flagd/src/test/resources/extensions/events.feature +++ b/providers/flagd/src/test/resources/extensions/events.feature @@ -1,4 +1,4 @@ -@ext-connection @events +@ext-connection @rpc @in-process @file @events Feature: flagd provider state changes # Ported from flagd-testbed gherkin/events.feature. Not ported: "Flag change event", which the diff --git a/providers/flagd/src/test/resources/extensions/fractional.feature b/providers/flagd/src/test/resources/extensions/fractional.feature index d42ac78152..9f7f6d5916 100644 --- a/providers/flagd/src/test/resources/extensions/fractional.feature +++ b/providers/flagd/src/test/resources/extensions/fractional.feature @@ -1,4 +1,4 @@ -@ext-fractional @rpc @in-process +@ext-fractional @rpc @in-process @file Feature: Fractional targeting # Ported from the flagd-testbed's gherkin/targeting.feature: every @fractional scenario, plus the diff --git a/providers/flagd/src/test/resources/extensions/metadata.feature b/providers/flagd/src/test/resources/extensions/metadata.feature index d584fbca4b..8d6c11d758 100644 --- a/providers/flagd/src/test/resources/extensions/metadata.feature +++ b/providers/flagd/src/test/resources/extensions/metadata.feature @@ -1,4 +1,4 @@ -@ext-metadata @in-process @metadata +@ext-metadata @in-process @file @metadata Feature: flag and flag set metadata # Ported from flagd-testbed gherkin/metadata.feature. The testbed's "a stable flagd provider" diff --git a/providers/flagd/src/test/resources/extensions/targeting.feature b/providers/flagd/src/test/resources/extensions/targeting.feature index 4480c61eb5..e8556b053a 100644 --- a/providers/flagd/src/test/resources/extensions/targeting.feature +++ b/providers/flagd/src/test/resources/extensions/targeting.feature @@ -1,4 +1,4 @@ -@ext-targeting @rpc @in-process +@ext-targeting @rpc @in-process @file Feature: Targeting rules # Ported from flagd-testbed's gherkin/targeting.feature: the json-evaluation scenarios the Provider From 89b573c7461c0da97a7401c229876784aafd1280 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 16:36:07 +0200 Subject: [PATCH 12/27] test(flagd): select extension scenarios with the TCK's tag annotations RpcTest and InProcessTest include their resolver tag with @TckIncludeTags and exclude @fractional-v2 with @TckExcludeTags, which replaces FlagdSteps' own resolver and fractional-version gates and the harness's resolver() and fractionalVersion() (-Dflagd.tck.fractional). FlagdSteps keeps failing an extension scenario with no resolver tag, now before the TCK's gate, which would otherwise skip it silently. Signed-off-by: Simon Schrottner --- .../flagd/tck/AbstractResolverTest.java | 1 - .../contrib/providers/flagd/tck/FileTest.java | 9 +++ .../providers/flagd/tck/FlagdTckHarness.java | 19 +----- .../providers/flagd/tck/InProcessTest.java | 4 ++ .../contrib/providers/flagd/tck/RpcTest.java | 4 ++ .../tck/extensions/FlagdSteps.java | 59 ++++--------------- .../resources/extensions/fractional.feature | 5 +- 7 files changed, 33 insertions(+), 68 deletions(-) diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java index 2ef42fd449..154592aec3 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java @@ -44,7 +44,6 @@ abstract class AbstractResolverTest extends ContainerizedProviderTckTest impleme private final List> customizations = new CopyOnWriteArrayList<>(); /** The resolver under test. */ - @Override public abstract Config.Resolver resolver(); /** diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FileTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FileTest.java index 6336d67719..7c2da53817 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FileTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FileTest.java @@ -4,6 +4,8 @@ import dev.openfeature.contrib.providers.flagd.FlagdOptions; import dev.openfeature.contrib.tools.tck.BackendEndpoint; import dev.openfeature.contrib.tools.tck.Capability; +import dev.openfeature.contrib.tools.tck.TckExcludeTags; +import dev.openfeature.contrib.tools.tck.TckIncludeTags; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; @@ -21,7 +23,14 @@ * file where this JVM can see it. The testbed's Compose file mounts {@code $FLAGS_DIR} at * {@code /flags}, where its launchpad writes {@code allFlags.json}, so naming a host directory * through {@link #composeEnvironment()} is all it takes. + * + *

Three extension exclusions: {@code @ext-connection}, whose every scenario turns on a gRPC + * connection this resolver does not have; {@code @metadata-provider}, which names a + * {@code selector-} flag file the launchpad leaves out of the combined {@code allFlags.json}; and + * {@code @fractional-v2}, as in the other two suites. */ +@TckIncludeTags("@file") +@TckExcludeTags({"@ext-connection", "@metadata-provider", "@fractional-v2"}) public class FileTest extends AbstractResolverTest { /** diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java index 954b63bd6d..bafb7fa66f 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java @@ -1,6 +1,5 @@ package dev.openfeature.contrib.providers.flagd.tck; -import dev.openfeature.contrib.providers.flagd.Config; import dev.openfeature.contrib.providers.flagd.FlagdOptions; import dev.openfeature.contrib.tools.tck.BackendEndpoint; import java.util.function.Consumer; @@ -28,27 +27,11 @@ public interface FlagdTckHarness { /** envoy's listener that answers every call with HTTP 403, gRPC {@code PERMISSION_DENIED}. */ int ENVOY_FORBIDDEN_PORT = 9212; - /** The resolver this suite runs, for gating {@code @rpc} and {@code @in-process} scenarios. */ - Config.Resolver resolver(); - - /** - * The fractional bucketing version the backend implements, as the testbed tags it - * ({@code v1}, {@code v2}, {@code v3}). A scenario tagged {@code @fractional-vN} for another - * version is skipped. - * - *

{@code v3} by default: the Java in-process evaluator and the flagd shipped with - * flagd-testbed v3.10.4 both implement it. Override with {@code -Dflagd.tck.fractional=v2} to - * run the v2 expectations, against an older image for instance. - */ - default String fractionalVersion() { - return System.getProperty("flagd.tck.fractional", "v3"); - } - /** * Records a change to the options of the next provider this scenario creates. * *

Applied after the adoption's defaults, in the order recorded, so a scenario can override - * any of them. Cleared before every scenario. + * any of them. Cleared after every scenario. * * @param customization a change to the options builder */ diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/InProcessTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/InProcessTest.java index b3f1dcc71d..105cd12c2c 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/InProcessTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/InProcessTest.java @@ -1,8 +1,12 @@ package dev.openfeature.contrib.providers.flagd.tck; import dev.openfeature.contrib.providers.flagd.Config; +import dev.openfeature.contrib.tools.tck.TckExcludeTags; +import dev.openfeature.contrib.tools.tck.TckIncludeTags; /** Runs the OpenFeature Provider TCK against the flagd provider in in-process mode. */ +@TckIncludeTags("@in-process") +@TckExcludeTags("@fractional-v2") public class InProcessTest extends AbstractResolverTest { /** diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/RpcTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/RpcTest.java index 4283d6058c..814b8eb75a 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/RpcTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/RpcTest.java @@ -1,8 +1,12 @@ package dev.openfeature.contrib.providers.flagd.tck; import dev.openfeature.contrib.providers.flagd.Config; +import dev.openfeature.contrib.tools.tck.TckExcludeTags; +import dev.openfeature.contrib.tools.tck.TckIncludeTags; /** Runs the OpenFeature Provider TCK against the flagd provider in RPC mode. */ +@TckIncludeTags("@rpc") +@TckExcludeTags("@fractional-v2") public class RpcTest extends AbstractResolverTest { /** diff --git a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java index 517f13f479..555059b8ac 100644 --- a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java +++ b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java @@ -3,7 +3,6 @@ import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.databind.ObjectMapper; -import dev.openfeature.contrib.providers.flagd.Config; import dev.openfeature.contrib.providers.flagd.FlagdOptions; import dev.openfeature.contrib.providers.flagd.resolver.rpc.cache.CacheType; import dev.openfeature.contrib.providers.flagd.tck.FlagdTckHarness; @@ -28,15 +27,13 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.EnumMap; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.opentest4j.TestAbortedException; /** - * Shared steps for the flagd extension scenarios: resolver gating, provider options and the + * Shared steps for the flagd extension scenarios: the resolver-tag check, provider options and the * launchpad endpoints the canonical {@code BackendControl} does not cover. * *

The features port flagd-testbed scenarios the TCK does not ask. They keep the testbed's @@ -71,60 +68,28 @@ public FlagdSteps(TckState state, ProviderTckHarness harness) { this.flagd = (FlagdTckHarness) harness; } - /** The tag that includes a scenario in each resolver's suite. */ - private static final Map RESOLVER_TAGS; - - static { - Map tags = new EnumMap<>(Config.Resolver.class); - tags.put(Config.Resolver.RPC, "@rpc"); - tags.put(Config.Resolver.IN_PROCESS, "@in-process"); - tags.put(Config.Resolver.FILE, "@file"); - RESOLVER_TAGS = Collections.unmodifiableMap(tags); - } + /** The resolver tags; each suite includes its own with {@code @TckIncludeTags}. */ + private static final List RESOLVER_TAGS = Arrays.asList("@rpc", "@in-process", "@file"); /** - * Runs an extension scenario only in the suites whose resolver tag it carries, and starts every - * scenario with the adoption's own options. + * Fails an extension scenario that carries none of {@code @rpc}, {@code @in-process} and + * {@code @file}, on the scenario or its feature. The resolver tag is the include, as in the + * flagd-testbed, so one without it would otherwise be skipped silently in every suite. * - *

Every extension scenario must carry at least one of {@code @rpc}, {@code @in-process} and - * {@code @file}, on the scenario or its feature. One with none of them fails rather than running - * everywhere: the resolver tag is the include, as in the flagd-testbed. Canonical scenarios carry - * no resolver tag and run in every suite. - * - *

Runs after the TCK's capability gate (order 0) and before it prepares the backend (order - * 10). + *

Runs before the TCK's gate (order 0), which applies the suites' {@code @TckIncludeTags} and + * would skip such a scenario first. Canonical scenarios carry no resolver tag and are exempt. * * @param scenario the scenario about to run */ - @Before(order = 1) - public void gateOnResolver(Scenario scenario) { - flagd.resetCustomizations(); + @Before(order = -1) + public void requireResolverTag(Scenario scenario) { if (!scenario.getUri().toString().contains("/" + ProviderTck.EXTENSIONS + "/")) { return; } Collection tags = scenario.getSourceTagNames(); - if (RESOLVER_TAGS.values().stream().noneMatch(tags::contains)) { + if (RESOLVER_TAGS.stream().noneMatch(tags::contains)) { throw new IllegalStateException("Extension scenario '" + scenario.getName() + "' (" + scenario.getUri() - + ") carries none of " + RESOLVER_TAGS.values() + "; tag it or its feature"); - } - String wanted = RESOLVER_TAGS.get(flagd.resolver()); - if (!tags.contains(wanted)) { - throw new TestAbortedException("Skipped: scenario is not tagged " + wanted); - } - gateOnFractionalVersion(tags); - } - - /** - * Skips a scenario whose {@code @fractional-vN} tags name only other bucketing versions than - * {@link FlagdTckHarness#fractionalVersion()}. A scenario with no version tag runs regardless. - */ - private void gateOnFractionalVersion(Collection tags) { - List versions = - tags.stream().filter(tag -> tag.matches("@fractional-v\\d+")).collect(Collectors.toList()); - String wanted = "@fractional-" + flagd.fractionalVersion(); - if (!versions.isEmpty() && !versions.contains(wanted)) { - throw new TestAbortedException("Skipped: scenario expects fractional " + versions - + ", the backend implements " + wanted + " (override with -Dflagd.tck.fractional)"); + + ") carries none of " + RESOLVER_TAGS + "; tag it or its feature"); } } diff --git a/providers/flagd/src/test/resources/extensions/fractional.feature b/providers/flagd/src/test/resources/extensions/fractional.feature index 9f7f6d5916..fbace4a397 100644 --- a/providers/flagd/src/test/resources/extensions/fractional.feature +++ b/providers/flagd/src/test/resources/extensions/fractional.feature @@ -5,8 +5,9 @@ Feature: Fractional targeting # fractional @operator-errors ones. The TCK's own @targeting scenarios do not cover the # fractional operator. # - # @fractional-vN tags select the bucketing version (FlagdSteps skips the other versions; v3 by - # default, -Dflagd.tck.fractional=v2 to override). Untagged scenarios must hold for every version. + # @fractional-vN tags select the bucketing version. The suites run v3, which the Java in-process + # evaluator and the testbed's flagd implement, and exclude @fractional-v2 with @TckExcludeTags. + # Untagged scenarios must hold for every version. # The @fractional-v1 blocks are left out: no backend here implements v1 bucketing. # # Where the testbed's context values are Objects or integers outside 32 bits, the TCK's context From 1bc82665a9705ad5314256fe19c427297a4b61b0 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 16:42:00 +0200 Subject: [PATCH 13/27] test(flagd): drop the resolver-tag check Every extension scenario carries a resolver tag; nothing needs to check it. Signed-off-by: Simon Schrottner --- .../tck/extensions/FlagdSteps.java | 34 ++----------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java index 555059b8ac..e081a0e028 100644 --- a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java +++ b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java @@ -6,15 +6,12 @@ import dev.openfeature.contrib.providers.flagd.FlagdOptions; import dev.openfeature.contrib.providers.flagd.resolver.rpc.cache.CacheType; import dev.openfeature.contrib.providers.flagd.tck.FlagdTckHarness; -import dev.openfeature.contrib.tools.tck.ProviderTck; import dev.openfeature.contrib.tools.tck.ProviderTckHarness; import dev.openfeature.contrib.tools.tck.TckState; import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.Value; import io.cucumber.datatable.DataTable; import io.cucumber.java.After; -import io.cucumber.java.Before; -import io.cucumber.java.Scenario; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; @@ -25,16 +22,14 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.stream.Collectors; /** - * Shared steps for the flagd extension scenarios: the resolver-tag check, provider options and the - * launchpad endpoints the canonical {@code BackendControl} does not cover. + * Shared steps for the flagd extension scenarios: provider options and the launchpad endpoints the + * canonical {@code BackendControl} does not cover. * *

The features port flagd-testbed scenarios the TCK does not ask. They keep the testbed's * wording wherever it does not collide with a TCK step. It collides on the provider: the TCK's @@ -68,31 +63,6 @@ public FlagdSteps(TckState state, ProviderTckHarness harness) { this.flagd = (FlagdTckHarness) harness; } - /** The resolver tags; each suite includes its own with {@code @TckIncludeTags}. */ - private static final List RESOLVER_TAGS = Arrays.asList("@rpc", "@in-process", "@file"); - - /** - * Fails an extension scenario that carries none of {@code @rpc}, {@code @in-process} and - * {@code @file}, on the scenario or its feature. The resolver tag is the include, as in the - * flagd-testbed, so one without it would otherwise be skipped silently in every suite. - * - *

Runs before the TCK's gate (order 0), which applies the suites' {@code @TckIncludeTags} and - * would skip such a scenario first. Canonical scenarios carry no resolver tag and are exempt. - * - * @param scenario the scenario about to run - */ - @Before(order = -1) - public void requireResolverTag(Scenario scenario) { - if (!scenario.getUri().toString().contains("/" + ProviderTck.EXTENSIONS + "/")) { - return; - } - Collection tags = scenario.getSourceTagNames(); - if (RESOLVER_TAGS.stream().noneMatch(tags::contains)) { - throw new IllegalStateException("Extension scenario '" + scenario.getName() + "' (" + scenario.getUri() - + ") carries none of " + RESOLVER_TAGS + "; tag it or its feature"); - } - } - /** * Leaves nothing behind for a canonical scenario that follows: no customization, and flagd back * on the {@code default} configuration. The TCK's own preparation may only reset flag state, From fe6f0a2cbfcd6bde1b3477224f35e627f0006ce0 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 17:31:28 +0200 Subject: [PATCH 14/27] test(flagd): run the TCK against the flagd-testbed's own compose file The suites use test-harness/docker-compose.yaml, which the legacy e2e suites already run, instead of a copy under tools/flagd-testbed: the image tag follows the submodule and the envoy listeners are the testbed's. The tck profile checks out the submodule and points the testbed's FLAGS_DIR bind mount into target. Drops tools/flagd-testbed; the OFREP adoption moves to the same file. Signed-off-by: Simon Schrottner --- providers/flagd/pom.xml | 40 +++++++++++++++++++ .../flagd/tck/AbstractResolverTest.java | 5 ++- .../resources/extensions/connection.feature | 4 +- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/providers/flagd/pom.xml b/providers/flagd/pom.xml index daf72975a4..8626886955 100644 --- a/providers/flagd/pom.xml +++ b/providers/flagd/pom.xml @@ -231,6 +231,46 @@ + + + tck + + + + org.codehaus.mojo + exec-maven-plugin + + + update-test-harness-submodule-tck + initialize + + exec + + + git + + submodule + update + --init + test-harness + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + ${project.build.directory}/tck-flags + + + + + + codequality diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java index 154592aec3..4b8d24d6eb 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java @@ -58,7 +58,10 @@ abstract class AbstractResolverTest extends ContainerizedProviderTckTest impleme * {@inheritDoc} * *

The flagd-testbed's own stack, from the {@code test-harness} submodule the legacy e2e suites - * also run, so the image tag follows the submodule pin instead of a second copy that can drift. + * also run, so the image tag follows the submodule. It adds an envoy sidecar, which the + * {@code @targetURI} and {@code @forbidden} extension scenarios connect through; no canonical + * scenario reaches it. Its {@code FLAGS_DIR} bind mount is pointed into {@code target} by the + * {@code tck} profile in this module's pom. */ @Override public File composeFile() { diff --git a/providers/flagd/src/test/resources/extensions/connection.feature b/providers/flagd/src/test/resources/extensions/connection.feature index 14be1fa34d..55f2fad7e4 100644 --- a/providers/flagd/src/test/resources/extensions/connection.feature +++ b/providers/flagd/src/test/resources/extensions/connection.feature @@ -13,8 +13,8 @@ Feature: flagd provider disconnect and reconnect # here is a set of steps rather than a value. Reconnect timing is the testbed's "stable flagd # provider", as in events.feature. # - # @targetURI and @forbidden connect through the envoy sidecar that - # tools/flagd-testbed/docker-compose.flagd.yaml adds to the shared stack. + # @targetURI and @forbidden connect through the envoy sidecar in the testbed's + # test-harness/docker-compose.yaml. @rpc @in-process @file @customCert Scenario: Connection with a custom certificate From d6d9c87c2762d3fdea0b4b36084947245472f2ca Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 18:20:46 +0200 Subject: [PATCH 15/27] test(flagd): drop the TCK profile's submodule checkout The test-harness submodule is already checked out with the repository. Signed-off-by: Simon Schrottner --- providers/flagd/pom.xml | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/providers/flagd/pom.xml b/providers/flagd/pom.xml index 8626886955..9037e5b4f0 100644 --- a/providers/flagd/pom.xml +++ b/providers/flagd/pom.xml @@ -232,32 +232,10 @@ - + tck - - org.codehaus.mojo - exec-maven-plugin - - - update-test-harness-submodule-tck - initialize - - exec - - - git - - submodule - update - --init - test-harness - - - - - org.apache.maven.plugins maven-surefire-plugin From 3780a01095808c92257ad65f76e70e3bf5319d60 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 18:28:40 +0200 Subject: [PATCH 16/27] test(flagd): drop the TCK profile Without FLAGS_DIR the testbed mounts its own checkout at /flags, and the launchpad removes the flag file it writes there on shutdown. Signed-off-by: Simon Schrottner --- providers/flagd/pom.xml | 18 ------------------ .../flagd/tck/AbstractResolverTest.java | 3 +-- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/providers/flagd/pom.xml b/providers/flagd/pom.xml index 9037e5b4f0..daf72975a4 100644 --- a/providers/flagd/pom.xml +++ b/providers/flagd/pom.xml @@ -231,24 +231,6 @@ - - - tck - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - ${project.build.directory}/tck-flags - - - - - - codequality diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java index 4b8d24d6eb..33c7e34735 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java @@ -60,8 +60,7 @@ abstract class AbstractResolverTest extends ContainerizedProviderTckTest impleme *

The flagd-testbed's own stack, from the {@code test-harness} submodule the legacy e2e suites * also run, so the image tag follows the submodule. It adds an envoy sidecar, which the * {@code @targetURI} and {@code @forbidden} extension scenarios connect through; no canonical - * scenario reaches it. Its {@code FLAGS_DIR} bind mount is pointed into {@code target} by the - * {@code tck} profile in this module's pom. + * scenario reaches it. */ @Override public File composeFile() { From a130054f5cbab8a228873020a90744a1ddd7f233 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Wed, 21 Oct 2026 21:18:09 +0200 Subject: [PATCH 17/27] test(flagd): use the TCK's steps instead of flagd-shaped copies Four of this suite's five steps said nothing about flagd. Two of them never touched a backend, and two spoke the control API's own operations through a vendor-named step: the launchpad FlagdSteps posted to is the control API the TCK already drives through BackendControl. So the feature files say "the backend serves the ... configuration" now, and FlagdSteps keeps only the provider options, whose names are flagd's. The @After no longer restores the default configuration either -- BackendControl records what a scenario served and takes the next scenario off it. Signed-off-by: Simon Schrottner --- .../tck/extensions/FlagdSteps.java | 102 ++---------------- .../resources/extensions/connection.feature | 4 +- .../resources/extensions/metadata.feature | 4 +- .../resources/extensions/sync-payload.feature | 6 +- 4 files changed, 17 insertions(+), 99 deletions(-) diff --git a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java index e081a0e028..bd94f94ee3 100644 --- a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java +++ b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java @@ -1,26 +1,16 @@ package openfeature.tck.extensions; -import static org.assertj.core.api.Assertions.assertThat; import com.fasterxml.jackson.databind.ObjectMapper; import dev.openfeature.contrib.providers.flagd.FlagdOptions; import dev.openfeature.contrib.providers.flagd.resolver.rpc.cache.CacheType; import dev.openfeature.contrib.providers.flagd.tck.FlagdTckHarness; import dev.openfeature.contrib.tools.tck.ProviderTckHarness; -import dev.openfeature.contrib.tools.tck.TckState; -import dev.openfeature.sdk.MutableContext; import dev.openfeature.sdk.Value; -import io.cucumber.datatable.DataTable; import io.cucumber.java.After; import io.cucumber.java.en.Given; -import io.cucumber.java.en.Then; -import io.cucumber.java.en.When; import java.io.IOException; import java.lang.reflect.Method; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -28,8 +18,12 @@ import java.util.stream.Collectors; /** - * Shared steps for the flagd extension scenarios: provider options and the launchpad endpoints the - * canonical {@code BackendControl} does not cover. + * Shared steps for the flagd extension scenarios: the provider options the testbed's features set. + * + *

Everything else these features once needed from here is the TCK's now -- serving a named + * configuration, a timed outage, a context table and the negative event assertion were about + * nothing flagd-specific, and two of them were the control API's own operations behind a + * flagd-shaped step. * *

The features port flagd-testbed scenarios the TCK does not ask. They keep the testbed's * wording wherever it does not collide with a TCK step. It collides on the provider: the TCK's @@ -38,8 +32,6 @@ */ public class FlagdSteps { - private static final HttpClient HTTP = HttpClient.newHttpClient(); - /** Option names the testbed spells differently from {@link FlagdOptions.FlagdOptionsBuilder}. */ private static final Map OPTION_NAMES; @@ -52,28 +44,20 @@ public class FlagdSteps { OPTION_NAMES = Collections.unmodifiableMap(names); } - private final TckState state; private final FlagdTckHarness flagd; - /** The launchpad configuration this scenario switched to, if any. */ - private String servedConfiguration; - - public FlagdSteps(TckState state, ProviderTckHarness harness) { - this.state = state; + public FlagdSteps(ProviderTckHarness harness) { this.flagd = (FlagdTckHarness) harness; } /** - * Leaves nothing behind for a canonical scenario that follows: no customization, and flagd back - * on the {@code default} configuration. The TCK's own preparation may only reset flag state, - * which would keep a scenario's {@code ssl} or {@code metadata} configuration running. + * Leaves no customization behind for a scenario that follows. Getting flagd off a scenario's own + * configuration is the TCK's business now: {@code BackendControl} records the one a scenario + * served and takes the next scenario back to this suite's. */ @After public void restoreDefaults() { flagd.resetCustomizations(); - if (servedConfiguration != null && !"default".equals(servedConfiguration)) { - launchpad("/start?config=default"); - } } /** @@ -100,72 +84,6 @@ public void anOption(String option, String type, String value) throws IOExceptio }); } - /** - * Restarts flagd on one of the launchpad's flag configurations ({@code default}, - * {@code metadata}, {@code ssl}, {@code sync-payload}). - * - *

Comes before {@code Given a stable provider}, so the provider connects to it. The TCK's - * {@code the connection is restored} returns to {@code default}. - * - * @param configuration the launchpad configuration name - */ - @Given("the flagd backend serves the {string} configuration") - public void theBackendServes(String configuration) { - launchpad("/start?config=" + configuration); - servedConfiguration = configuration; - } - - /** - * Stops flagd and starts it again after the given delay, without waiting for it. - * - * @param seconds how long flagd stays down - */ - @When("the connection is lost for {int}s") - public void theConnectionIsLostFor(int seconds) { - launchpad("/restart?seconds=" + seconds); - } - - /** - * Adds typed entries to the evaluation context. - * - * @param table rows with {@code key}, {@code type} and {@code value} columns - */ - @Given("a context with the following keys:") - public void aContextWithTheFollowingKeys(DataTable table) throws IOException { - Map entries = new HashMap<>(state.context.asMap()); - for (Map row : table.asMaps(String.class, String.class)) { - entries.put(row.get("key"), Value.objectToValue(convert(row.get("value"), row.get("type")))); - } - state.context = new MutableContext(state.context.getTargetingKey(), entries); - } - - /** - * Asserts that no event of the given kind was recorded during the scenario. - * - * @param eventType the event kind - */ - @Then("the {} event handler should not have been executed") - public void theEventHandlerShouldNotHaveBeenExecuted(String eventType) { - assertThat(state.events.stream().anyMatch(event -> event.type().equals(eventType))) - .as("no %s event should have fired", eventType) - .isFalse(); - } - - private void launchpad(String path) { - HttpRequest request = HttpRequest.newBuilder(URI.create(flagd.launchpadUrl() + path)) - .POST(HttpRequest.BodyPublishers.noBody()) - .build(); - try { - HttpResponse response = HTTP.send(request, HttpResponse.BodyHandlers.ofString()); - assertThat(response.statusCode()).as("launchpad POST %s", path).isEqualTo(200); - } catch (IOException e) { - throw new IllegalStateException("launchpad POST " + path + " failed", e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("interrupted during launchpad POST " + path, e); - } - } - /** The testbed's value conversion, for the types the extension features use. */ static Object convert(String value, String type) throws IOException { if ("null".equals(value) && !"String".equals(type)) { diff --git a/providers/flagd/src/test/resources/extensions/connection.feature b/providers/flagd/src/test/resources/extensions/connection.feature index 55f2fad7e4..45c78dc05e 100644 --- a/providers/flagd/src/test/resources/extensions/connection.feature +++ b/providers/flagd/src/test/resources/extensions/connection.feature @@ -18,7 +18,7 @@ Feature: flagd provider disconnect and reconnect @rpc @in-process @file @customCert Scenario: Connection with a custom certificate - Given the flagd backend serves the "ssl" configuration + Given the backend serves the "ssl" configuration And an option "tls" of type "Boolean" with value "true" And an option "certPath" of type "String" with value "test-harness/ssl/custom-root-cert.crt" And a stable provider @@ -40,7 +40,7 @@ Feature: flagd provider disconnect and reconnect @rpc @in-process @file @reconnect @customCert Scenario: Provider reconnection with a custom certificate - Given the flagd backend serves the "ssl" configuration + Given the backend serves the "ssl" configuration And an option "tls" of type "Boolean" with value "true" And an option "certPath" of type "String" with value "test-harness/ssl/custom-root-cert.crt" And an option "retryGracePeriod" of type "Integer" with value "5" diff --git a/providers/flagd/src/test/resources/extensions/metadata.feature b/providers/flagd/src/test/resources/extensions/metadata.feature index 8d6c11d758..bb6b856620 100644 --- a/providers/flagd/src/test/resources/extensions/metadata.feature +++ b/providers/flagd/src/test/resources/extensions/metadata.feature @@ -20,7 +20,7 @@ Feature: flag and flag set metadata @metadata-provider Scenario: Returns flag set metadata Given an option "selector" of type "String" with value "rawflags/selector-flag-combined-metadata.json" - And the flagd backend serves the "metadata" configuration + And the backend serves the "metadata" configuration And a stable provider And a Boolean-flag with key "only-set-metadata-flag" and a default value "true" When the flag was evaluated with details @@ -38,7 +38,7 @@ Feature: flag and flag set metadata @metadata-provider Scenario: Flag metadata overwrites flag set metadata Given an option "selector" of type "String" with value "rawflags/selector-flag-combined-metadata.json" - And the flagd backend serves the "metadata" configuration + And the backend serves the "metadata" configuration And a stable provider And a Boolean-flag with key "combined-metadata-flag" and a default value "true" When the flag was evaluated with details diff --git a/providers/flagd/src/test/resources/extensions/sync-payload.feature b/providers/flagd/src/test/resources/extensions/sync-payload.feature index 125da0ef11..2ee4691557 100644 --- a/providers/flagd/src/test/resources/extensions/sync-payload.feature +++ b/providers/flagd/src/test/resources/extensions/sync-payload.feature @@ -10,7 +10,7 @@ Feature: Disabled Sync Metadata # the provider is back within the event timeout once flagd returns after the 6s outage. Scenario: Use enriched context - Given the flagd backend serves the "sync-payload" configuration + Given the backend serves the "sync-payload" configuration And a stable provider And a String-flag with key "flagd-context-aware" and a default value "not" When the flag was evaluated with details @@ -21,7 +21,7 @@ Feature: Disabled Sync Metadata Given an option "retryGracePeriod" of type "Integer" with value "5" And an option "retryBackoffMs" of type "Integer" with value "500" And an option "retryBackoffMaxMs" of type "Integer" with value "500" - And the flagd backend serves the "sync-payload" configuration + And the backend serves the "sync-payload" configuration And a stable provider And a String-flag with key "flagd-context-aware" and a default value "not" And a stale event handler @@ -35,7 +35,7 @@ Feature: Disabled Sync Metadata When a ready event was fired Scenario: Resolve value independent of context - Given the flagd backend serves the "sync-payload" configuration + Given the backend serves the "sync-payload" configuration And a stable provider And a Boolean-flag with key "boolean-flag" and a default value "false" When the flag was evaluated with details From 5067e6f07b281adedfbf6142b1c6bf9b45fff250 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Thu, 22 Oct 2026 10:44:56 +0200 Subject: [PATCH 18/27] test(flagd): assert the outage by its events, not by its length The testbed timed every outage: lost for 8s, lost for 6s, lost for 1s. What each scenario is actually asking is a sequence -- stale, then error, then ready -- and the length was only a way of deciding whether the retry grace period would be crossed. So the chains say it directly: the connection is lost, the events that follow are awaited, the connection is restored, ready is awaited. The 1s outage is the clearest case. It asked for stale without error, by being shorter than the 5s grace period; restoring the connection as soon as stale has fired asks the same thing without a wall clock to lose a race against on a slow machine. The configuration moves onto the provider step as well -- "a stable provider with "sync-payload" configuration" -- so it cannot be stated after the provider it is supposed to configure. Signed-off-by: Simon Schrottner --- .../resources/extensions/connection.feature | 22 ++++++++++--------- .../extensions/context-enrichment.feature | 16 ++++++++------ .../test/resources/extensions/events.feature | 16 +++++++++----- .../resources/extensions/metadata.feature | 11 ++++------ .../resources/extensions/rpc-caching.feature | 3 ++- .../resources/extensions/sync-payload.feature | 20 ++++++++--------- 6 files changed, 46 insertions(+), 42 deletions(-) diff --git a/providers/flagd/src/test/resources/extensions/connection.feature b/providers/flagd/src/test/resources/extensions/connection.feature index 45c78dc05e..9cfeec235a 100644 --- a/providers/flagd/src/test/resources/extensions/connection.feature +++ b/providers/flagd/src/test/resources/extensions/connection.feature @@ -18,10 +18,9 @@ Feature: flagd provider disconnect and reconnect @rpc @in-process @file @customCert Scenario: Connection with a custom certificate - Given the backend serves the "ssl" configuration - And an option "tls" of type "Boolean" with value "true" + Given an option "tls" of type "Boolean" with value "true" And an option "certPath" of type "String" with value "test-harness/ssl/custom-root-cert.crt" - And a stable provider + And a stable provider with "ssl" configuration And a Boolean-flag with key "boolean-flag" and a default value "false" When the flag was evaluated with details Then the resolved details value should be "true" @@ -34,23 +33,24 @@ Feature: flagd provider disconnect and reconnect And a ready event handler And a error event handler When a ready event was fired - When the connection is lost for 8s + When the connection is lost Then the error event handler should have been executed + When the connection is restored Then the ready event handler should have been executed @rpc @in-process @file @reconnect @customCert Scenario: Provider reconnection with a custom certificate - Given the backend serves the "ssl" configuration - And an option "tls" of type "Boolean" with value "true" + Given an option "tls" of type "Boolean" with value "true" And an option "certPath" of type "String" with value "test-harness/ssl/custom-root-cert.crt" And an option "retryGracePeriod" of type "Integer" with value "5" And an option "retryBackoffMaxMs" of type "Integer" with value "500" - And a stable provider + And a stable provider with "ssl" configuration And a ready event handler And a error event handler When a ready event was fired - When the connection is lost for 8s + When the connection is lost Then the error event handler should have been executed + When the connection is restored Then the ready event handler should have been executed @rpc @in-process @file @unavailable @@ -96,8 +96,9 @@ Feature: flagd provider disconnect and reconnect And a ready event handler And a error event handler When a ready event was fired - When the connection is lost for 8s + When the connection is lost Then the error event handler should have been executed + When the connection is restored Then the ready event handler should have been executed @targetURI @in-process @reconnect @grace @@ -110,7 +111,8 @@ Feature: flagd provider disconnect and reconnect And a stale event handler And a error event handler When a ready event was fired - When the connection is lost for 8s + When the connection is lost Then the stale event handler should have been executed Then the error event handler should have been executed + When the connection is restored Then the ready event handler should have been executed diff --git a/providers/flagd/src/test/resources/extensions/context-enrichment.feature b/providers/flagd/src/test/resources/extensions/context-enrichment.feature index 05020513ad..fcbf2a6eb9 100644 --- a/providers/flagd/src/test/resources/extensions/context-enrichment.feature +++ b/providers/flagd/src/test/resources/extensions/context-enrichment.feature @@ -7,9 +7,9 @@ Feature: Context Enrichment # # The @grace scenarios restore the testbed provider's retry timing (grace period 5s, backoff # 500ms capped at 500ms) over the adoption's (grace 2s, backoff capped at the 5000ms default), so - # the provider is back within the event timeout once flagd returns after the 6s outage. The RPC - # one also restores the testbed's 1000ms deadline: at the adoption's 15000ms an evaluation issued - # while flagd is down could wait out the outage and succeed, which is not what it asks. + # the provider is back within the event timeout once the connection is restored. The RPC one also + # restores the testbed's 1000ms deadline: at the adoption's 15000ms an evaluation issued while + # flagd is down could outlast the outage and succeed, which is not what it asks. # # The testbed's @caching is renamed @flagd-caching: the TCK reserves @caching and refuses a # scenario that carries it. @@ -32,11 +32,12 @@ Feature: Context Enrichment And a ready event handler When the flag was evaluated with details Then the resolved details value should be "INTERNAL" - When the connection is lost for 6s + When the connection is lost And a stale event was fired When the flag was evaluated with details Then the resolved details value should be "INTERNAL" - When a ready event was fired + When the connection is restored + And a ready event was fired @grace @rpc Scenario: Use enriched context on connection error for RPC @@ -50,11 +51,12 @@ Feature: Context Enrichment And a ready event handler When the flag was evaluated with details Then the resolved details value should be "INTERNAL" - When the connection is lost for 6s + When the connection is lost And a stale event was fired When the flag was evaluated with details Then the resolved details value should be "not" - When a ready event was fired + When the connection is restored + And a ready event was fired @rpc @flagd-caching Scenario: Use enriched context on RPC connection will not cache the value diff --git a/providers/flagd/src/test/resources/extensions/events.feature b/providers/flagd/src/test/resources/extensions/events.feature index 9c59f83977..c16a45d786 100644 --- a/providers/flagd/src/test/resources/extensions/events.feature +++ b/providers/flagd/src/test/resources/extensions/events.feature @@ -5,9 +5,10 @@ Feature: flagd provider state changes # canonical events.feature asks in "A configuration change is signalled and applied", together # with the changed evaluation it leaves out. # - # The canonical @stale scenario stops flagd and starts it again on request. These chains instead - # take flagd down for a fixed time with the launchpad's /restart, so they ask what the retry grace - # period decides: an outage shorter than it ends stale -> ready, a longer one escalates to error. + # These chains ask what the retry grace period decides, and they ask it by sequence rather than by + # clock: stale -> ready when the connection returns inside the grace period, stale -> error -> ready + # when the error assertion has first waited the grace period out. The testbed timed the outage + # instead (the launchpad's /restart), which made the same question depend on a 1s or 8s wall clock. # # Each "should have been executed" drains the recorded events through the one it matched, so the # next assertion waits for a newer event and a chain asserts its order. @@ -25,8 +26,9 @@ Feature: flagd provider state changes Given a ready event handler And a error event handler Then the ready event handler should have been executed - When the connection is lost for 8s + When the connection is lost Then the error event handler should have been executed + When the connection is restored Then the ready event handler should have been executed @grace @@ -35,9 +37,10 @@ Feature: flagd provider state changes And a error event handler And a stale event handler Then the ready event handler should have been executed - When the connection is lost for 8s + When the connection is lost Then the stale event handler should have been executed Then the error event handler should have been executed + When the connection is restored Then the ready event handler should have been executed @grace @@ -46,7 +49,8 @@ Feature: flagd provider state changes And a error event handler And a stale event handler Then the ready event handler should have been executed - When the connection is lost for 1s + When the connection is lost Then the stale event handler should have been executed + When the connection is restored Then the ready event handler should have been executed with no error event before it And the error event handler should not have been executed diff --git a/providers/flagd/src/test/resources/extensions/metadata.feature b/providers/flagd/src/test/resources/extensions/metadata.feature index bb6b856620..f4ee6fe38b 100644 --- a/providers/flagd/src/test/resources/extensions/metadata.feature +++ b/providers/flagd/src/test/resources/extensions/metadata.feature @@ -2,9 +2,8 @@ Feature: flag and flag set metadata # Ported from flagd-testbed gherkin/metadata.feature. The testbed's "a stable flagd provider" - # becomes the TCK's "a stable provider", and its "a metadata flagd provider" the "metadata" - # launchpad configuration followed by "a stable provider". @file is dropped: this suite has no - # file resolver. + # becomes the TCK's "a stable provider", and its "a metadata flagd provider" becomes "a stable + # provider with "metadata" configuration". @file is dropped: this suite has no file resolver. Scenario: Returns metadata Given a stable provider @@ -20,8 +19,7 @@ Feature: flag and flag set metadata @metadata-provider Scenario: Returns flag set metadata Given an option "selector" of type "String" with value "rawflags/selector-flag-combined-metadata.json" - And the backend serves the "metadata" configuration - And a stable provider + And a stable provider with "metadata" configuration And a Boolean-flag with key "only-set-metadata-flag" and a default value "true" When the flag was evaluated with details Then the resolved metadata should contain @@ -38,8 +36,7 @@ Feature: flag and flag set metadata @metadata-provider Scenario: Flag metadata overwrites flag set metadata Given an option "selector" of type "String" with value "rawflags/selector-flag-combined-metadata.json" - And the backend serves the "metadata" configuration - And a stable provider + And a stable provider with "metadata" configuration And a Boolean-flag with key "combined-metadata-flag" and a default value "true" When the flag was evaluated with details Then the resolved metadata should contain diff --git a/providers/flagd/src/test/resources/extensions/rpc-caching.feature b/providers/flagd/src/test/resources/extensions/rpc-caching.feature index 65beb871b6..a86f5697ac 100644 --- a/providers/flagd/src/test/resources/extensions/rpc-caching.feature +++ b/providers/flagd/src/test/resources/extensions/rpc-caching.feature @@ -57,11 +57,12 @@ Feature: Flag evaluation with caching Then the reason should be "STATIC" When the flag was evaluated with details Then the reason should be "CACHED" - When the connection is lost for 6s + When the connection is lost And a stale event was fired And the flag was evaluated with details Then the reason should be "CACHED" When a error event was fired + And the connection is restored And a ready event was fired And the flag was evaluated with details Then the reason should be "STATIC" diff --git a/providers/flagd/src/test/resources/extensions/sync-payload.feature b/providers/flagd/src/test/resources/extensions/sync-payload.feature index 2ee4691557..eb5c438159 100644 --- a/providers/flagd/src/test/resources/extensions/sync-payload.feature +++ b/providers/flagd/src/test/resources/extensions/sync-payload.feature @@ -2,16 +2,15 @@ Feature: Disabled Sync Metadata # Ported from flagd-testbed gherkin/sync-payload.feature. The testbed's "a syncpayload flagd - # provider" becomes the "sync-payload" launchpad configuration followed by "a stable provider", - # moved from the Background into each scenario so the @grace one can record options first. + # provider" becomes "a stable provider with "sync-payload" configuration", moved from the + # Background into each scenario so the @grace one can record options first. # # The @grace scenario restores the testbed provider's retry timing (grace period 5s, backoff # 500ms capped at 500ms) over the adoption's (grace 2s, backoff capped at the 5000ms default), so - # the provider is back within the event timeout once flagd returns after the 6s outage. + # the provider is back within the event timeout once the connection is restored. Scenario: Use enriched context - Given the backend serves the "sync-payload" configuration - And a stable provider + Given a stable provider with "sync-payload" configuration And a String-flag with key "flagd-context-aware" and a default value "not" When the flag was evaluated with details Then the resolved details value should be "INTERNAL" @@ -21,22 +20,21 @@ Feature: Disabled Sync Metadata Given an option "retryGracePeriod" of type "Integer" with value "5" And an option "retryBackoffMs" of type "Integer" with value "500" And an option "retryBackoffMaxMs" of type "Integer" with value "500" - And the backend serves the "sync-payload" configuration - And a stable provider + And a stable provider with "sync-payload" configuration And a String-flag with key "flagd-context-aware" and a default value "not" And a stale event handler And a ready event handler When the flag was evaluated with details Then the resolved details value should be "INTERNAL" - When the connection is lost for 6s + When the connection is lost And a stale event was fired When the flag was evaluated with details Then the resolved details value should be "INTERNAL" - When a ready event was fired + When the connection is restored + And a ready event was fired Scenario: Resolve value independent of context - Given the backend serves the "sync-payload" configuration - And a stable provider + Given a stable provider with "sync-payload" configuration And a Boolean-flag with key "boolean-flag" and a default value "false" When the flag was evaluated with details Then the resolved details value should be "true" From 0ba02f261b19290e0143d08b755826b911790f8b Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Thu, 22 Oct 2026 10:52:42 +0200 Subject: [PATCH 19/27] test(flagd): hand the event-order assertion back to the TCK ConnectionSteps had written "the {} event handler should have been executed with no {} event before it" locally, and its javadoc explained why: the TCK's await drained the records ahead of its match, so an error recorded before an awaited ready was gone and "should not have been executed" passed vacuously. The TCK keeps those records and asks the question itself now, so the local copy goes, along with the redundant negative assertion the chain carried after it. What stays here is flagd's own: the envoy target URI and the forbidden listener. Signed-off-by: Simon Schrottner --- .../tck/extensions/ConnectionSteps.java | 61 ++----------------- .../test/resources/extensions/events.feature | 6 +- 2 files changed, 9 insertions(+), 58 deletions(-) diff --git a/providers/flagd/src/test/java/openfeature/tck/extensions/ConnectionSteps.java b/providers/flagd/src/test/java/openfeature/tck/extensions/ConnectionSteps.java index 825f92fe0e..653c41c84a 100644 --- a/providers/flagd/src/test/java/openfeature/tck/extensions/ConnectionSteps.java +++ b/providers/flagd/src/test/java/openfeature/tck/extensions/ConnectionSteps.java @@ -1,44 +1,26 @@ package openfeature.tck.extensions; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; - import dev.openfeature.contrib.providers.flagd.tck.FlagdTckHarness; import dev.openfeature.contrib.tools.tck.BackendEndpoint; -import dev.openfeature.contrib.tools.tck.ProviderEventRecord; import dev.openfeature.contrib.tools.tck.ProviderTckHarness; -import dev.openfeature.contrib.tools.tck.TckState; import io.cucumber.java.en.Given; -import io.cucumber.java.en.Then; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; /** - * Steps for the connection, event-chain and caching extension scenarios: pointing the provider at - * the envoy sidecar, and asserting that one event arrived without another before it. + * Steps for the connection extension scenarios: pointing the provider at the envoy sidecar. * - *

Event ordering needs nothing more than that. The TCK's {@code the {} event handler should have - * been executed} drains the recorded events through the one it matched, so each assertion in a - * chain waits for an event newer than the previous one. What draining cannot express is an - * absence between two events: once {@code ready} is matched, an {@code error} recorded before it - * has been discarded, and {@code the error event handler should not have been executed} passes - * vacuously. {@link #executedWithNoneBefore} asserts the absence before it drains. + *

Event ordering used to live here too, because the TCK's await discarded the records ahead of + * its match and an {@code error} recorded before an awaited {@code ready} was gone by the time + * anything could object to it. The TCK keeps its own records now and asks the question itself, with + * {@code the {} event handler should have been executed with no {} event before it}. */ public class ConnectionSteps { /** A port nothing listens on, so a provider given a target URI cannot fall back to host and port. */ private static final int UNAVAILABLE_PORT = 9999; - private final TckState state; - private final ProviderTckHarness harness; private final FlagdTckHarness flagd; - public ConnectionSteps(TckState state, ProviderTckHarness harness) { - this.state = state; - this.harness = harness; + public ConnectionSteps(ProviderTckHarness harness) { this.flagd = (FlagdTckHarness) harness; } @@ -76,35 +58,4 @@ public void theProviderConnectsToTheForbiddenListener() { flagd.customize(builder -> builder.host(host).port(port)); } - /** - * Awaits an event of one kind, asserts that no event of another kind was recorded before it, - * and then drains through it as the TCK's own await does. - * - * @param eventType the event kind to await - * @param forbiddenType the event kind that must not precede it - */ - @Then("the {} event handler should have been executed with no {} event before it") - public void executedWithNoneBefore(String eventType, String forbiddenType) { - await().alias("provider event " + eventType) - .atMost(harness.eventTimeout().toMillis(), MILLISECONDS) - .pollInterval(10, MILLISECONDS) - .until(() -> - state.events.stream().anyMatch(event -> event.type().equals(eventType))); - - List before = new ArrayList<>(); - ProviderEventRecord matched = null; - while (!state.events.isEmpty()) { - ProviderEventRecord head = state.events.poll(); - if (head.type().equals(eventType)) { - matched = head; - break; - } - before.add(head); - } - state.lastEvent = Optional.ofNullable(matched); - - assertThat(before.stream().map(ProviderEventRecord::type).collect(Collectors.toList())) - .as("events recorded before the %s event", eventType) - .doesNotContain(forbiddenType); - } } diff --git a/providers/flagd/src/test/resources/extensions/events.feature b/providers/flagd/src/test/resources/extensions/events.feature index c16a45d786..e37cd76f7d 100644 --- a/providers/flagd/src/test/resources/extensions/events.feature +++ b/providers/flagd/src/test/resources/extensions/events.feature @@ -10,8 +10,9 @@ Feature: flagd provider state changes # when the error assertion has first waited the grace period out. The testbed timed the outage # instead (the launchpad's /restart), which made the same question depend on a 1s or 8s wall clock. # - # Each "should have been executed" drains the recorded events through the one it matched, so the - # next assertion waits for a newer event and a chain asserts its order. + # Each "should have been executed" claims the record it matched and leaves the rest, so the next + # assertion waits for a newer event of its own kind while an event of another kind stays visible + # to whatever asks about it later. # # Timing is the testbed's "stable flagd provider" except for the deadline, which stays at the # adoption's 15000ms for the in-process sync (see AbstractResolverTest): a 5s grace period, and a @@ -53,4 +54,3 @@ Feature: flagd provider state changes Then the stale event handler should have been executed When the connection is restored Then the ready event handler should have been executed with no error event before it - And the error event handler should not have been executed From f515bdb5b838a5e3f52b187422f753d5e48ca6e5 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Thu, 22 Oct 2026 11:12:00 +0200 Subject: [PATCH 20/27] test(flagd): hand the flag-metadata assertions back to the TCK MetadataSteps held "the resolved metadata should contain" and "the resolved metadata is empty" -- the specification's own wording, down to the metadata_type column, for behaviour the specification defines rather than flagd. The TCK asks them now, so the local copies would be duplicate step definitions. Nothing of this file was flagd's: it read the last evaluation's flag metadata through the SDK and compared typed entries. What remains in this package is the provider options and the envoy routing. Signed-off-by: Simon Schrottner --- .../tck/extensions/MetadataSteps.java | 74 ------------------- 1 file changed, 74 deletions(-) delete mode 100644 providers/flagd/src/test/java/openfeature/tck/extensions/MetadataSteps.java diff --git a/providers/flagd/src/test/java/openfeature/tck/extensions/MetadataSteps.java b/providers/flagd/src/test/java/openfeature/tck/extensions/MetadataSteps.java deleted file mode 100644 index 8e17d90e41..0000000000 --- a/providers/flagd/src/test/java/openfeature/tck/extensions/MetadataSteps.java +++ /dev/null @@ -1,74 +0,0 @@ -package openfeature.tck.extensions; - -import static org.assertj.core.api.Assertions.assertThat; - -import dev.openfeature.contrib.tools.tck.TckState; -import dev.openfeature.sdk.ImmutableMetadata; -import io.cucumber.datatable.DataTable; -import io.cucumber.java.en.Then; -import java.io.IOException; -import java.util.Map; - -/** - * Assertions on the flag metadata of the last evaluation, for the scenarios ported from the - * flagd-testbed's {@code metadata.feature}. - */ -public class MetadataSteps { - - private final TckState state; - - public MetadataSteps(TckState state) { - this.state = state; - } - - /** - * Asserts typed metadata entries on the last evaluation. Entries not listed are not checked. - * - * @param table rows with {@code key}, {@code metadata_type} and {@code value} columns - */ - @Then("the resolved metadata should contain") - public void theResolvedMetadataShouldContain(DataTable table) throws IOException { - ImmutableMetadata metadata = requireMetadata(); - for (Map row : table.asMaps(String.class, String.class)) { - String key = row.get("key"); - String type = row.get("metadata_type"); - Object expected = FlagdSteps.convert(row.get("value"), type); - Object actual; - switch (type) { - case "String": - actual = metadata.getString(key); - break; - case "Boolean": - actual = metadata.getBoolean(key); - break; - case "Integer": - actual = metadata.getInteger(key); - break; - case "Float": - actual = metadata.getDouble(key); - break; - default: - throw new IllegalArgumentException("Unknown metadata type '" + type + "'"); - } - assertThat(actual) - .as("metadata '%s' (%s) in %s", key, type, metadata.asUnmodifiableMap()) - .isEqualTo(expected); - } - } - - /** Asserts that the last evaluation carries no flag metadata. */ - @Then("the resolved metadata is empty") - public void theResolvedMetadataIsEmpty() { - ImmutableMetadata metadata = requireMetadata(); - assertThat(metadata.asUnmodifiableMap()).as("flag metadata").isEmpty(); - } - - private ImmutableMetadata requireMetadata() { - assertThat(state.evaluation) - .as("an evaluation was recorded; did the scenario forget 'When the flag was evaluated with details'?") - .isNotNull(); - ImmutableMetadata metadata = state.evaluation.getFlagMetadata(); - assertThat(metadata).as("flag metadata").isNotNull(); - return metadata; - } -} From 1394e216e33b66367de01d24c01880e2b0207aeb Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Thu, 22 Oct 2026 13:06:02 +0200 Subject: [PATCH 21/27] style(flagd): drop two blank lines the step removals left behind Signed-off-by: Simon Schrottner --- .../contrib/providers/flagd/tck/AbstractResolverTest.java | 2 +- .../test/java/openfeature/tck/extensions/ConnectionSteps.java | 1 - .../src/test/java/openfeature/tck/extensions/FlagdSteps.java | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java index 33c7e34735..8abd9178b4 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/AbstractResolverTest.java @@ -73,7 +73,7 @@ public String backendService() { return "flagd"; } - /** The envoy sidecar's two listeners, which the extension scenarios connect through. */ + /** The envoy sidecar's two listeners; see {@link #composeFile()}. */ @Override public Map> additionalPorts() { return Collections.singletonMap(ENVOY_SERVICE, Arrays.asList(ENVOY_ROUTING_PORT, ENVOY_FORBIDDEN_PORT)); diff --git a/providers/flagd/src/test/java/openfeature/tck/extensions/ConnectionSteps.java b/providers/flagd/src/test/java/openfeature/tck/extensions/ConnectionSteps.java index 653c41c84a..655033ef36 100644 --- a/providers/flagd/src/test/java/openfeature/tck/extensions/ConnectionSteps.java +++ b/providers/flagd/src/test/java/openfeature/tck/extensions/ConnectionSteps.java @@ -57,5 +57,4 @@ public void theProviderConnectsToTheForbiddenListener() { int port = endpoint.port(FlagdTckHarness.ENVOY_SERVICE, FlagdTckHarness.ENVOY_FORBIDDEN_PORT); flagd.customize(builder -> builder.host(host).port(port)); } - } diff --git a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java index bd94f94ee3..43565f601e 100644 --- a/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java +++ b/providers/flagd/src/test/java/openfeature/tck/extensions/FlagdSteps.java @@ -1,6 +1,5 @@ package openfeature.tck.extensions; - import com.fasterxml.jackson.databind.ObjectMapper; import dev.openfeature.contrib.providers.flagd.FlagdOptions; import dev.openfeature.contrib.providers.flagd.resolver.rpc.cache.CacheType; From a41c007aea949e1633e7984f46456b287f7e5eed Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sat, 31 Oct 2026 13:29:06 +0100 Subject: [PATCH 22/27] docs(flagd): the metadata extension does keep @file The porting note said @file was dropped because this suite has no file resolver, but the tag is still on the feature, as it is on the four other ported features. The two scenarios a file resolver genuinely cannot serve name a selector flag file and already carry @metadata-provider, so the tag was right and the sentence was wrong. Signed-off-by: Simon Schrottner --- providers/flagd/src/test/resources/extensions/metadata.feature | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/flagd/src/test/resources/extensions/metadata.feature b/providers/flagd/src/test/resources/extensions/metadata.feature index f4ee6fe38b..73d26e3e67 100644 --- a/providers/flagd/src/test/resources/extensions/metadata.feature +++ b/providers/flagd/src/test/resources/extensions/metadata.feature @@ -3,7 +3,8 @@ Feature: flag and flag set metadata # Ported from flagd-testbed gherkin/metadata.feature. The testbed's "a stable flagd provider" # becomes the TCK's "a stable provider", and its "a metadata flagd provider" becomes "a stable - # provider with "metadata" configuration". @file is dropped: this suite has no file resolver. + # provider with "metadata" configuration". The two scenarios a file resolver cannot serve name a + # selector flag file and carry @metadata-provider. Scenario: Returns metadata Given a stable provider From 3d1647365291f7407a7507fba675cc63106caa08 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Sat, 31 Oct 2026 16:37:07 +0100 Subject: [PATCH 23/27] test(flagd): port the fractional composition scenarios new in testbed v3.11.0 v3.11.0 added five @fractional-nested @fractional-v3 outlines to gherkin/targeting.feature: a null or boolean bucket as the whole rule or as an if condition, a fractional result consumed by another operator, and one fractional bucketing another. The legacy e2e suites picked them up with the submodule bump, but this extension predates them, so the TCK ran 11 rows fewer than the suite it replaces. Their flags are already in the testbed's custom-ops.json, which the default launchpad configuration serves, so nothing else is needed. Signed-off-by: Simon Schrottner --- .../resources/extensions/fractional.feature | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/providers/flagd/src/test/resources/extensions/fractional.feature b/providers/flagd/src/test/resources/extensions/fractional.feature index fbace4a397..2444e98154 100644 --- a/providers/flagd/src/test/resources/extensions/fractional.feature +++ b/providers/flagd/src/test/resources/extensions/fractional.feature @@ -201,6 +201,70 @@ Feature: Fractional targeting When the flag was evaluated with details Then the resolved details value should be "ones" + # fractional composes like any other operator + @fractional @fractional-nested @fractional-v3 + Scenario Outline: Fractional with null bucket as whole rule + Given a String-flag with key "fractional-null-bucket-flag" and a default value "code-default" + And a context containing a targeting key with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "" + And the error-code should be "" + Examples: + | targetingKey | value | reason | + | user2 | fallback | DEFAULT | + | user1 | a | TARGETING_MATCH | + + @fractional @fractional-nested @fractional-v3 + Scenario Outline: Fractional with null bucket as if condition + Given a String-flag with key "fractional-null-bucket-condition-flag" and a default value "code-default" + And a context containing a targeting key with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "TARGETING_MATCH" + Examples: + | targetingKey | value | + | user1 | else | + | user2 | then | + + @fractional @fractional-nested @fractional-v3 + Scenario Outline: Fractional with boolean buckets as whole rule + Given a String-flag with key "fractional-bool-bucket-flag" and a default value "code-default" + And a context containing a targeting key with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the variant should be "" + And the reason should be "TARGETING_MATCH" + Examples: + | targetingKey | value | variant | + | user1 | on | true | + | user2 | off | false | + + @fractional @fractional-nested @fractional-v3 + Scenario Outline: Fractional result consumed by another operator + Given a String-flag with key "fractional-cat-flag" and a default value "code-default" + And a context containing a targeting key with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "TARGETING_MATCH" + Examples: + | targetingKey | value | + | user2 | big | + | user1 | dig | + + @fractional @fractional-nested @fractional-v3 + Scenario Outline: Fractional as bucketing input of another fractional + Given a String-flag with key "fractional-of-fractional-flag" and a default value "code-default" + And a context containing a targeting key with value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "TARGETING_MATCH" + Examples: + | targetingKey | value | + | user2 | red | + | user3 | red | + | user1 | blue | + @operator-errors Scenario: fractional operator with missing bucket key falls back to default variant Given a String-flag with key "fractional-null-bucket-key-flag" and a default value "wrong" From 7f33c93c470b09eadca6a71e6dd166f993b4e8ca Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 2 Nov 2026 20:37:32 +0100 Subject: [PATCH 24/27] test(flagd): replace the legacy e2e suites with the TCK extensions The flagd-testbed scenarios now run as Provider TCK extensions, so the Run*Test suites, their steps and the container pool go. config.feature needs no backend: its steps move to a plain test package and ConfigCucumberTest keeps reading it from the test-harness submodule. Signed-off-by: Simon Schrottner --- providers/flagd/CONTRIBUTING.md | 14 +- providers/flagd/README.md | 8 +- providers/flagd/pom.xml | 20 +- .../providers/flagd/ConfigCucumberTest.java | 8 +- .../providers/flagd/config/ConfigState.java | 10 + .../{e2e/steps => }/config/ConfigSteps.java | 14 +- .../EnvironmentVariableUtils.java | 2 +- .../flagd/{e2e/steps => config}/Utils.java | 2 +- .../providers/flagd/e2e/ContainerEntry.java | 50 ----- .../providers/flagd/e2e/ContainerPool.java | 116 ---------- .../providers/flagd/e2e/ContainerUtil.java | 32 --- .../providers/flagd/e2e/RunFileTest.java | 51 ----- .../providers/flagd/e2e/RunInProcessTest.java | 40 ---- .../providers/flagd/e2e/RunRpcTest.java | 40 ---- .../contrib/providers/flagd/e2e/State.java | 36 --- .../flagd/e2e/steps/AbstractSteps.java | 11 - .../flagd/e2e/steps/ContextSteps.java | 53 ----- .../providers/flagd/e2e/steps/Event.java | 13 -- .../providers/flagd/e2e/steps/EventSteps.java | 89 -------- .../providers/flagd/e2e/steps/FlagSteps.java | 148 ------------- .../flagd/e2e/steps/ProviderSteps.java | 206 ------------------ .../flagd/e2e/steps/ProviderType.java | 7 - .../resources/extensions/evaluation.feature | 6 +- .../test/resources/junit-platform.properties | 21 -- 24 files changed, 40 insertions(+), 957 deletions(-) create mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/ConfigState.java rename providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/{e2e/steps => }/config/ConfigSteps.java (90%) rename providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/{e2e/steps => config}/EnvironmentVariableUtils.java (98%) rename providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/{e2e/steps => config}/Utils.java (97%) delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerEntry.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerPool.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerUtil.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunFileTest.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunInProcessTest.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunRpcTest.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/State.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/AbstractSteps.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ContextSteps.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/Event.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/EventSteps.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/FlagSteps.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ProviderSteps.java delete mode 100644 providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ProviderType.java delete mode 100644 providers/flagd/src/test/resources/junit-platform.properties diff --git a/providers/flagd/CONTRIBUTING.md b/providers/flagd/CONTRIBUTING.md index 583b4be548..137fb9b9df 100644 --- a/providers/flagd/CONTRIBUTING.md +++ b/providers/flagd/CONTRIBUTING.md @@ -20,20 +20,14 @@ In vscode for instance, the following settings are recommended: ## End-to-End Tests -The continuous integration runs a set of [gherkin e2e tests](https://github.com/open-feature/test-harness/blob/main/features/evaluation.feature) using [`flagd`](https://github.com/open-feature/flagd). These tests do not run with the default maven profile. If you'd like to run them locally, you can start the flagd testbed with +The [flagd-testbed](https://github.com/open-feature/flagd-testbed) scenarios run through the +Provider TCK, which starts the testbed itself (Docker required): ``` -docker compose up -``` -and then run -``` -mvn test -P e2e +mvn -Ptck -pl providers/flagd -am test ``` -Note if your docker compose networking does not resolve to localhost, you can set a custom host -``` -FLAGD_HOST=192.168.100.1 mvn test -P e2e -``` +See [Provider conformance (TCK)](README.md#provider-conformance-tck) for what runs and how. If test-harness is being updated, commit the changes before running `mvn`. Otherwise, uncommitted changes will be overridden by maven test plugins. diff --git a/providers/flagd/README.md b/providers/flagd/README.md index 4daa06b84e..40579d4584 100644 --- a/providers/flagd/README.md +++ b/providers/flagd/README.md @@ -375,9 +375,11 @@ mvn -Ptck -pl providers/flagd -am -Dtest=InProcessTest -Dsurefire.failIfNoSp The root pom's `tck` profile runs only the `tck` package, so `-am` adds no other module's tests. -**`-Pe2e` does not run these suites.** The root pom excludes every provider's `tck` package from -the default build and from `-Pe2e`, and its `tck` profile runs only that package. This module -excludes just `**/e2e/*.java`, which the `e2e` profile clears. +**These suites also carry the rest of flagd-testbed's scenarios**, ported into +`src/test/resources/extensions/` and run through the TCK's extension hook, so they replace the +former `e2e` suites. `ConfigCucumberTest` still reads `config.feature` from the `test-harness` +submodule; it needs no backend and runs in the default build. The root pom excludes every +provider's `tck` package from the default build and from `-Pe2e`. **CI runs them in a job of their own**, `Provider TCK` in `ci.yml`, on every pull request. It sits apart from the `main` matrix so a conformance failure cannot be read as a regression there. Set diff --git a/providers/flagd/pom.xml b/providers/flagd/pom.xml index daf72975a4..75161a0296 100644 --- a/providers/flagd/pom.xml +++ b/providers/flagd/pom.xml @@ -14,12 +14,6 @@ 0.15.0 - - **/e2e/*.java 1.85.0 3.25.6 @@ -269,16 +263,12 @@ - + e2e - - - - diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/ConfigCucumberTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/ConfigCucumberTest.java index d38c8339dd..29b543987d 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/ConfigCucumberTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/ConfigCucumberTest.java @@ -12,17 +12,17 @@ import org.junit.platform.suite.api.Suite; /** - * Class for running the tests associated with "stable" e2e tests (no fake disconnection) for the - * in-process provider + * Runs the flagd-testbed's config.feature, which checks option defaults, environment variables + * and builder validation. No backend is involved. */ @Order(value = Integer.MAX_VALUE) @Suite @IncludeEngines("cucumber") @SelectFile("test-harness/gherkin/config.feature") @ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "summary") -@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "dev.openfeature.contrib.providers.flagd.e2e.steps.config") +@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "dev.openfeature.contrib.providers.flagd.config") // The TCK ships its own Cucumber ObjectFactory, so two are on the test classpath and Cucumber -// refuses to guess. The e2e suites name the same one; this suite needs it for the same reason. +// refuses to guess. @ConfigurationParameter(key = OBJECT_FACTORY_PROPERTY_NAME, value = "io.cucumber.picocontainer.PicoFactory") // Config scenarios read System env vars in FlagdOptions.build() and some scenarios also // mutate them. Parallel execution causes env-var races (e.g. FLAGD_PORT=3456 leaking into diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/ConfigState.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/ConfigState.java new file mode 100644 index 0000000000..0a3d36167b --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/ConfigState.java @@ -0,0 +1,10 @@ +package dev.openfeature.contrib.providers.flagd.config; + +import dev.openfeature.contrib.providers.flagd.FlagdOptions; + +/** Scenario state shared by the config steps, one instance per scenario. */ +public class ConfigState { + public FlagdOptions options; + public FlagdOptions.FlagdOptionsBuilder builder = FlagdOptions.builder(); + public boolean hasError; +} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/config/ConfigSteps.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/ConfigSteps.java similarity index 90% rename from providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/config/ConfigSteps.java rename to providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/ConfigSteps.java index 7418fe1b03..8e4cfece54 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/config/ConfigSteps.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/ConfigSteps.java @@ -1,12 +1,8 @@ -package dev.openfeature.contrib.providers.flagd.e2e.steps.config; +package dev.openfeature.contrib.providers.flagd.config; import static org.assertj.core.api.Assertions.assertThat; import dev.openfeature.contrib.providers.flagd.Config; -import dev.openfeature.contrib.providers.flagd.e2e.State; -import dev.openfeature.contrib.providers.flagd.e2e.steps.AbstractSteps; -import dev.openfeature.contrib.providers.flagd.e2e.steps.EnvironmentVariableUtils; -import dev.openfeature.contrib.providers.flagd.e2e.steps.Utils; import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; @@ -19,7 +15,9 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class ConfigSteps extends AbstractSteps { +public class ConfigSteps { + private final ConfigState state; + /** * Not all properties are correctly implemented, hence that we need to ignore them till this is * fixed @@ -30,8 +28,8 @@ public class ConfigSteps extends AbstractSteps { } }; - public ConfigSteps(State state) { - super(state); + public ConfigSteps(ConfigState state) { + this.state = state; } @When("a config was initialized") diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/EnvironmentVariableUtils.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/EnvironmentVariableUtils.java similarity index 98% rename from providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/EnvironmentVariableUtils.java rename to providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/EnvironmentVariableUtils.java index f7427cfa26..b493950962 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/EnvironmentVariableUtils.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/EnvironmentVariableUtils.java @@ -1,4 +1,4 @@ -package dev.openfeature.contrib.providers.flagd.e2e.steps; +package dev.openfeature.contrib.providers.flagd.config; /* * Copy of JUnit Pioneer's EnvironmentVariable Utils diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/Utils.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/Utils.java similarity index 97% rename from providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/Utils.java rename to providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/Utils.java index e53a83d290..c88eaac278 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/Utils.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/config/Utils.java @@ -1,4 +1,4 @@ -package dev.openfeature.contrib.providers.flagd.e2e.steps; +package dev.openfeature.contrib.providers.flagd.config; import dev.openfeature.contrib.providers.flagd.Config; import dev.openfeature.contrib.providers.flagd.resolver.rpc.cache.CacheType; diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerEntry.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerEntry.java deleted file mode 100644 index 820f808692..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerEntry.java +++ /dev/null @@ -1,50 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.RandomStringUtils; -import org.testcontainers.containers.ComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; - -/** A single pre-warmed Docker Compose stack (flagd + envoy) and its associated temp directory. */ -public class ContainerEntry { - - public static final int FORBIDDEN_PORT = 9212; - - public final ComposeContainer container; - public final Path tempDir; - - private ContainerEntry(ComposeContainer container, Path tempDir) { - this.container = container; - this.tempDir = tempDir; - } - - /** Start a new container entry. Blocks until all services are ready. */ - public static ContainerEntry start() throws IOException { - Path tempDir = Files.createDirectories( - Paths.get("tmp/" + RandomStringUtils.randomAlphanumeric(8).toLowerCase() + "/")); - - ComposeContainer container = new ComposeContainer(new File("test-harness/docker-compose.yaml")) - .withEnv("FLAGS_DIR", tempDir.toAbsolutePath().toString()) - .withExposedService("flagd", 8013, Wait.forListeningPort()) - .withExposedService("flagd", 8015, Wait.forListeningPort()) - .withExposedService("flagd", 8080, Wait.forListeningPort()) - .withExposedService("envoy", 9211, Wait.forListeningPort()) - .withExposedService("envoy", FORBIDDEN_PORT, Wait.forListeningPort()) - .withStartupTimeout(Duration.ofSeconds(45)); - container.start(); - - return new ContainerEntry(container, tempDir); - } - - /** Stop the container and clean up the temp directory. */ - public void stop() throws IOException { - container.stop(); - FileUtils.deleteDirectory(tempDir.toFile()); - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerPool.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerPool.java deleted file mode 100644 index 4d687c57c2..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerPool.java +++ /dev/null @@ -1,116 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import lombok.extern.slf4j.Slf4j; - -/** - * A pool of pre-warmed {@link ContainerEntry} instances. - * - *

All containers are started in parallel during {@link #initialize()}, paying the ~45s Docker - * Compose startup cost only once. Scenarios borrow a container via {@link #acquire()} and return - * it via {@link #release(ContainerEntry)} after teardown, allowing the next scenario to reuse it - * immediately without any cold-start overhead. - * - *

Pool size is controlled by the system property {@code flagd.e2e.pool.size} (default: 2). - * - *

Multiple test classes may share the same JVM fork (Surefire {@code reuseForks=true}). Each - * class calls {@link #initialize()} and {@link #shutdown()} once. A reference counter ensures - * that containers are only started on the first {@code initialize()} call and only stopped when - * the last {@code shutdown()} call is made, preventing one class from destroying containers that - * are still in use by another class running concurrently in the same JVM. - */ -@Slf4j -public class ContainerPool { - - private static final int POOL_SIZE = Integer.getInteger( - "flagd.e2e.pool.size", Math.min(Runtime.getRuntime().availableProcessors(), 4)); - - private static final BlockingQueue pool = new LinkedBlockingQueue<>(); - private static final List all = new ArrayList<>(); - private static final java.util.concurrent.atomic.AtomicInteger refCount = - new java.util.concurrent.atomic.AtomicInteger(0); - - public static void initialize() throws Exception { - if (refCount.getAndIncrement() > 0) { - log.info("Container pool already initialized (refCount={}), reusing existing pool.", refCount.get()); - return; - } - log.info("Starting container pool of size {}...", POOL_SIZE); - ExecutorService executor = Executors.newFixedThreadPool(POOL_SIZE); - List> futures = new ArrayList<>(); - try { - for (int i = 0; i < POOL_SIZE; i++) { - futures.add(executor.submit(ContainerEntry::start)); - } - for (Future future : futures) { - ContainerEntry entry = future.get(); - pool.add(entry); - all.add(entry); - } - } catch (Exception e) { - // Collect any containers that started successfully (including from in-flight futures) - for (Future future : futures) { - try { - ContainerEntry entry = future.get(); - if (!all.contains(entry)) { - all.add(entry); - } - } catch (Exception ignored) { - // future failed; nothing to stop for it - } - } - all.forEach(entry -> { - try { - entry.stop(); - } catch (IOException suppressed) { - e.addSuppressed(suppressed); - } - }); - pool.clear(); - all.clear(); - refCount.decrementAndGet(); - throw e; - } finally { - executor.shutdown(); - } - log.info("Container pool ready ({} containers).", POOL_SIZE); - } - - public static void shutdown() { - int remaining = refCount.decrementAndGet(); - if (remaining > 0) { - log.info("Container pool still in use by {} class(es), deferring shutdown.", remaining); - return; - } - log.info("Last shutdown call — stopping all containers."); - all.forEach(entry -> { - try { - entry.stop(); - } catch (IOException e) { - log.warn("Error stopping container entry", e); - } - }); - pool.clear(); - all.clear(); - } - - /** - * Borrow a container from the pool, blocking until one becomes available. - * The caller MUST call {@link #release(ContainerEntry)} when done. - */ - public static ContainerEntry acquire() throws InterruptedException { - return pool.take(); - } - - /** Return a container to the pool so the next scenario can use it. */ - public static void release(ContainerEntry entry) { - pool.add(entry); - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerUtil.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerUtil.java deleted file mode 100644 index b639672230..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerUtil.java +++ /dev/null @@ -1,32 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e; - -import dev.openfeature.contrib.providers.flagd.Config; -import java.util.Optional; -import org.testcontainers.containers.ComposeContainer; -import org.testcontainers.containers.ContainerState; - -public class ContainerUtil { - public static int getPort(ComposeContainer container, Config.Resolver resolver) { - Optional flagd = container.getContainerByServiceName("flagd"); - - return flagd.map(containerState -> { - switch (resolver) { - case RPC: - return containerState.getMappedPort(8013); - case IN_PROCESS: - return containerState.getMappedPort(8015); - default: - return 0; - } - }) - .orElseThrow(() -> new RuntimeException("Could not map port")); - } - - public static String getLaunchpadUrl(ComposeContainer container) { - Optional flagd = container.getContainerByServiceName("flagd"); - return flagd.map(containerState -> { - return containerState.getHost() + ":" + containerState.getMappedPort(8080); - }) - .orElseThrow(() -> new RuntimeException("Could not find launchpad url")); - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunFileTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunFileTest.java deleted file mode 100644 index c7387b6e33..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunFileTest.java +++ /dev/null @@ -1,51 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e; - -import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; -import static io.cucumber.junit.platform.engine.Constants.OBJECT_FACTORY_PROPERTY_NAME; -import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; - -import dev.openfeature.contrib.providers.flagd.Config; -import org.apache.logging.log4j.core.config.Order; -import org.junit.platform.suite.api.BeforeSuite; -import org.junit.platform.suite.api.ConfigurationParameter; -import org.junit.platform.suite.api.ExcludeTags; -import org.junit.platform.suite.api.IncludeEngines; -import org.junit.platform.suite.api.IncludeTags; -import org.junit.platform.suite.api.SelectDirectories; -import org.junit.platform.suite.api.Suite; -import org.testcontainers.junit.jupiter.Testcontainers; - -/** - * Class for running the reconnection tests for the RPC provider - */ -@Order(value = Integer.MAX_VALUE) -@Suite -@IncludeEngines("cucumber") -@SelectDirectories("test-harness/gherkin") -// if you want to run just one feature file, use the following line instead of @SelectDirectories -// @SelectFile("test-harness/gherkin/connection.feature") -@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "summary") -@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "dev.openfeature.contrib.providers.flagd.e2e.steps") -@ConfigurationParameter(key = OBJECT_FACTORY_PROPERTY_NAME, value = "io.cucumber.picocontainer.PicoFactory") -@IncludeTags("file") -@ExcludeTags({ - "unixsocket", - "targetURI", - "reconnect", - "customCert", - "events", - "contextEnrichment", - "fractional-v1", - "fractional-v2", - "deprecated", - // reads 2^53-1 via 32-bit getInteger; Long scenarios: open-feature/flagd-testbed#379 - "large-integers" -}) -@Testcontainers -public class RunFileTest { - - @BeforeSuite - public static void before() { - State.resolverType = Config.Resolver.FILE; - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunInProcessTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunInProcessTest.java deleted file mode 100644 index 05eb219db1..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunInProcessTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e; - -import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; -import static io.cucumber.junit.platform.engine.Constants.OBJECT_FACTORY_PROPERTY_NAME; -import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; - -import dev.openfeature.contrib.providers.flagd.Config; -import org.apache.logging.log4j.core.config.Order; -import org.junit.platform.suite.api.BeforeSuite; -import org.junit.platform.suite.api.ConfigurationParameter; -import org.junit.platform.suite.api.ExcludeTags; -import org.junit.platform.suite.api.IncludeEngines; -import org.junit.platform.suite.api.IncludeTags; -import org.junit.platform.suite.api.SelectDirectories; -import org.junit.platform.suite.api.Suite; -import org.testcontainers.junit.jupiter.Testcontainers; - -/** - * Class for running the reconnection tests for the RPC provider - */ -@Order(value = Integer.MAX_VALUE) -@Suite -@IncludeEngines("cucumber") -@SelectDirectories("test-harness/gherkin") -// if you want to run just one feature file, use the following line instead of @SelectDirectories -// @SelectFile("test-harness/gherkin/selector.feature") -@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "summary") -@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "dev.openfeature.contrib.providers.flagd.e2e.steps") -@ConfigurationParameter(key = OBJECT_FACTORY_PROPERTY_NAME, value = "io.cucumber.picocontainer.PicoFactory") -@IncludeTags("in-process") -// large-integers reads 2^53-1 via 32-bit getInteger; Long scenarios: open-feature/flagd-testbed#379 -@ExcludeTags({"unixsocket", "fractional-v1", "fractional-v2", "deprecated", "large-integers"}) -@Testcontainers -public class RunInProcessTest { - - @BeforeSuite - public static void before() { - State.resolverType = Config.Resolver.IN_PROCESS; - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunRpcTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunRpcTest.java deleted file mode 100644 index 4fe721d093..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunRpcTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e; - -import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; -import static io.cucumber.junit.platform.engine.Constants.OBJECT_FACTORY_PROPERTY_NAME; -import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; - -import dev.openfeature.contrib.providers.flagd.Config; -import org.apache.logging.log4j.core.config.Order; -import org.junit.platform.suite.api.BeforeSuite; -import org.junit.platform.suite.api.ConfigurationParameter; -import org.junit.platform.suite.api.ExcludeTags; -import org.junit.platform.suite.api.IncludeEngines; -import org.junit.platform.suite.api.IncludeTags; -import org.junit.platform.suite.api.SelectDirectories; -import org.junit.platform.suite.api.Suite; -import org.testcontainers.junit.jupiter.Testcontainers; - -/** - * Class for running the reconnection tests for the RPC provider - */ -@Order(value = Integer.MAX_VALUE) -@Suite -@IncludeEngines("cucumber") -@SelectDirectories("test-harness/gherkin") -// if you want to run just one feature file, use the following line instead of @SelectDirectories -// @SelectFile("test-harness/gherkin/rpc-caching.feature") -@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "summary") -@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "dev.openfeature.contrib.providers.flagd.e2e.steps") -@ConfigurationParameter(key = OBJECT_FACTORY_PROPERTY_NAME, value = "io.cucumber.picocontainer.PicoFactory") -@IncludeTags({"rpc"}) -// large-integers reads 2^53-1 via 32-bit getInteger; Long scenarios: open-feature/flagd-testbed#379 -@ExcludeTags({"unixsocket", "fractional-v1", "fractional-v2", "deprecated", "large-integers"}) -@Testcontainers -public class RunRpcTest { - - @BeforeSuite - public static void before() { - State.resolverType = Config.Resolver.RPC; - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/State.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/State.java deleted file mode 100644 index 2c671d5425..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/State.java +++ /dev/null @@ -1,36 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e; - -import dev.openfeature.contrib.providers.flagd.Config; -import dev.openfeature.contrib.providers.flagd.FlagdOptions; -import dev.openfeature.contrib.providers.flagd.e2e.steps.Event; -import dev.openfeature.contrib.providers.flagd.e2e.steps.FlagSteps; -import dev.openfeature.contrib.providers.flagd.e2e.steps.ProviderType; -import dev.openfeature.sdk.Client; -import dev.openfeature.sdk.FeatureProvider; -import dev.openfeature.sdk.FlagEvaluationDetails; -import dev.openfeature.sdk.MutableContext; -import java.util.Optional; -import java.util.concurrent.ConcurrentLinkedQueue; - -public class State { - public ProviderType providerType; - public Client client; - public FeatureProvider provider; - /** The domain name under which this scenario's provider is registered with OpenFeatureAPI. */ - public String providerName; - /** The container borrowed from {@link ContainerPool} for this scenario. */ - public ContainerEntry containerEntry; - - // events not yet consumed by a positive assertion; drained as they are matched - public ConcurrentLinkedQueue assertedEvents = new ConcurrentLinkedQueue<>(); - // complete log of every event emitted, used for "never fired" assertions - public ConcurrentLinkedQueue allEvents = new ConcurrentLinkedQueue<>(); - public Optional lastEvent; - public FlagSteps.Flag flag; - public MutableContext context = new MutableContext(); - public FlagEvaluationDetails evaluation; - public FlagdOptions options; - public FlagdOptions.FlagdOptionsBuilder builder = FlagdOptions.builder(); - public static Config.Resolver resolverType; - public boolean hasError; -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/AbstractSteps.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/AbstractSteps.java deleted file mode 100644 index 4a74400c48..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/AbstractSteps.java +++ /dev/null @@ -1,11 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e.steps; - -import dev.openfeature.contrib.providers.flagd.e2e.State; - -public abstract class AbstractSteps { - protected State state; - - protected AbstractSteps(State state) { - this.state = state; - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ContextSteps.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ContextSteps.java deleted file mode 100644 index b92c994b7e..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ContextSteps.java +++ /dev/null @@ -1,53 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e.steps; - -import dev.openfeature.contrib.providers.flagd.e2e.State; -import dev.openfeature.sdk.ImmutableStructure; -import dev.openfeature.sdk.MutableContext; -import dev.openfeature.sdk.Value; -import io.cucumber.datatable.DataTable; -import io.cucumber.java.en.Given; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class ContextSteps extends AbstractSteps { - - public ContextSteps(State state) { - super(state); - } - - @Given("^a context containing a key \"([^\"]*)\", with type \"([^\"]*)\" and with value \"(.*)\"$") - public void a_context_containing_a_key_with_type_and_with_value(String key, String type, String value) - throws ClassNotFoundException, IOException { - Map map = new HashMap<>(state.context.asMap()); - map.put(key, Value.objectToValue(Utils.convert(value, type))); - state.context = new MutableContext(state.context.getTargetingKey(), map); - } - - @Given("a context with the following keys:") - public void a_context_with_the_following_keys(DataTable dataTable) throws ClassNotFoundException, IOException { - List> rows = dataTable.asMaps(String.class, String.class); - Map map = new HashMap<>(state.context.asMap()); - for (Map row : rows) { - String key = row.get("key"); - String type = row.get("type"); - String value = row.get("value"); - map.put(key, Value.objectToValue(Utils.convert(value, type))); - } - state.context = new MutableContext(state.context.getTargetingKey(), map); - } - - @Given("a context containing a targeting key with value {string}") - public void a_context_containing_a_targeting_key_with_value(String string) { - state.context.setTargetingKey(string); - } - - @Given("a context containing a nested property with outer key {string} and inner key {string}, with value {string}") - public void a_context_containing_a_nested_property_with_outer_key_and_inner_key_with_value( - String outer, String inner, String value) { - Map innerMap = new HashMap<>(); - innerMap.put(inner, new Value(value)); - state.context.add(outer, new ImmutableStructure(innerMap)); - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/Event.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/Event.java deleted file mode 100644 index f93c327e6e..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/Event.java +++ /dev/null @@ -1,13 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e.steps; - -import dev.openfeature.sdk.EventDetails; - -public class Event { - public String type; - public EventDetails details; - - public Event(String eventType, EventDetails eventDetails) { - this.type = eventType; - this.details = eventDetails; - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/EventSteps.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/EventSteps.java deleted file mode 100644 index 9c3fd91163..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/EventSteps.java +++ /dev/null @@ -1,89 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e.steps; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; - -import dev.openfeature.contrib.providers.flagd.e2e.State; -import dev.openfeature.sdk.ProviderEvent; -import io.cucumber.java.en.Given; -import io.cucumber.java.en.Then; -import io.cucumber.java.en.When; -import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; - -@Slf4j -public class EventSteps extends AbstractSteps { - - private static final int EVENT_TIMEOUT_MS = 12_000; - - public EventSteps(State state) { - super(state); - } - - @Given("a {} event handler") - public void a_stale_event_handler(String eventType) { - state.client.on(mapEventType(eventType), eventDetails -> { - log.info("{} event tracked", eventType); - Event event = new Event(eventType, eventDetails); - state.allEvents.add(event); - state.assertedEvents.add(event); - }); - } - - private static @NotNull ProviderEvent mapEventType(String eventType) { - switch (eventType) { - case "stale": - return ProviderEvent.PROVIDER_STALE; - case "ready": - return ProviderEvent.PROVIDER_READY; - case "error": - return ProviderEvent.PROVIDER_ERROR; - case "change": - return ProviderEvent.PROVIDER_CONFIGURATION_CHANGED; - default: - throw new IllegalArgumentException("Unknown event type: " + eventType); - } - } - - @When("a {} event was fired") - public void eventWasFired(String eventType) { - eventHandlerShouldBeExecutedWithin(eventType, EVENT_TIMEOUT_MS); - } - - @Then("the {} event handler should have been executed") - public void eventHandlerShouldBeExecuted(String eventType) { - eventHandlerShouldBeExecutedWithin(eventType, EVENT_TIMEOUT_MS); - } - - @Then("the {} event handler should not have been executed") - public void eventHandlerShouldNotHaveBeenExecuted(String eventType) { - // checks the full log, not assertedEvents: preceding positive - // assertions may have drained an intervening event of this type - assertThat(state.allEvents.stream().anyMatch(event -> event.type.equals(eventType))) - .as("no %s event should have fired", eventType) - .isFalse(); - } - - @Then("the {} event handler should have been executed within {int}ms") - public void eventHandlerShouldBeExecutedWithin(String eventType, int ms) { - log.info("waiting for eventtype: {}", eventType); - await().alias("waiting for eventtype " + eventType) - .atMost(ms, MILLISECONDS) - .pollInterval(10, MILLISECONDS) - .until(() -> state.assertedEvents.stream().anyMatch(event -> event.type.equals(eventType))); - // Drain all events up to and including the first match. This ensures that - // older events (e.g. a READY from before a disconnect) cannot satisfy a - // later assertion that expects a *new* event of the same type, while still - // preserving events that arrived *after* the match for subsequent steps. - Event matched = null; - while (!state.assertedEvents.isEmpty()) { - Event head = state.assertedEvents.poll(); - if (head != null && head.type.equals(eventType)) { - matched = head; - break; - } - } - state.lastEvent = java.util.Optional.ofNullable(matched); - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/FlagSteps.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/FlagSteps.java deleted file mode 100644 index 462fa13819..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/FlagSteps.java +++ /dev/null @@ -1,148 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e.steps; - -import static org.assertj.core.api.Assertions.assertThat; - -import dev.openfeature.contrib.providers.flagd.e2e.State; -import dev.openfeature.sdk.ErrorCode; -import dev.openfeature.sdk.FlagEvaluationDetails; -import dev.openfeature.sdk.ImmutableMetadata; -import dev.openfeature.sdk.Value; -import io.cucumber.java.en.Given; -import io.cucumber.java.en.Then; -import io.cucumber.java.en.When; -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.List; -import java.util.Map; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class FlagSteps extends AbstractSteps { - - public FlagSteps(State state) { - super(state); - } - - @Given("a {}-flag with key {string} and a default value {string}") - public void givenAFlag(String type, String name, String defaultValue) throws Throwable { - state.flag = new Flag(type, name, Utils.convert(defaultValue, type)); - } - - @When("the flag was evaluated with details") - public void the_flag_was_evaluated_with_details() throws InterruptedException { - FlagEvaluationDetails details; - switch (state.flag.type) { - case "String": - details = - state.client.getStringDetails(state.flag.name, (String) state.flag.defaultValue, state.context); - break; - case "Boolean": - details = state.client.getBooleanDetails( - state.flag.name, (Boolean) state.flag.defaultValue, state.context); - break; - case "Float": - details = - state.client.getDoubleDetails(state.flag.name, (Double) state.flag.defaultValue, state.context); - break; - case "Integer": - details = state.client.getIntegerDetails( - state.flag.name, (Integer) state.flag.defaultValue, state.context); - break; - case "Object": - details = - state.client.getObjectDetails(state.flag.name, (Value) state.flag.defaultValue, state.context); - break; - default: - throw new AssertionError(); - } - state.evaluation = details; - } - - @Then("the resolved details value should be \"{}\"") - public void the_resolved_details_value_should_be(String value) throws Throwable { - if (state.evaluation.getErrorCode() != null) { - log.warn(state.evaluation.getErrorMessage()); - } - assertThat(state.evaluation.getValue()).isEqualTo(Utils.convert(value, state.flag.type)); - } - - @Then("the reason should be {string}") - public void the_reason_should_be(String reason) { - assertThat(state.evaluation.getReason()).isEqualTo(reason); - } - - @Then("the variant should be {string}") - public void the_variant_should_be(String variant) { - assertThat(state.evaluation.getVariant()).isEqualTo(variant); - } - - @Then("the error-code should be {string}") - public void the_error_code_should_be(String errorCode) { - if (errorCode.isEmpty()) { - assertThat(state.evaluation.getErrorCode()).isNull(); - } else { - assertThat(state.evaluation.getErrorCode()).isEqualTo(ErrorCode.valueOf(errorCode)); - } - } - - @Then("the flag should be part of the event payload") - public void the_flag_was_modified() { - Event event = state.lastEvent.orElseThrow(AssertionError::new); - assertThat(event.details.getFlagsChanged()).contains(state.flag.name); - } - - public class Flag { - String name; - Object defaultValue; - String type; - - public Flag(String type, String name, Object defaultValue) { - this.name = name; - this.defaultValue = defaultValue; - this.type = type; - } - } - - @Then("the resolved metadata is empty") - @SuppressWarnings("unchecked") - public void the_resolved_metadata_is_empty() throws NoSuchFieldException, IllegalAccessException { - ImmutableMetadata flagMetadata = state.evaluation.getFlagMetadata(); - - Field metadataField = flagMetadata.getClass().getDeclaredField("metadata"); - metadataField.setAccessible(true); - Map metadataMap = (Map) metadataField.get(flagMetadata); - assertThat(metadataMap).isEmpty(); - } - - @Then("the resolved metadata should contain") - @SuppressWarnings("unchecked") - public void the_resolved_metadata_should_contain(io.cucumber.datatable.DataTable dataTable) - throws IOException, ClassNotFoundException { - - ImmutableMetadata flagMetadata = state.evaluation.getFlagMetadata(); - - List> rows = dataTable.asMaps(String.class, String.class); - for (Map row : rows) { - switch (row.get("metadata_type")) { - case "String": - assertThat(flagMetadata.getString(row.get("key"))) - .isEqualTo(Utils.convert(row.get("value"), row.get("metadata_type"))); - break; - case "Boolean": - assertThat(flagMetadata.getBoolean(row.get("key"))) - .isEqualTo(Utils.convert(row.get("value"), row.get("metadata_type"))); - break; - case "Float": - assertThat(flagMetadata.getDouble(row.get("key"))) - .isEqualTo(Utils.convert(row.get("value"), row.get("metadata_type"))); - break; - case "Integer": - assertThat(flagMetadata.getInteger(row.get("key"))) - .isEqualTo(Utils.convert(row.get("value"), row.get("metadata_type"))); - break; - default: - throw new AssertionError("type not supported"); - } - } - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ProviderSteps.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ProviderSteps.java deleted file mode 100644 index f9e1b846f1..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ProviderSteps.java +++ /dev/null @@ -1,206 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e.steps; - -import static io.restassured.RestAssured.when; -import static org.assertj.core.api.Assertions.assertThat; - -import dev.openfeature.contrib.providers.flagd.Config; -import dev.openfeature.contrib.providers.flagd.FlagdOptions; -import dev.openfeature.contrib.providers.flagd.FlagdProvider; -import dev.openfeature.contrib.providers.flagd.e2e.ContainerEntry; -import dev.openfeature.contrib.providers.flagd.e2e.ContainerPool; -import dev.openfeature.contrib.providers.flagd.e2e.ContainerUtil; -import dev.openfeature.contrib.providers.flagd.e2e.State; -import dev.openfeature.sdk.FeatureProvider; -import dev.openfeature.sdk.NoOpProvider; -import dev.openfeature.sdk.OpenFeatureAPI; -import dev.openfeature.sdk.ProviderState; -import io.cucumber.java.After; -import io.cucumber.java.AfterAll; -import io.cucumber.java.BeforeAll; -import io.cucumber.java.en.Given; -import io.cucumber.java.en.Then; -import io.cucumber.java.en.When; -import java.io.File; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.testcontainers.containers.ComposeContainer; - -@Slf4j -public class ProviderSteps extends AbstractSteps { - - public static final int UNAVAILABLE_PORT = 9999; - - public ProviderSteps(State state) { - super(state); - } - - @BeforeAll - public static void beforeAll() throws Exception { - ContainerPool.initialize(); - } - - @AfterAll - public static void afterAll() { - ContainerPool.shutdown(); - } - - @After - public void tearDown() { - if (state.containerEntry != null) { - if (state.client != null) { - when().post("http://" + ContainerUtil.getLaunchpadUrl(state.containerEntry.container) + "/stop") - .then() - .statusCode(200); - } - ContainerPool.release(state.containerEntry); - state.containerEntry = null; - } - // Replace the domain provider with a NoOp through the SDK lifecycle so the SDK - // properly calls detachEventProvider (nulls onEmit) and shuts down the emitter - // executor — neither of which happens when calling provider.shutdown() directly. - if (state.providerName != null) { - OpenFeatureAPI.getInstance().setProvider(state.providerName, new NoOpProvider()); - } - } - - @Given("a {} flagd provider") - public void setupProvider(String providerType) throws InterruptedException { - state.containerEntry = ContainerPool.acquire(); - ComposeContainer container = state.containerEntry.container; - - String flagdConfig = "default"; - state.builder - .deadline(1000) - .keepAlive(0) - .retryGracePeriod(5) - .retryBackoffMs(500) - .retryBackoffMaxMs(500); - boolean wait = true; - - switch (providerType) { - case "unavailable": - this.state.providerType = ProviderType.SOCKET; - state.builder.port(UNAVAILABLE_PORT); - if (State.resolverType == Config.Resolver.FILE) { - state.builder.offlineFlagSourcePath("not-existing"); - } - wait = false; - break; - case "forbidden": - state.builder.port(container.getServicePort("envoy", ContainerEntry.FORBIDDEN_PORT)); - wait = false; - break; - case "socket": - this.state.providerType = ProviderType.SOCKET; - String socketPath = state.containerEntry - .tempDir - .resolve("socket.sock") - .toAbsolutePath() - .toString(); - state.builder.socketPath(socketPath); - state.builder.port(UNAVAILABLE_PORT); - break; - case "ssl": - String path = "test-harness/ssl/custom-root-cert.crt"; - File file = new File(path); - String absolutePath = file.getAbsolutePath(); - this.state.providerType = ProviderType.SSL; - state.builder - .port(ContainerUtil.getPort(container, State.resolverType)) - .tls(true) - .certPath(absolutePath); - flagdConfig = "ssl"; - break; - case "metadata": - flagdConfig = "metadata"; - if (State.resolverType == Config.Resolver.FILE) { - FlagdOptions build = state.builder.build(); - String selector = build.getSelector(); - String replace = selector.replace("rawflags/", ""); - state.builder - .port(UNAVAILABLE_PORT) - .offlineFlagSourcePath(new File("test-harness/flags/" + replace).getAbsolutePath()); - } else { - state.builder.port(ContainerUtil.getPort(container, State.resolverType)); - } - break; - case "syncpayload": - flagdConfig = "sync-payload"; - state.builder.port(ContainerUtil.getPort(container, State.resolverType)); - break; - case "stable": - this.state.providerType = ProviderType.DEFAULT; - if (State.resolverType == Config.Resolver.FILE) { - state.builder - .port(UNAVAILABLE_PORT) - .offlineFlagSourcePath(state.containerEntry - .tempDir - .resolve("allFlags.json") - .toAbsolutePath() - .toString()); - } else { - state.builder.port(ContainerUtil.getPort(container, State.resolverType)); - } - break; - default: - throw new IllegalStateException(); - } - - // Setting TargetUri if this setting is set - FlagdOptions tempBuild = state.builder.build(); - if (!StringUtils.isEmpty(tempBuild.getTargetUri())) { - String replace = tempBuild.getTargetUri().replace("", "" + container.getServicePort("envoy", 9211)); - state.builder.targetUri(replace); - state.builder.port(UNAVAILABLE_PORT); - } - - when().post("http://" + ContainerUtil.getLaunchpadUrl(container) + "/start?config={config}", flagdConfig) - .then() - .statusCode(200); - - Thread.sleep(50); - - FeatureProvider provider = - new FlagdProvider(state.builder.resolverType(State.resolverType).build()); - String providerName = "Provider " + Math.random(); - OpenFeatureAPI api = OpenFeatureAPI.getInstance(); - - if (wait) { - api.setProviderAndWait(providerName, provider); - } else { - api.setProvider(providerName, provider); - } - this.state.provider = provider; - this.state.providerName = providerName; - this.state.client = api.getClient(providerName); - } - - @When("the connection is lost") - public void the_connection_is_lost() { - when().post("http://" + ContainerUtil.getLaunchpadUrl(state.containerEntry.container) + "/stop") - .then() - .statusCode(200); - } - - @When("the connection is lost for {int}s") - public void the_connection_is_lost_for(int seconds) { - when().post( - "http://" + ContainerUtil.getLaunchpadUrl(state.containerEntry.container) - + "/restart?seconds={seconds}", - seconds) - .then() - .statusCode(200); - } - - @When("the flag was modified") - public void the_flag_was_modded() { - when().post("http://" + ContainerUtil.getLaunchpadUrl(state.containerEntry.container) + "/change") - .then() - .statusCode(200); - } - - @Then("the client should be in {} state") - public void the_client_should_be_in_fatal_state(String clientState) { - assertThat(state.client.getProviderState()).isEqualTo(ProviderState.valueOf(clientState.toUpperCase())); - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ProviderType.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ProviderType.java deleted file mode 100644 index eac0ba9674..0000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ProviderType.java +++ /dev/null @@ -1,7 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e.steps; - -public enum ProviderType { - DEFAULT, - SSL, - SOCKET -} diff --git a/providers/flagd/src/test/resources/extensions/evaluation.feature b/providers/flagd/src/test/resources/extensions/evaluation.feature index 418558ba5a..c32b00db43 100644 --- a/providers/flagd/src/test/resources/extensions/evaluation.feature +++ b/providers/flagd/src/test/resources/extensions/evaluation.feature @@ -5,8 +5,10 @@ Feature: flagd evaluations # and reason.feature do not: the reason flagd reports for a zero value, a float zero value, # zero values served by a targeting rule, and flags with no default variant. # - # Not ported: "Resolve values" (the TCK's evaluation.feature asks it) and the @deprecated - # @no-default outline, which @no-default-variant replaces. + # Not ported: "Resolve values" (the TCK's evaluation.feature asks it), the @deprecated + # @no-default outline, which @no-default-variant replaces, and the two @precision scenarios, + # "Resolves numeric values without loss of precision" and "Resolves an integer beyond 32 bits + # without loss of precision", which the TCK's own precision scenarios ask. Background: Given an option "cache" of type "CacheType" with value "disabled" diff --git a/providers/flagd/src/test/resources/junit-platform.properties b/providers/flagd/src/test/resources/junit-platform.properties deleted file mode 100644 index cc6bbf8f62..0000000000 --- a/providers/flagd/src/test/resources/junit-platform.properties +++ /dev/null @@ -1,21 +0,0 @@ -# Enable parallel scenario execution within each suite runner. -# Each scenario borrows its own ContainerEntry from ContainerPool, so -# concurrent scenarios are fully isolated — no shared flagd process. -cucumber.execution.parallel.enabled=true -# Dynamic strategy scales with available CPUs (factor=1.0 → 1 thread per core). -# ContainerPool caps at min(availableProcessors, 4) containers so Docker isn't -# overwhelmed; extra threads simply block waiting for a free container. -# Override pool size via -Dflagd.e2e.pool.size=N if needed. -cucumber.execution.parallel.config.strategy=dynamic -cucumber.execution.parallel.config.dynamic.factor=1 -# Scenarios tagged @env-var mutate System env vars globally, which every config -# scenario reads when building FlagdOptions defaults. A plain ENV_VARS lock only -# serialises @env-var scenarios against each other, so a concurrent default-reading -# scenario could still observe a mutated var. Acquire the JUnit global lock so -# @env-var scenarios run in full isolation (no other scenario runs concurrently). -cucumber.execution.exclusive-resources.env-var.read-write=org.junit.platform.engine.support.hierarchical.ExclusiveResource.GLOBAL_KEY -# Scenarios tagged @grace involve container restart + reconnection timing. -# Running two concurrent restarts under parallel load can push the -# reconnection past the 12-second EVENT_TIMEOUT_MS threshold. Serialise -# them so each restart has the full machine resources to itself. -cucumber.execution.exclusive-resources.grace.read-write=CONTAINER_RESTART From 453ec7820772839a786e0f27ea71a4b50a47a155 Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 2 Nov 2026 20:41:52 +0100 Subject: [PATCH 25/27] test(flagd): read the TCK extensions from flagd-testbed The extension scenarios now live in flagd-testbed's tck-extensions/, so every flagd provider adopting the TCK runs the same files. The test-harness submodule is pinned to open-feature/flagd-testbed#445 until it is released, and the module puts tck-extensions/ on the test classpath as extensions/, where the TCK looks; the local copies go. Signed-off-by: Simon Schrottner --- providers/flagd/README.md | 4 +- providers/flagd/pom.xml | 14 + .../providers/flagd/tck/FlagdTckHarness.java | 4 +- .../resources/extensions/connection.feature | 118 ---- .../extensions/context-enrichment.feature | 73 --- .../resources/extensions/disabled.feature | 39 -- .../resources/extensions/evaluation.feature | 76 --- .../test/resources/extensions/events.feature | 56 -- .../resources/extensions/fractional.feature | 525 ------------------ .../resources/extensions/metadata.feature | 58 -- .../resources/extensions/rpc-caching.feature | 68 --- .../resources/extensions/selector.feature | 27 - .../resources/extensions/sync-payload.feature | 40 -- .../resources/extensions/targeting.feature | 222 -------- providers/flagd/test-harness | 2 +- 15 files changed, 19 insertions(+), 1307 deletions(-) delete mode 100644 providers/flagd/src/test/resources/extensions/connection.feature delete mode 100644 providers/flagd/src/test/resources/extensions/context-enrichment.feature delete mode 100644 providers/flagd/src/test/resources/extensions/disabled.feature delete mode 100644 providers/flagd/src/test/resources/extensions/evaluation.feature delete mode 100644 providers/flagd/src/test/resources/extensions/events.feature delete mode 100644 providers/flagd/src/test/resources/extensions/fractional.feature delete mode 100644 providers/flagd/src/test/resources/extensions/metadata.feature delete mode 100644 providers/flagd/src/test/resources/extensions/rpc-caching.feature delete mode 100644 providers/flagd/src/test/resources/extensions/selector.feature delete mode 100644 providers/flagd/src/test/resources/extensions/sync-payload.feature delete mode 100644 providers/flagd/src/test/resources/extensions/targeting.feature diff --git a/providers/flagd/README.md b/providers/flagd/README.md index 40579d4584..99b4dc2e18 100644 --- a/providers/flagd/README.md +++ b/providers/flagd/README.md @@ -375,8 +375,8 @@ mvn -Ptck -pl providers/flagd -am -Dtest=InProcessTest -Dsurefire.failIfNoSp The root pom's `tck` profile runs only the `tck` package, so `-am` adds no other module's tests. -**These suites also carry the rest of flagd-testbed's scenarios**, ported into -`src/test/resources/extensions/` and run through the TCK's extension hook, so they replace the +**These suites also carry the rest of flagd-testbed's scenarios**: its `tck-extensions/`, read +from the `test-harness` submodule and run through the TCK's extension hook, so they replace the former `e2e` suites. `ConfigCucumberTest` still reads `config.feature` from the `test-harness` submodule; it needs no backend and runs in the default build. The root pom excludes every provider's `tck` package from the default build and from `-Pe2e`. diff --git a/providers/flagd/pom.xml b/providers/flagd/pom.xml index 75161a0296..2ea718e9e8 100644 --- a/providers/flagd/pom.xml +++ b/providers/flagd/pom.xml @@ -176,6 +176,20 @@ + + + src/test/resources + + + + test-harness/tck-extensions + extensions + + *.feature + + + + org.codehaus.mojo diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java index bafb7fa66f..dfca90d48e 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/tck/FlagdTckHarness.java @@ -5,8 +5,8 @@ import java.util.function.Consumer; /** - * What the flagd extension scenarios under {@code src/test/resources/extensions} need from the - * adoption beyond the TCK's own harness. + * What the flagd extension scenarios, flagd-testbed's {@code tck-extensions/} on the classpath, + * need from the adoption beyond the TCK's own harness. * *

The TCK builds every provider through {@code createProvider()} with no arguments, which suits * the canonical scenarios: they ask the same question of every provider. The extensions port the diff --git a/providers/flagd/src/test/resources/extensions/connection.feature b/providers/flagd/src/test/resources/extensions/connection.feature deleted file mode 100644 index 9cfeec235a..0000000000 --- a/providers/flagd/src/test/resources/extensions/connection.feature +++ /dev/null @@ -1,118 +0,0 @@ -@ext-connection -Feature: flagd provider disconnect and reconnect - - # Ported from flagd-testbed gherkin/connection.feature. - # - # Not ported: the "stable" row of the Connection outline, which every canonical evaluation - # scenario asks, and the @unixsocket rows, which need a socket shared between host and container - # that the TCK stack does not mount. The testbed's "Provider unavailable" is kept although the - # canonical lifecycle.feature asks the same, because it holds flagd to a 3000ms bound where the - # canonical scenario allows 10000ms. - # - # The outlines over provider flavours are spelled out, one scenario per flavour, because a flavour - # here is a set of steps rather than a value. Reconnect timing is the testbed's "stable flagd - # provider", as in events.feature. - # - # @targetURI and @forbidden connect through the envoy sidecar in the testbed's - # test-harness/docker-compose.yaml. - - @rpc @in-process @file @customCert - Scenario: Connection with a custom certificate - Given an option "tls" of type "Boolean" with value "true" - And an option "certPath" of type "String" with value "test-harness/ssl/custom-root-cert.crt" - And a stable provider with "ssl" configuration - And a Boolean-flag with key "boolean-flag" and a default value "false" - When the flag was evaluated with details - Then the resolved details value should be "true" - - @rpc @in-process @file @reconnect - Scenario: Provider reconnection - Given an option "retryGracePeriod" of type "Integer" with value "5" - And an option "retryBackoffMaxMs" of type "Integer" with value "500" - And a stable provider - And a ready event handler - And a error event handler - When a ready event was fired - When the connection is lost - Then the error event handler should have been executed - When the connection is restored - Then the ready event handler should have been executed - - @rpc @in-process @file @reconnect @customCert - Scenario: Provider reconnection with a custom certificate - Given an option "tls" of type "Boolean" with value "true" - And an option "certPath" of type "String" with value "test-harness/ssl/custom-root-cert.crt" - And an option "retryGracePeriod" of type "Integer" with value "5" - And an option "retryBackoffMaxMs" of type "Integer" with value "500" - And a stable provider with "ssl" configuration - And a ready event handler - And a error event handler - When a ready event was fired - When the connection is lost - Then the error event handler should have been executed - When the connection is restored - Then the ready event handler should have been executed - - @rpc @in-process @file @unavailable - Scenario: Provider unavailable - Given an option "deadlineMs" of type "Integer" with value "1000" - And a unavailable provider - And a error event handler - Then the error event handler should have been executed within 3000ms - - # A forbidden response from flagd results in a fatal client state. "a unavailable provider" is the - # TCK's flavour that does not wait for readiness; the step before it points that provider at envoy. - @rpc @in-process @forbidden - Scenario: Provider forbidden - Given an option "fatalStatusCodes" of type "StringList" with value "PERMISSION_DENIED" - And the provider connects to envoy's forbidden listener - And a unavailable provider - And a error event handler - Then the error event handler should have been executed within 5000ms - And the client should be in fatal state - - @targetURI @rpc - Scenario: Connection via TargetUri rpc - Given a target URI "envoy://localhost:/rpc.service" routed through envoy - And a stable provider - And a Boolean-flag with key "boolean-flag" and a default value "false" - When the flag was evaluated with details - Then the resolved details value should be "true" - - @targetURI @in-process - Scenario: Connection via TargetUri in-process - Given a target URI "envoy://localhost:/sync.service" routed through envoy - And a stable provider - And a Boolean-flag with key "boolean-flag" and a default value "false" - When the flag was evaluated with details - Then the resolved details value should be "true" - - @targetURI @in-process @reconnect - Scenario: Re-Connection via TargetUri in-process - Given a target URI "envoy://localhost:/sync.service" routed through envoy - And an option "retryGracePeriod" of type "Integer" with value "5" - And an option "retryBackoffMaxMs" of type "Integer" with value "500" - And a stable provider - And a ready event handler - And a error event handler - When a ready event was fired - When the connection is lost - Then the error event handler should have been executed - When the connection is restored - Then the ready event handler should have been executed - - @targetURI @in-process @reconnect @grace - Scenario: Re-Connection via TargetUri in-process with grace period - Given a target URI "envoy://localhost:/sync.service" routed through envoy - And an option "retryGracePeriod" of type "Integer" with value "5" - And an option "retryBackoffMaxMs" of type "Integer" with value "500" - And a stable provider - And a ready event handler - And a stale event handler - And a error event handler - When a ready event was fired - When the connection is lost - Then the stale event handler should have been executed - Then the error event handler should have been executed - When the connection is restored - Then the ready event handler should have been executed diff --git a/providers/flagd/src/test/resources/extensions/context-enrichment.feature b/providers/flagd/src/test/resources/extensions/context-enrichment.feature deleted file mode 100644 index fcbf2a6eb9..0000000000 --- a/providers/flagd/src/test/resources/extensions/context-enrichment.feature +++ /dev/null @@ -1,73 +0,0 @@ -@ext-metadata @contextEnrichment -Feature: Context Enrichment - - # Ported from flagd-testbed gherkin/contextEnrichment.feature. The testbed's Background "a stable - # flagd provider" moves into each scenario as the TCK's "a stable provider", because the @grace - # scenarios record options that have to come before it. - # - # The @grace scenarios restore the testbed provider's retry timing (grace period 5s, backoff - # 500ms capped at 500ms) over the adoption's (grace 2s, backoff capped at the 5000ms default), so - # the provider is back within the event timeout once the connection is restored. The RPC one also - # restores the testbed's 1000ms deadline: at the adoption's 15000ms an evaluation issued while - # flagd is down could outlast the outage and succeed, which is not what it asks. - # - # The testbed's @caching is renamed @flagd-caching: the TCK reserves @caching and refuses a - # scenario that carries it. - - @in-process @rpc - Scenario: Use enriched context - Given a stable provider - And a String-flag with key "flagd-context-aware" and a default value "not" - When the flag was evaluated with details - Then the resolved details value should be "INTERNAL" - - @grace @in-process - Scenario: Use enriched context on connection error for IN-PROCESS - Given an option "retryGracePeriod" of type "Integer" with value "5" - And an option "retryBackoffMs" of type "Integer" with value "500" - And an option "retryBackoffMaxMs" of type "Integer" with value "500" - And a stable provider - And a String-flag with key "flagd-context-aware" and a default value "not" - And a stale event handler - And a ready event handler - When the flag was evaluated with details - Then the resolved details value should be "INTERNAL" - When the connection is lost - And a stale event was fired - When the flag was evaluated with details - Then the resolved details value should be "INTERNAL" - When the connection is restored - And a ready event was fired - - @grace @rpc - Scenario: Use enriched context on connection error for RPC - Given an option "deadlineMs" of type "Integer" with value "1000" - And an option "retryGracePeriod" of type "Integer" with value "5" - And an option "retryBackoffMs" of type "Integer" with value "500" - And an option "retryBackoffMaxMs" of type "Integer" with value "500" - And a stable provider - And a String-flag with key "flagd-context-aware" and a default value "not" - And a stale event handler - And a ready event handler - When the flag was evaluated with details - Then the resolved details value should be "INTERNAL" - When the connection is lost - And a stale event was fired - When the flag was evaluated with details - Then the resolved details value should be "not" - When the connection is restored - And a ready event was fired - - @rpc @flagd-caching - Scenario: Use enriched context on RPC connection will not cache the value - Given a stable provider - And a String-flag with key "flagd-context-aware" and a default value "not" - And a change event handler - And a ready event handler - When the flag was modified - And a change event was fired - And the flag was evaluated with details - Then the reason should be "TARGETING_MATCH" - # ensure that we do not cache a "TARGETING_MATCH", we should only cache evaluation with a "STATIC" reason - When the flag was evaluated with details - Then the reason should be "TARGETING_MATCH" diff --git a/providers/flagd/src/test/resources/extensions/disabled.feature b/providers/flagd/src/test/resources/extensions/disabled.feature deleted file mode 100644 index 44a29337e3..0000000000 --- a/providers/flagd/src/test/resources/extensions/disabled.feature +++ /dev/null @@ -1,39 +0,0 @@ -@ext-evaluation @disabled @in-process @rpc -Feature: Disabled flag evaluation - - # Ported from flagd-testbed gherkin/disabled.feature. A flag with state=DISABLED resolves - # successfully with reason=DISABLED; the evaluator substitutes the caller-provided default value - # and omits the variant. Relates to: https://github.com/open-feature/flagd/issues/1965 - # - # The TCK's @disabled-flags scenarios ask the value for the four scalar flags and the reason for - # disabled-boolean-flag only, so the boolean row is left out here: the remaining rows add the - # DISABLED reason for each type and the structured flag. - - Scenario Outline: Evaluating a disabled flag returns reason DISABLED and code default - Given an option "cache" of type "CacheType" with value "disabled" - And a stable provider - And a -flag with key "" and a default value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "DISABLED" - - Examples: - | key | type | default | - | disabled-string-flag | String | bye | - | disabled-integer-flag | Integer | 1 | - | disabled-float-flag | Float | 0.1 | - | disabled-object-flag | Object | {} | - - Scenario Outline: Flag disabled in one flag set, enabled in another - Given an option "cache" of type "CacheType" with value "disabled" - And an option "selector" of type "String" with value "" - And a stable provider - And a Boolean-flag with key "cross-flagset-flag" and a default value "false" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "" - - Examples: - | selector | resolved_value | reason | - | flags/allFlags.json | false | DISABLED | - | rawflags/selector-flags.json | true | STATIC | diff --git a/providers/flagd/src/test/resources/extensions/evaluation.feature b/providers/flagd/src/test/resources/extensions/evaluation.feature deleted file mode 100644 index c32b00db43..0000000000 --- a/providers/flagd/src/test/resources/extensions/evaluation.feature +++ /dev/null @@ -1,76 +0,0 @@ -@ext-evaluation @rpc @in-process @file -Feature: flagd evaluations - - # Ported from flagd-testbed gherkin/evaluation.feature. Asks what the TCK's evaluation.feature - # and reason.feature do not: the reason flagd reports for a zero value, a float zero value, - # zero values served by a targeting rule, and flags with no default variant. - # - # Not ported: "Resolve values" (the TCK's evaluation.feature asks it), the @deprecated - # @no-default outline, which @no-default-variant replaces, and the two @precision scenarios, - # "Resolves numeric values without loss of precision" and "Resolves an integer beyond 32 bits - # without loss of precision", which the TCK's own precision scenarios ask. - - Background: - Given an option "cache" of type "CacheType" with value "disabled" - And a stable provider - - Scenario Outline: Resolves zero value - Given a -flag with key "" and a default value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "STATIC" - - Examples: - | key | type | default | resolved_value | - | boolean-zero-flag | Boolean | true | false | - | string-zero-flag | String | hi | | - | integer-zero-flag | Integer | 1 | 0 | - | float-zero-flag | Float | 0.1 | 0.0 | - - @targeting - Scenario Outline: Resolves zero value with targeting - Given a -flag with key "" and a default value "" - And a context containing a key "email", with type "String" and with value "ballmer@macrosoft.com" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "TARGETING_MATCH" - - Examples: - | key | type | default | resolved_value | - | boolean-targeted-zero-flag | Boolean | true | false | - | string-targeted-zero-flag | String | hi | | - | integer-targeted-zero-flag | Integer | 1 | 0 | - | float-targeted-zero-flag | Float | 0.1 | 0.0 | - - @targeting - Scenario Outline: Resolves zero value with targeting using default - Given a -flag with key "" and a default value "" - And a context containing a key "email", with type "String" and with value "ballmer@none.com" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "DEFAULT" - - Examples: - | key | type | default | resolved_value | - | boolean-targeted-zero-flag | Boolean | true | false | - | string-targeted-zero-flag | String | hi | | - | integer-targeted-zero-flag | Integer | 1 | 0 | - | float-targeted-zero-flag | Float | 0.1 | 0.0 | - - @no-default-variant - Scenario Outline: Resolves flag with no default variant correctly - Given a -flag with key "" and a default value "" - And a context containing a key "email", with type "String" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "" - - Examples: - | key | type | email | code_default | resolved_value | reason | - | null-default-flag | Boolean | | true | true | DEFAULT | - | null-default-flag | Boolean | | false | false | DEFAULT | - | undefined-default-flag | Integer | | 100 | 100 | DEFAULT | - | no-default-flag-null-targeting-variant | String | wozniak@orange.com | Inventor | Inventor | DEFAULT | - | no-default-flag-null-targeting-variant | String | wozniak@orange.com | Founder | Founder | DEFAULT | - | no-default-flag-null-targeting-variant | String | jobs@orange.com | CEO | CEO | TARGETING_MATCH | - | no-default-flag-undefined-targeting-variant | String | wozniak@orange.com | Retired | Retired | DEFAULT | diff --git a/providers/flagd/src/test/resources/extensions/events.feature b/providers/flagd/src/test/resources/extensions/events.feature deleted file mode 100644 index e37cd76f7d..0000000000 --- a/providers/flagd/src/test/resources/extensions/events.feature +++ /dev/null @@ -1,56 +0,0 @@ -@ext-connection @rpc @in-process @file @events -Feature: flagd provider state changes - - # Ported from flagd-testbed gherkin/events.feature. Not ported: "Flag change event", which the - # canonical events.feature asks in "A configuration change is signalled and applied", together - # with the changed evaluation it leaves out. - # - # These chains ask what the retry grace period decides, and they ask it by sequence rather than by - # clock: stale -> ready when the connection returns inside the grace period, stale -> error -> ready - # when the error assertion has first waited the grace period out. The testbed timed the outage - # instead (the launchpad's /restart), which made the same question depend on a 1s or 8s wall clock. - # - # Each "should have been executed" claims the record it matched and leaves the rest, so the next - # assertion waits for a newer event of its own kind while an event of another kind stays visible - # to whatever asks about it later. - # - # Timing is the testbed's "stable flagd provider" except for the deadline, which stays at the - # adoption's 15000ms for the in-process sync (see AbstractResolverTest): a 5s grace period, and a - # retry backoff capped at 500ms so reconnecting does not wait out an exponential backoff. - - Background: - Given an option "retryGracePeriod" of type "Integer" with value "5" - And an option "retryBackoffMaxMs" of type "Integer" with value "500" - And a stable provider - - Scenario: Provider events chain ready -> error -> ready - Given a ready event handler - And a error event handler - Then the ready event handler should have been executed - When the connection is lost - Then the error event handler should have been executed - When the connection is restored - Then the ready event handler should have been executed - - @grace - Scenario: Provider events chain ready -> stale -> error -> ready - Given a ready event handler - And a error event handler - And a stale event handler - Then the ready event handler should have been executed - When the connection is lost - Then the stale event handler should have been executed - Then the error event handler should have been executed - When the connection is restored - Then the ready event handler should have been executed - - @grace - Scenario: Provider events chain ready -> stale -> ready - Given a ready event handler - And a error event handler - And a stale event handler - Then the ready event handler should have been executed - When the connection is lost - Then the stale event handler should have been executed - When the connection is restored - Then the ready event handler should have been executed with no error event before it diff --git a/providers/flagd/src/test/resources/extensions/fractional.feature b/providers/flagd/src/test/resources/extensions/fractional.feature deleted file mode 100644 index 2444e98154..0000000000 --- a/providers/flagd/src/test/resources/extensions/fractional.feature +++ /dev/null @@ -1,525 +0,0 @@ -@ext-fractional @rpc @in-process @file -Feature: Fractional targeting - - # Ported from the flagd-testbed's gherkin/targeting.feature: every @fractional scenario, plus the - # fractional @operator-errors ones. The TCK's own @targeting scenarios do not cover the - # fractional operator. - # - # @fractional-vN tags select the bucketing version. The suites run v3, which the Java in-process - # evaluator and the testbed's flagd implement, and exclude @fractional-v2 with @TckExcludeTags. - # Untagged scenarios must hold for every version. - # The @fractional-v1 blocks are left out: no backend here implements v1 bucketing. - # - # Where the testbed's context values are Objects or integers outside 32 bits, the TCK's context - # step cannot convert them, so those outlines use the table step with the testbed's conversion. - - Background: - Given an option "cache" of type "CacheType" with value "disabled" - And a stable provider - - @fractional - Scenario Outline: Fractional operator - Given a String-flag with key "fractional-flag" and a default value "fallback" - And a context containing a nested property with outer key "user" and inner key "name", with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - - @fractional-v2 - Examples: v2 - | name | value | - | jack | hearts | - | queen | spades | - | ten | clubs | - | nine | diamonds | - | 3 | clubs | - - @fractional-v3 - Examples: v3 - | name | value | - | jack | diamonds | - | queen | diamonds | - | ten | clubs | - | nine | clubs | - | 3 | spades | - - @fractional - Scenario Outline: Fractional operator shorthand - Given a String-flag with key "fractional-flag-shorthand" and a default value "fallback" - And a context containing a targeting key with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - - @fractional-v2 - Examples: v2 - | targeting key | value | - | jon@company.com | heads | - | jane@company.com | tails | - - @fractional-v3 - Examples: v3 - | targeting key | value | - | jon@company.com | tails | - | jane@company.com | tails | - - @fractional @fractional-v2 @fractional-single-entry - Scenario: Fractional operator with single entry always resolves to the only variant - Given a String-flag with key "fractional-single-entry-flag" and a default value "fallback" - And a context containing a targeting key with value "some-targeting-key" - When the flag was evaluated with details - Then the resolved details value should be "single" - - @fractional - Scenario Outline: Fractional operator with shared seed - Given a String-flag with key "fractional-flag-A-shared-seed" and a default value "fallback" - And a context containing a nested property with outer key "user" and inner key "name", with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - - @fractional-v2 - Examples: v2 - | name | value | - | seven | hearts | - | eight | diamonds | - | nine | clubs | - | two | spades | - - @fractional-v3 - Examples: v3 - | name | value | - | seven | hearts | - | eight | hearts | - | nine | diamonds | - | two | diamonds | - - @fractional - Scenario Outline: Second fractional operator with shared seed - Given a String-flag with key "fractional-flag-B-shared-seed" and a default value "fallback" - And a context containing a nested property with outer key "user" and inner key "name", with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - - @fractional-v2 - Examples: v2 - | name | value | - | seven | ace-of-hearts | - | eight | ace-of-diamonds | - | nine | ace-of-clubs | - | two | ace-of-spades | - - @fractional-v3 - Examples: v3 - | name | value | - | seven | ace-of-hearts | - | eight | ace-of-hearts | - | nine | ace-of-diamonds | - | two | ace-of-diamonds | - - # Hash edge-case vectors — keys chosen by brute-force search so their - # MurmurHash3-x86-32 (seed=0) falls at the six critical boundary values. - # All keys are exact 6-char MurmurHash3-x86-32 (seed=0) preimages found by exhaustive search. - # ejOoVL → hash=0 EXACT → bv(100)=0 → lower - # bY9fO- → hash=1 EXACT → bv(100)=0 → lower - # SI7p- → hash=2147483647 EXACT i32::MAX → bv(100)=49 → lower - # 6LvT0 → hash=2147483648 EXACT i32::MIN u32 → bv(100)=50 → upper - # ceQdGm → hash=4294967295 EXACT u32::MAX → bv(100)=99 → upper - @fractional @fractional-v2 - Scenario Outline: Fractional operator hash edge cases - Given a String-flag with key "fractional-hash-edge-flag" and a default value "fallback" - And a context containing a targeting key with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - Examples: - | key | value | - | ejOoVL | lower | - | bY9fO- | lower | - | SI7p- | lower | - | 6LvT0 | upper | - | ceQdGm | upper | - - # Nested JSON Logic expressions as bucket variant names. - # Requires providers to support the @fractional-nested feature. - # Use -t "not @fractional-nested" to exclude during transition. - @fractional @fractional-nested - Scenario Outline: Fractional operator with nested if expression as variant name - # fractional-nested-if-flag: seed=targetingKey, bucket0=[if(tier=="premium","premium","standard"),50], bucket1=["standard",50] - # jon@company.com bv(100)=36 → bucket0; user1 bv(100)=76 → bucket1 - Given a String-flag with key "fractional-nested-if-flag" and a default value "fallback" - And a context containing a targeting key with value "" - And a context containing a key "tier", with type "String" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - Examples: - | targetingKey | tier | value | - | jon@company.com | premium | premium | - | jon@company.com | basic | standard | - | user1 | premium | standard | - | user1 | basic | standard | - - @fractional @fractional-nested - Scenario Outline: Fractional operator with nested var expression as variant name - # fractional-nested-var-flag: seed=targetingKey, bucket0=[var("color"),50], bucket1=["blue",50] - # jon@company.com bv(100)=36 → bucket0 (resolves var "color"); user1 bv(100)=76 → bucket1 ("blue") - Given a String-flag with key "fractional-nested-var-flag" and a default value "fallback" - And a context containing a targeting key with value "" - And a context containing a key "color", with type "String" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - Examples: - | targetingKey | color | value | - | jon@company.com | red | red | - | jon@company.com | green | green | - | user1 | red | blue | - | jon@company.com | yellow | fallback | - | jon@company.com | | fallback | - - @fractional @fractional-nested - Scenario Outline: Fractional operator with nested if expression as weight - # fractional-nested-weight-flag: seed=targetingKey, bucket0=["red",if(tier=="premium",100,0)], bucket1=["blue",10] - Given a String-flag with key "fractional-nested-weight-flag" and a default value "fallback" - And a context containing a targeting key with value "" - And a context containing a key "tier", with type "String" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - Examples: - | targetingKey | tier | value | - | jon@company.com | premium | red | - | jon@company.com | basic | blue | - | user1 | premium | red | - | user1 | basic | blue | - - @fractional @fractional-nested - Scenario: Fractional as condition - Given a String-flag with key "fractional-as-condition-flag" and a default value "zero" - And a context containing a targeting key with value "some-targeting-key" - When the flag was evaluated with details - Then the resolved details value should be "hundreds" - - @fractional @fractional-nested - Scenario: Fractional as condition evaluates false path - Given a String-flag with key "fractional-as-condition-false-flag" and a default value "zero" - And a context containing a targeting key with value "some-targeting-key" - When the flag was evaluated with details - Then the resolved details value should be "ones" - - # fractional composes like any other operator - @fractional @fractional-nested @fractional-v3 - Scenario Outline: Fractional with null bucket as whole rule - Given a String-flag with key "fractional-null-bucket-flag" and a default value "code-default" - And a context containing a targeting key with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "" - And the error-code should be "" - Examples: - | targetingKey | value | reason | - | user2 | fallback | DEFAULT | - | user1 | a | TARGETING_MATCH | - - @fractional @fractional-nested @fractional-v3 - Scenario Outline: Fractional with null bucket as if condition - Given a String-flag with key "fractional-null-bucket-condition-flag" and a default value "code-default" - And a context containing a targeting key with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "TARGETING_MATCH" - Examples: - | targetingKey | value | - | user1 | else | - | user2 | then | - - @fractional @fractional-nested @fractional-v3 - Scenario Outline: Fractional with boolean buckets as whole rule - Given a String-flag with key "fractional-bool-bucket-flag" and a default value "code-default" - And a context containing a targeting key with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the variant should be "" - And the reason should be "TARGETING_MATCH" - Examples: - | targetingKey | value | variant | - | user1 | on | true | - | user2 | off | false | - - @fractional @fractional-nested @fractional-v3 - Scenario Outline: Fractional result consumed by another operator - Given a String-flag with key "fractional-cat-flag" and a default value "code-default" - And a context containing a targeting key with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "TARGETING_MATCH" - Examples: - | targetingKey | value | - | user2 | big | - | user1 | dig | - - @fractional @fractional-nested @fractional-v3 - Scenario Outline: Fractional as bucketing input of another fractional - Given a String-flag with key "fractional-of-fractional-flag" and a default value "code-default" - And a context containing a targeting key with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "TARGETING_MATCH" - Examples: - | targetingKey | value | - | user2 | red | - | user3 | red | - | user1 | blue | - - @operator-errors - Scenario: fractional operator with missing bucket key falls back to default variant - Given a String-flag with key "fractional-null-bucket-key-flag" and a default value "wrong" - When the flag was evaluated with details - Then the resolved details value should be "fallback" - And the reason should be "DEFAULT" - - @operator-errors @fractional - Scenario: fractional with all-zero bucket weights falls back to default variant - Given a String-flag with key "fractional-zero-weights-flag" and a default value "wrong" - And a context containing a targeting key with value "any-user" - When the flag was evaluated with details - Then the resolved details value should be "fallback" - And the reason should be "DEFAULT" - - @operator-errors @fractional - Scenario: fractional negative bucket weight is clamped to zero - # ["one", -50] is treated as ["one", 0]; "two" gets 100% of the weight - Given a String-flag with key "fractional-negative-weight-flag" and a default value "wrong" - And a context containing a targeting key with value "any-user" - When the flag was evaluated with details - Then the resolved details value should be "two" - - @fractional @fractional-v3 - Scenario Outline: Fractional operator with basic types - Given a String-flag with key "fractional-basic-flag" and a default value "fallback" - And a context containing a key "hashing_input", with type "" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - - Examples: v3 - | hashing_input | type | value | - | true | Boolean | bucket5 | - | false | Boolean | bucket1 | - | user1 | String | bucket22 | - | user2 | String | bucket16 | - | 123 | Integer | bucket20 | - | 456 | Integer | bucket12 | - | 1.23 | Float | bucket14 | - | 4.56 | Float | bucket18 | - | 123 | String | bucket22 | - | true | String | bucket24 | - | false | String | bucket1 | - | null | String | bucket8 | - | 1.23 | String | bucket4 | - | 0 | Integer | bucket8 | - - @fractional @fractional-v3 - Scenario Outline: Fractional operator with float mapping - Given a String-flag with key "fractional-basic-flag" and a default value "fallback" - And a context with the following keys: - | key | type | value | - | hashing_input | | | - When the flag was evaluated with details - Then the resolved details value should be "" - - Examples: v3 - | hashing_input | type | value | - | 1.0 | Float | bucket23 | - | 1 | Integer | bucket23 | - | 1.0000000000000001 | Float | bucket23 | - | -2.0 | Float | bucket12 | - | -2 | Integer | bucket12 | - | 9007199254740992.0 | Float | bucket10 | - | 9007199254740992 | Integer | bucket10 | - - @fractional @fractional-v3 - Scenario Outline: Fractional operator with zero values - Given a String-flag with key "fractional-basic-flag" and a default value "fallback" - And a context containing a key "hashing_input", with type "" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - - Examples: v3 - | hashing_input | type | value | - | 0.0 | Float | bucket8 | - | -0.0 | Float | bucket8 | - | 0 | Integer | bucket8 | - - @fractional @fractional-v3 - Scenario Outline: Fractional operator with integer limits - Given a String-flag with key "fractional-basic-flag" and a default value "fallback" - And a context with the following keys: - | key | type | value | - | hashing_input | Integer | | - When the flag was evaluated with details - Then the resolved details value should be "" - - Examples: v3 - | hashing_input | value | - | 2147483647 | bucket10 | - | 2147483648 | bucket25 | - | -2147483648 | bucket8 | - | -2147483649 | bucket18 | - | 9007199254740991 | bucket2 | - | 23 | bucket21 | - | 24 | bucket11 | - | 255 | bucket13 | - | 256 | bucket13 | - | 65535 | bucket4 | - | 65536 | bucket11 | - | 4294967295 | bucket14 | - | 4294967296 | bucket19 | - | -24 | bucket4 | - | -25 | bucket16 | - - @fractional @fractional-v3 - Scenario Outline: Fractional operator with floats limits - Given a String-flag with key "fractional-basic-flag" and a default value "fallback" - And a context containing a key "hashing_input", with type "Float" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - - Examples: v3 - | hashing_input | value | - | 1e20 | bucket19 | - | -1e20 | bucket25 | - | 65504.0 | bucket16 | - | -65504.0 | bucket9 | - | 0.00006103515625 | bucket22 | - | 0.000000059604644775390625 | bucket23 | - | 65505.0 | bucket17 | - | -65505.0 | bucket17 | - | 3.4028234663852886e+38 | bucket10 | - | -3.4028234663852886e+38 | bucket3 | - | 1.1754943508222875e-38 | bucket24 | - | 3.5e+38 | bucket1 | - | -3.5e+38 | bucket6 | - | 1.7976931348623157e+308 | bucket17 | - | -1.7976931348623157e+308 | bucket9 | - | 2.2250738585072014e-308 | bucket11 | - | 4.9406564584124654e-324 | bucket2 | - - @fractional @fractional-v3 - Scenario Outline: Fractional operator with map key ordering - Given a String-flag with key "fractional-basic-flag" and a default value "fallback" - And a context with the following keys: - | key | type | value | - | hashing_input | Object | | - When the flag was evaluated with details - Then the resolved details value should be "" - - Examples: v3 - | hashing_input | value | - | {"a": 1, "b": 2} | bucket23 | - | {"b": 2, "a": 1} | bucket23 | - | {"a": 1, "b": 2, "c": 3} | bucket18 | - | {"c": 3, "a": 1, "b": 2} | bucket18 | - | {"z": 1, "aa": 2} | bucket21 | - | {"aa": 2, "z": 1} | bucket21 | - | {"a": {"b": 1}, "b": 2} | bucket9 | - | {"b": 2, "a": {"b": 1}} | bucket9 | - | {"a": true, "bbb": 1, "c": "text", "dddd": 2.5} | bucket25 | - | {"c": "text", "bbb": 1, "a": true, "dddd": 2.5} | bucket25 | - | {"aaa": 1, "ÿ": 2} | bucket12 | - | {"ÿ": 2, "aaa": 1} | bucket12 | - | {"": 2, "abc": 1} | bucket25 | - | {"key": -0.0} | bucket10 | - | {"key": 0} | bucket10 | - | {"key": 0.0} | bucket10 | - | {"b": -0.0, "a": {"b": 1}, "c": 0.0} | bucket8 | - | {"c": 0.0, "b": -0.0, "a": {"b": 1}} | bucket8 | - | {"café": "façade", "b": {"résumé": {"ÿ": true}}} | bucket22 | - - @fractional @fractional-v3 - Scenario Outline: Fractional operator with advanced structures - Given a String-flag with key "fractional-basic-flag" and a default value "fallback" - And a context with the following keys: - | key | type | value | - | hashing_input | Object | | - When the flag was evaluated with details - Then the resolved details value should be "" - - Examples: v3 - | hashing_input | value | - | {} | bucket3 | - | {"a": {} } | bucket20 | - | {"b": 2, "a": {} } | bucket20 | - | {"a": {}, "b": {}} | bucket21 | - | {"a": {}, "b": []} | bucket7 | - | {"c": 0.0, "a": []} | bucket18 | - | {"c": 0.0, "b": -0.0, "a": {"b": []}} | bucket9 | - | {"a": [] } | bucket12 | - | {"a": [1, [], 3], "b": null} | bucket3 | - | {"a": ["a", "b", {}], "b": [1, null, 2]} | bucket23 | - | {"a": [1, "two", true, [], null]} | bucket23 | - | {"a": [false, 2.5, ""], "b": [], "c": [{}], "c": null} | bucket15 | - | {"x": [{"a": 1}, {"b": 2}, {"b": null}]} | bucket3 | - | {"x": [{"a": {"b": []}}] } | bucket16 | - | {"x": [{"a": {"b": null}}] } | bucket6 | - | {"x": [{"a": {"b": {}}}] } | bucket2 | - | {"": "", "": [], "": {}, "": null} | bucket25 | - | {"": {"": ""}, "": ["", {"": ""}], "": {"": {"": ""}}} | bucket11 | - | {"a": [[[[[]]]]] } | bucket15 | - | {"a": [[[[null]]]] } | bucket7 | - | {"a": {"b": {"c": {"d": {}}} } } | bucket21 | - | {"a": {"b": {"c": {"d": null}} } } | bucket19 | - - @fractional @fractional-v3 - Scenario Outline: Fractional operator with string length boundaries - Given a String-flag with key "fractional-basic-flag" and a default value "fallback" - And a context containing a key "hashing_input", with type "String" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - - Examples: v3 - | hashing_input | value | - | | bucket4 | - | a | bucket6 | - | 12345678901234567890123 | bucket4 | - | 123456789012345678901234 | bucket14 | - | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | bucket18 | - | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | bucket7 | - - @fractional @fractional-v3 - Scenario Outline: Valid well-formed UTF-8 String consistency across different languages - Given a String-flag with key "fractional-basic-flag" and a default value "fallback" - And a context containing a key "hashing_input", with type "String" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - - Examples: v3 - | hashing_input | value | - | あいうえおかき | bucket17 | - | あいうえおかきく | bucket16 | - | ééééééééééé | bucket16 | - | éééééééééééé | bucket19 | - | café façade résumé café | bucket9 | - | A\u0000B | bucket15 | - | e\u0301 | bucket7 | - | \uD83D\uDE00 | bucket23 | - - @fractional @fractional-v3 - Scenario Outline: Fractional operator using implicit targeting key - Given a String-flag with key "fractional-flag-shorthand" and a default value "draw" - And a context containing a targeting key with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - - Examples: v3 - | targeting_key | value | - | user-1 | heads | - | user-2 | tails | - | user-3 | heads | - - @fractional @fractional-v3 - Scenario Outline: Fractional operator invalid implicit inputs - Given a String-flag with key "fractional-flag-shorthand" and a default value "fallback" - And a context containing a key "", with type "" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "draw" - Then the reason should be "DEFAULT" - - Examples: v3 - | key | type | value | - | irrelevant | String | none | - | targetingKey | Integer | 12345 | - | targetingKey | Boolean | true | diff --git a/providers/flagd/src/test/resources/extensions/metadata.feature b/providers/flagd/src/test/resources/extensions/metadata.feature deleted file mode 100644 index 73d26e3e67..0000000000 --- a/providers/flagd/src/test/resources/extensions/metadata.feature +++ /dev/null @@ -1,58 +0,0 @@ -@ext-metadata @in-process @file @metadata -Feature: flag and flag set metadata - - # Ported from flagd-testbed gherkin/metadata.feature. The testbed's "a stable flagd provider" - # becomes the TCK's "a stable provider", and its "a metadata flagd provider" becomes "a stable - # provider with "metadata" configuration". The two scenarios a file resolver cannot serve name a - # selector flag file and carry @metadata-provider. - - Scenario: Returns metadata - Given a stable provider - And a Boolean-flag with key "metadata-flag" and a default value "true" - When the flag was evaluated with details - Then the resolved metadata should contain - | key | metadata_type | value | - | string | String | a | - | integer | Integer | 1 | - | float | Float | 1.2 | - | boolean | Boolean | true | - - @metadata-provider - Scenario: Returns flag set metadata - Given an option "selector" of type "String" with value "rawflags/selector-flag-combined-metadata.json" - And a stable provider with "metadata" configuration - And a Boolean-flag with key "only-set-metadata-flag" and a default value "true" - When the flag was evaluated with details - Then the resolved metadata should contain - | key | metadata_type | value | - | string | String | b | - | integer | Integer | 2 | - | float | Float | 2.2 | - | boolean | Boolean | false | - | flag-set-string | String | c | - | flag-set-integer | Integer | 3 | - | flag-set-float | Float | 3.2 | - | flag-set-boolean | Boolean | false | - - @metadata-provider - Scenario: Flag metadata overwrites flag set metadata - Given an option "selector" of type "String" with value "rawflags/selector-flag-combined-metadata.json" - And a stable provider with "metadata" configuration - And a Boolean-flag with key "combined-metadata-flag" and a default value "true" - When the flag was evaluated with details - Then the resolved metadata should contain - | key | metadata_type | value | - | string | String | a | - | integer | Integer | 1 | - | float | Float | 1.2 | - | boolean | Boolean | true | - | flag-set-string | String | c | - | flag-set-integer | Integer | 3 | - | flag-set-float | Float | 3.2 | - | flag-set-boolean | Boolean | false | - - Scenario: Returns no metadata - Given a Boolean-flag with key "boolean-flag" and a default value "true" - And a stable provider - When the flag was evaluated with details - Then the resolved metadata is empty diff --git a/providers/flagd/src/test/resources/extensions/rpc-caching.feature b/providers/flagd/src/test/resources/extensions/rpc-caching.feature deleted file mode 100644 index a86f5697ac..0000000000 --- a/providers/flagd/src/test/resources/extensions/rpc-caching.feature +++ /dev/null @@ -1,68 +0,0 @@ -@ext-connection @rpc @flagd-caching -Feature: Flag evaluation with caching - - # Ported from flagd-testbed gherkin/rpc-caching.feature (RPC only). The testbed's @caching tag is - # a reserved TCK capability, which the TCK refuses on any scenario, so this feature carries - # @flagd-caching instead. - # - # Reconnect timing is the testbed's "stable flagd provider", as in events.feature. - - Background: - Given an option "cache" of type "CacheType" with value "lru" - And an option "retryGracePeriod" of type "Integer" with value "5" - And an option "retryBackoffMaxMs" of type "Integer" with value "500" - And a stable provider - - Scenario Outline: Resolves details with caching - Given a -flag with key "" and a default value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the variant should be "" - And the reason should be "STATIC" - When the flag was evaluated with details - Then the resolved details value should be "" - And the variant should be "" - And the reason should be "CACHED" - - Examples: - | key | type | default | resolved_variant | resolved_value | - | boolean-flag | Boolean | false | on | true | - | string-flag | String | bye | greeting | hi | - | integer-flag | Integer | 1 | ten | 10 | - | float-flag | Float | 0.1 | half | 0.5 | - | object-flag | Object | {} | template | {"showImages": true, "title": "Check out these pics!", "imagesPerPage": 100.0 } | - - Scenario: Flag change event with caching - Given a String-flag with key "changing-flag" and a default value "false" - And a change event handler - When the flag was modified - And a change event was fired - And the flag was evaluated with details - Then the reason should be "STATIC" - When the flag was evaluated with details - Then the reason should be "CACHED" - When the flag was modified - And a change event was fired - And the flag was evaluated with details - Then the reason should be "STATIC" - When the flag was evaluated with details - Then the reason should be "CACHED" - - Scenario: Stale and Error stage - Given a ready event handler - And a stale event handler - And a error event handler - And a Boolean-flag with key "boolean-flag" and a default value "false" - When the flag was evaluated with details - Then the reason should be "STATIC" - When the flag was evaluated with details - Then the reason should be "CACHED" - When the connection is lost - And a stale event was fired - And the flag was evaluated with details - Then the reason should be "CACHED" - When a error event was fired - And the connection is restored - And a ready event was fired - And the flag was evaluated with details - Then the reason should be "STATIC" diff --git a/providers/flagd/src/test/resources/extensions/selector.feature b/providers/flagd/src/test/resources/extensions/selector.feature deleted file mode 100644 index 8351c56ff7..0000000000 --- a/providers/flagd/src/test/resources/extensions/selector.feature +++ /dev/null @@ -1,27 +0,0 @@ -@ext-evaluation @in-process -Feature: flagd selector - - # Ported from flagd-testbed gherkin/selector.feature. The launchpad's default configuration - # serves both flags/allFlags.json and rawflags/selector-flags.json, so the selector narrows the - # sync to the second source. - - Background: - Given an option "selector" of type "String" with value "rawflags/selector-flags.json" - And a stable provider - - Scenario Outline: Flags not found - Given a -flag with key "" and a default value "" - When the flag was evaluated with details - Then the reason should be "ERROR" - - Examples: - | key | type | default | - | boolean-flag | Boolean | false | - | string-flag | String | bye | - | integer-flag | Integer | 1 | - | float-flag | Float | 0.1 | - - Scenario: Resolve values - Given a String-flag with key "selector-flag" and a default value "foo" - When the flag was evaluated with details - Then the reason should be "STATIC" diff --git a/providers/flagd/src/test/resources/extensions/sync-payload.feature b/providers/flagd/src/test/resources/extensions/sync-payload.feature deleted file mode 100644 index eb5c438159..0000000000 --- a/providers/flagd/src/test/resources/extensions/sync-payload.feature +++ /dev/null @@ -1,40 +0,0 @@ -@ext-metadata @in-process @sync-payload -Feature: Disabled Sync Metadata - - # Ported from flagd-testbed gherkin/sync-payload.feature. The testbed's "a syncpayload flagd - # provider" becomes "a stable provider with "sync-payload" configuration", moved from the - # Background into each scenario so the @grace one can record options first. - # - # The @grace scenario restores the testbed provider's retry timing (grace period 5s, backoff - # 500ms capped at 500ms) over the adoption's (grace 2s, backoff capped at the 5000ms default), so - # the provider is back within the event timeout once the connection is restored. - - Scenario: Use enriched context - Given a stable provider with "sync-payload" configuration - And a String-flag with key "flagd-context-aware" and a default value "not" - When the flag was evaluated with details - Then the resolved details value should be "INTERNAL" - - @grace - Scenario: Use enriched context on connection error for IN-PROCESS - Given an option "retryGracePeriod" of type "Integer" with value "5" - And an option "retryBackoffMs" of type "Integer" with value "500" - And an option "retryBackoffMaxMs" of type "Integer" with value "500" - And a stable provider with "sync-payload" configuration - And a String-flag with key "flagd-context-aware" and a default value "not" - And a stale event handler - And a ready event handler - When the flag was evaluated with details - Then the resolved details value should be "INTERNAL" - When the connection is lost - And a stale event was fired - When the flag was evaluated with details - Then the resolved details value should be "INTERNAL" - When the connection is restored - And a ready event was fired - - Scenario: Resolve value independent of context - Given a stable provider with "sync-payload" configuration - And a Boolean-flag with key "boolean-flag" and a default value "false" - When the flag was evaluated with details - Then the resolved details value should be "true" diff --git a/providers/flagd/src/test/resources/extensions/targeting.feature b/providers/flagd/src/test/resources/extensions/targeting.feature deleted file mode 100644 index e8556b053a..0000000000 --- a/providers/flagd/src/test/resources/extensions/targeting.feature +++ /dev/null @@ -1,222 +0,0 @@ -@ext-targeting @rpc @in-process @file -Feature: Targeting rules - - # Ported from flagd-testbed's gherkin/targeting.feature: the json-evaluation scenarios the Provider - # TCK does not ask. The @fractional scenarios live in their own extension feature. - # - # Not ported: - # - "Targeting by targeting key": the TCK's evaluation.feature (@targeting) asserts both values - # and reason.feature asserts TARGETING_MATCH and DEFAULT for the same flag and keys. - # - "fractional operator with missing bucket key": a fractional scenario, despite its tags. - # - # Deviations from the testbed's wording: - # - "a stable flagd provider" collides with the TCK's "a {} provider", so the provider step is the - # TCK's "a stable provider". - # - "Time-based operations" sets the context with "a context with the following keys:", because the - # TCK's context step parses Integer as 32 bits and 4133980802 does not fit. - - Background: - Given an option "cache" of type "CacheType" with value "disabled" - And a stable provider - - # evaluator refs - Scenario Outline: Evaluator reuse - Given a String-flag with key "" and a default value "fallback" - And a context containing a key "email", with type "String" and with value "ballmer@macrosoft.com" - When the flag was evaluated with details - Then the resolved details value should be "" - Examples: - | key | value | - | some-email-targeted-flag | hi | - | some-other-email-targeted-flag | yes | - - @evaluator-refs @evaluator-refs-whitespace - Scenario Outline: Evaluator $ref resolves regardless of whitespace around the colon - Given a String-flag with key "" and a default value "fallback" - And a context containing a key "email", with type "String" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - And the reason should be "TARGETING_MATCH" - Examples: - | key | email | value | - | ref-whitespace-compact-flag | ballmer@macrosoft.com | hi | - | ref-whitespace-space-after-colon-flag | ballmer@macrosoft.com | hi | - | ref-whitespace-space-around-colon-flag | ballmer@macrosoft.com | hi | - | ref-whitespace-compact-flag | user@example.com | bye | - | ref-whitespace-space-after-colon-flag | user@example.com | bye | - | ref-whitespace-space-around-colon-flag | user@example.com | bye | - - @evaluator-refs @non-existent-evaluator-ref - Scenario: Ref to nonexistent evaluator yields parse error - Given a String-flag with key "ref-to-nonexistent-evaluator-flag" and a default value "fallback" - When the flag was evaluated with details - Then the resolved details value should be "fallback" - And the error-code should be "PARSE_ERROR" - - @string - Scenario Outline: Substring operators - Given a String-flag with key "starts-ends-flag" and a default value "fallback" - And a context containing a key "id", with type "String" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - Examples: - | id | value | - | abcdef | prefix | - | uvwxyz | postfix | - | abcxyz | prefix | - | lmnopq | none | - | 3 | none | - - @semver - Scenario Outline: Semantic version operator numeric comparison - Given a String-flag with key "equal-greater-lesser-version-flag" and a default value "fallback" - And a context containing a key "version", with type "String" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - Examples: - | version | value | - | 2.0.0 | equal | - | 2.1.0 | greater | - | 1.9.0 | lesser | - | 2.0.0-alpha | lesser | - | 2.0.0.0 | none | - - @semver - Scenario Outline: Semantic version operator semantic comparison - Given a String-flag with key "major-minor-version-flag" and a default value "fallback" - And a context containing a key "version", with type "String" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - Examples: - | version | value | - | 3.0.1 | minor | - | 3.1.0 | major | - | 4.0.0 | none | - - @semver @semver-edge-cases @semver-v-prefix - Scenario Outline: sem_ver v-prefix handling - Given a String-flag with key "semver-v-prefix-flag" and a default value "fallback" - And a context containing a key "version", with type "String" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - Examples: - | version | value | - | 1.0.0 | match | - | v1.0.0 | match | - | V1.0.0 | match | - | 2.0.0 | no-match | - - @semver @semver-edge-cases @semver-partial-version - Scenario Outline: sem_ver partial version string handling - Given a String-flag with key "semver-partial-version-flag" and a default value "fallback" - And a context containing a key "version", with type "" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - Examples: - | version | type | value | - | 1.5.0 | String | match | - | 1.0.0 | String | match | - | 1.0 | String | match | - | 1 | String | match | - | 1 | Integer | match | - | 1 | Float | match | - | 1.2 | Float | match | - | 2.0.0 | String | no-match | - - @semver @semver-edge-cases @semver-numeric-context - Scenario Outline: sem_ver numeric context value coercion - Given a String-flag with key "semver-numeric-context-flag" and a default value "fallback" - And a context containing a key "version", with type "" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - Examples: - | version | type | value | - | 1.2 | Float | match | - | 1.1 | Float | no-match | - | 2 | Integer | match | - | 1 | Integer | no-match | - | 1.2 | String | match | - - @semver @semver-edge-cases @semver-build-metadata - Scenario Outline: sem_ver build metadata ignored - Given a String-flag with key "semver-build-metadata-flag" and a default value "fallback" - And a context containing a key "version", with type "String" and with value "" - When the flag was evaluated with details - Then the resolved details value should be "" - Examples: - | version | value | - | 1.0.0 | match | - | 1.0.0+other | match | - | 2.0.0 | no-match | - - Scenario Outline: Time-based operations - Given a Integer-flag with key "timestamp-flag" and a default value "0" - And a context with the following keys: - | key | type | value | - | time | Integer |