diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunFileTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunFileTest.java index c7387b6e33..551812c0bc 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunFileTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunFileTest.java @@ -2,6 +2,7 @@ import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; import static io.cucumber.junit.platform.engine.Constants.OBJECT_FACTORY_PROPERTY_NAME; +import static io.cucumber.junit.platform.engine.Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME; import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; import dev.openfeature.contrib.providers.flagd.Config; @@ -41,6 +42,15 @@ // reads 2^53-1 via 32-bit getInteger; Long scenarios: open-feature/flagd-testbed#379 "large-integers" }) +// Only the switch: the strategy and factor stay at Cucumber's defaults, so +// -Dcucumber.execution.parallel.config.* can still tune them. +@ConfigurationParameter(key = PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, value = "true") +// Concurrent container restarts can outlast the reconnection timeout. +@ConfigurationParameter(key = "cucumber.execution.exclusive-resources.grace.read-write", value = "CONTAINER_RESTART") +// @env-var scenarios mutate process-wide variables that every config scenario reads. +@ConfigurationParameter( + key = "cucumber.execution.exclusive-resources.env-var.read-write", + value = "org.junit.platform.engine.support.hierarchical.ExclusiveResource.GLOBAL_KEY") @Testcontainers public class RunFileTest { diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunInProcessTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunInProcessTest.java index 05eb219db1..10bfa75ae6 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunInProcessTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunInProcessTest.java @@ -2,6 +2,7 @@ import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; import static io.cucumber.junit.platform.engine.Constants.OBJECT_FACTORY_PROPERTY_NAME; +import static io.cucumber.junit.platform.engine.Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME; import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; import dev.openfeature.contrib.providers.flagd.Config; @@ -30,6 +31,15 @@ @IncludeTags("in-process") // large-integers reads 2^53-1 via 32-bit getInteger; Long scenarios: open-feature/flagd-testbed#379 @ExcludeTags({"unixsocket", "fractional-v1", "fractional-v2", "deprecated", "large-integers"}) +// Only the switch: the strategy and factor stay at Cucumber's defaults, so +// -Dcucumber.execution.parallel.config.* can still tune them. +@ConfigurationParameter(key = PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, value = "true") +// Concurrent container restarts can outlast the reconnection timeout. +@ConfigurationParameter(key = "cucumber.execution.exclusive-resources.grace.read-write", value = "CONTAINER_RESTART") +// @env-var scenarios mutate process-wide variables that every config scenario reads. +@ConfigurationParameter( + key = "cucumber.execution.exclusive-resources.env-var.read-write", + value = "org.junit.platform.engine.support.hierarchical.ExclusiveResource.GLOBAL_KEY") @Testcontainers public class RunInProcessTest { diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunRpcTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunRpcTest.java index 4fe721d093..d2eb856516 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunRpcTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunRpcTest.java @@ -2,6 +2,7 @@ import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; import static io.cucumber.junit.platform.engine.Constants.OBJECT_FACTORY_PROPERTY_NAME; +import static io.cucumber.junit.platform.engine.Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME; import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; import dev.openfeature.contrib.providers.flagd.Config; @@ -30,6 +31,15 @@ @IncludeTags({"rpc"}) // large-integers reads 2^53-1 via 32-bit getInteger; Long scenarios: open-feature/flagd-testbed#379 @ExcludeTags({"unixsocket", "fractional-v1", "fractional-v2", "deprecated", "large-integers"}) +// Only the switch: the strategy and factor stay at Cucumber's defaults, so +// -Dcucumber.execution.parallel.config.* can still tune them. +@ConfigurationParameter(key = PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, value = "true") +// Concurrent container restarts can outlast the reconnection timeout. +@ConfigurationParameter(key = "cucumber.execution.exclusive-resources.grace.read-write", value = "CONTAINER_RESTART") +// @env-var scenarios mutate process-wide variables that every config scenario reads. +@ConfigurationParameter( + key = "cucumber.execution.exclusive-resources.env-var.read-write", + value = "org.junit.platform.engine.support.hierarchical.ExclusiveResource.GLOBAL_KEY") @Testcontainers public class RunRpcTest { diff --git a/providers/flagd/src/test/resources/junit-platform.properties b/providers/flagd/src/test/resources/junit-platform.properties deleted file mode 100644 index cc6bbf8f62..0000000000 --- a/providers/flagd/src/test/resources/junit-platform.properties +++ /dev/null @@ -1,21 +0,0 @@ -# Enable parallel scenario execution within each suite runner. -# Each scenario borrows its own ContainerEntry from ContainerPool, so -# concurrent scenarios are fully isolated — no shared flagd process. -cucumber.execution.parallel.enabled=true -# Dynamic strategy scales with available CPUs (factor=1.0 → 1 thread per core). -# ContainerPool caps at min(availableProcessors, 4) containers so Docker isn't -# overwhelmed; extra threads simply block waiting for a free container. -# Override pool size via -Dflagd.e2e.pool.size=N if needed. -cucumber.execution.parallel.config.strategy=dynamic -cucumber.execution.parallel.config.dynamic.factor=1 -# Scenarios tagged @env-var mutate System env vars globally, which every config -# scenario reads when building FlagdOptions defaults. A plain ENV_VARS lock only -# serialises @env-var scenarios against each other, so a concurrent default-reading -# scenario could still observe a mutated var. Acquire the JUnit global lock so -# @env-var scenarios run in full isolation (no other scenario runs concurrently). -cucumber.execution.exclusive-resources.env-var.read-write=org.junit.platform.engine.support.hierarchical.ExclusiveResource.GLOBAL_KEY -# Scenarios tagged @grace involve container restart + reconnection timing. -# Running two concurrent restarts under parallel load can push the -# reconnection past the 12-second EVENT_TIMEOUT_MS threshold. Serialise -# them so each restart has the full machine resources to itself. -cucumber.execution.exclusive-resources.grace.read-write=CONTAINER_RESTART diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTck.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTck.java index 1c14985cd2..4b832ad3af 100644 --- a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTck.java +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTck.java @@ -29,9 +29,6 @@ public final class ProviderTck { /** The Cucumber plugins the suite registers. */ static final String PLUGINS = "summary"; - /** Parallel execution off; control API state is global to the backend. */ - static final String PARALLEL = "false"; - /** Scenarios of one feature stay on one thread. */ static final String FEATURE_EXECUTION_MODE = "same_thread"; diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckTest.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckTest.java index 31da2bda55..2f9e12d0d5 100644 --- a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckTest.java +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/ProviderTckTest.java @@ -27,7 +27,6 @@ @ConfigurationParameter( key = Constants.EXECUTION_MODE_FEATURE_PROPERTY_NAME, value = ProviderTck.FEATURE_EXECUTION_MODE) -@ConfigurationParameter(key = Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, value = ProviderTck.PARALLEL) @ConfigurationParameter(key = Constants.GLUE_PROPERTY_NAME, value = ProviderTck.ALL_GLUE) @ConfigurationParameter(key = Constants.OBJECT_FACTORY_PROPERTY_NAME, value = ProviderTck.OBJECT_FACTORY) public abstract class ProviderTckTest implements ProviderTckHarness {} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/ProviderSteps.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/ProviderSteps.java index 8bae27b35c..2ab74f4b1a 100644 --- a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/ProviderSteps.java +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/ProviderSteps.java @@ -74,7 +74,13 @@ public void gateOnCapabilities(Scenario scenario) { */ @Before(order = 10) public void prepareBackend() { - backend().prepareScenario(); + try { + backend().prepareScenario(); + } catch (RuntimeException e) { + // the object factory only learns that a scenario ended, never why + StackPool.markLeaseBroken(); + throw e; + } } /** @@ -83,7 +89,8 @@ public void prepareBackend() { @After public void tearDown() { if (state.domain != null) { - OpenFeatureAPI.getInstance().setProvider(state.domain, new NoOpProvider()); + // awaited: the backend goes to the next scenario as soon as this returns + OpenFeatureAPI.getInstance().setProviderAndWait(state.domain, new NoOpProvider()); } } diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/StackPool.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/StackPool.java new file mode 100644 index 0000000000..477da6fd8a --- /dev/null +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/StackPool.java @@ -0,0 +1,171 @@ +package dev.openfeature.contrib.tools.tck.internal.steps; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Backends warmed once and leased to one scenario at a time. + * + *

Off by default: with no size set, every {@link TckObjectFactory} starts one backend and keeps + * it for as long as its thread lives. The size caps how many stand at once, which a thread count + * cannot do: {@code cucumber.execution.parallel.config.fixed.parallelism} is a target, not a limit. + * Surplus threads queue on {@link #acquire()}. + */ +final class StackPool { + + /** How many backends to warm; absent or zero keeps one backend per thread. */ + static final String SIZE_PROPERTY = "openfeature.tck.pool.size"; + + /** Every lease is returned by {@code stop()}, so waiting this long means one was lost. */ + private static final long ACQUIRE_TIMEOUT_MINUTES = 10; + + private static final Logger log = LoggerFactory.getLogger(StackPool.class); + + private static final BlockingQueue idle = new LinkedBlockingQueue<>(); + + private static final ThreadLocal leaseBroken = ThreadLocal.withInitial(() -> false); + + private static volatile int size; + + /** Backends still fit to lease; drops lower it, so waiting scenarios can fail at zero. */ + private static final AtomicInteger live = new AtomicInteger(); + + private StackPool() { + // warmed and drained by TckLifecycle, leased by TckObjectFactory + } + + /** Whether scenarios lease backends from this pool. */ + static boolean enabled() { + return size > 0; + } + + /** + * Starts the requested number of backends, concurrently, or does nothing when none was asked + * for. + * + * @param start supplies one started backend, as the per-thread path does + * @throws IllegalStateException if any backend fails to start + */ + static void warm(Supplier start) { + int requested = requestedSize(); + if (requested <= 0) { + size = 0; + return; + } + + log.info("Warming {} backends ({}={})", requested, SIZE_PROPERTY, requested); + ExecutorService starters = Executors.newFixedThreadPool(requested); + List> futures = new ArrayList<>(); + try { + for (int i = 0; i < requested; i++) { + futures.add(starters.submit(start::get)); + } + List warmed = new ArrayList<>(); + RuntimeException failure = null; + for (Future future : futures) { + try { + warmed.add(future.get()); + } catch (ExecutionException e) { + RuntimeException cause = e.getCause() instanceof RuntimeException + ? (RuntimeException) e.getCause() + : new IllegalStateException(e.getCause()); + failure = failure == null ? cause : failure; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while warming the backend pool", e); + } + } + if (failure != null) { + // failing beats letting scenarios queue for backends that will never arrive + throw new IllegalStateException( + "only " + warmed.size() + " of " + requested + " backends started", failure); + } + idle.addAll(warmed); + size = requested; + live.set(requested); + } finally { + starters.shutdownNow(); + } + } + + /** Takes a backend for one scenario, waiting while every one of them is in use. */ + static TckLifecycle acquire() { + leaseBroken.set(false); + long deadline = System.nanoTime() + TimeUnit.MINUTES.toNanos(ACQUIRE_TIMEOUT_MINUTES); + try { + while (System.nanoTime() < deadline) { + if (live.get() <= 0) { + throw new IllegalStateException("every backend was dropped after its preparation failed (" + + SIZE_PROPERTY + "=" + size + "); there is nothing left to lease"); + } + // in steps, so a scenario does not wait out the whole deadline for a pool that emptied + TckLifecycle leased = idle.poll(1, TimeUnit.SECONDS); + if (leased != null) { + return leased; + } + } + throw new IllegalStateException("no backend became free within " + ACQUIRE_TIMEOUT_MINUTES + + " minutes, with " + live.get() + " of " + size + " in the pool (" + SIZE_PROPERTY + + "); a lease was not returned"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted while waiting for a backend", e); + } + } + + /** Returns a leased backend, dropping it if the scenario reported it broken. */ + static void release(TckLifecycle lifecycle) { + if (Boolean.TRUE.equals(leaseBroken.get())) { + leaseBroken.set(false); + log.warn( + "Dropping a backend whose preparation failed; {} of {} left in the pool", + live.decrementAndGet(), + size); + return; + } + idle.add(lifecycle); + } + + /** Marks the backend this thread holds as unfit to pass on; called where the breakage shows. */ + static void markLeaseBroken() { + if (enabled()) { + leaseBroken.set(true); + } + } + + /** Forgets every lease; the backends themselves are stopped by {@code endSuite()}. */ + static void drain() { + idle.clear(); + size = 0; + live.set(0); + } + + private static int requestedSize() { + String raw = System.getProperty(SIZE_PROPERTY); + if (raw == null || raw.trim().isEmpty()) { + return 0; + } + try { + int parsed = Integer.parseInt(raw.trim()); + if (parsed < 0) { + throw new IllegalArgumentException( + SIZE_PROPERTY + " is " + raw + "; a pool cannot hold fewer than no backends."); + } + return parsed; + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + SIZE_PROPERTY + " is '" + raw + "', which is not a number of backends.", e); + } + } +} diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckLifecycle.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckLifecycle.java index b3f094f80d..dbdbdcedcb 100644 --- a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckLifecycle.java +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckLifecycle.java @@ -51,12 +51,16 @@ static void beginSuite() { AdopterTags.requireValid(harness.getClass()); startFailure.set(null); suiteRunning = true; + // after the declarations are validated: a suite that cannot run should start no stacks + StackPool.warm(() -> start(discoverHarness())); } - /** Stops every lifecycle this suite started. */ + /** Stops every lifecycle this suite started, pooled or per-thread. */ static void endSuite() { suiteRunning = false; startFailure.set(null); + // the pooled backends are in `started` too, so dropping the leases is all this needs + StackPool.drain(); List toStop; synchronized (started) { toStop = new ArrayList<>(started); diff --git a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckObjectFactory.java b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckObjectFactory.java index 046e31037d..a44d80f65d 100644 --- a/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckObjectFactory.java +++ b/tools/tck/src/main/java/dev/openfeature/contrib/tools/tck/internal/steps/TckObjectFactory.java @@ -23,6 +23,9 @@ public final class TckObjectFactory implements ObjectFactory { private TckLifecycle lifecycle; + /** The pooled backend this scenario holds, or null when backends are per-thread. */ + private TckLifecycle leased; + @Override public boolean addClass(Class glueClass) { if (isInstantiable(glueClass) && glue.add(glueClass)) { @@ -60,7 +63,11 @@ private static boolean isInstantiable(Class candidate) { @Override public void start() { pico = new PicoBuilder().withCaching().build(); - if (lifecycle == null) { + if (StackPool.enabled()) { + // the pool's size, not this thread's existence, decides how many stand at once + leased = StackPool.acquire(); + lifecycle = leased; + } else if (lifecycle == null) { // empty when a launcher selects only the extension glue and never reaches @BeforeAll lifecycle = TckLifecycle.startForExecutingSuite().orElse(null); } @@ -76,10 +83,19 @@ public void start() { @Override public void stop() { - if (pico != null) { - pico.stop(); - pico.dispose(); - pico = null; + try { + if (pico != null) { + pico.stop(); + pico.dispose(); + pico = null; + } + } finally { + if (leased != null) { + // after the container is disposed, so nothing can still be using the backend + StackPool.release(leased); + leased = null; + lifecycle = null; + } } }