diff --git a/providers/openfeature-provider-flagd/README.md b/providers/openfeature-provider-flagd/README.md index 4203fb068..8925f5bd1 100644 --- a/providers/openfeature-provider-flagd/README.md +++ b/providers/openfeature-provider-flagd/README.md @@ -229,6 +229,42 @@ api.set_provider( ) ``` +## Provider conformance suite + +This provider runs the [OpenFeature Provider Conformance Suite][tck] against a flagd-testbed stack, +once per resolver, in `tests/tck`. Each run adds flagd-testbed's own scenarios, `tck-extensions/` in +the `openfeature/test-harness` submodule, to the canonical ones; `tests/tck/steps.py` defines the +flagd steps they use. The stack is the testbed's `docker-compose.yaml`, so the image follows the +submodule pin. + +`config.feature` is not part of that run: it needs no backend, and `tests/test_config_feature.py` +runs it in the default build. + +**It is excluded from the default build, and a maintainer runs it by hand before merging a change to +it.** + +``` +poe test-tck # both resolvers, needs Docker +poe test # everything else, which is what CI runs +``` + +The exclusion lives in `pyproject.toml`: `--ignore=tests/tck` on the two tasks `build.yml` reaches, +with the reason in a comment above them. Why a conformance suite is not a required gate is +[Appendix F, "Running the suite in CI"][appendix-f], and is not restated here. + +Two things that are this provider's rather than the policy's: + +- **The run is not green.** It is **5 failed, 648 passed, 75 skipped**, and + `tests/tck/conftest.py` accounts for each failure, so a reviewer running the suite can tell a new + failure from a known one. +- **The default build still collects the suite** — `poe test` and `poe test-cov` end in + `pytest tests/tck --collect-only`, which imports every module and starts no container. An excluded + suite that has quietly stopped importing against the harness is worse than one that runs and + fails, and `mypy` here is configured over `src` alone, so nothing else would notice. + +[tck]: ../../tools/openfeature-tck/README.md +[appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md + ## License Apache 2.0 - See [LICENSE](./LICENSE) for more information. diff --git a/providers/openfeature-provider-flagd/openfeature/test-harness b/providers/openfeature-provider-flagd/openfeature/test-harness index 2f8f72176..c5ac5de57 160000 --- a/providers/openfeature-provider-flagd/openfeature/test-harness +++ b/providers/openfeature-provider-flagd/openfeature/test-harness @@ -1 +1 @@ -Subproject commit 2f8f72176b99d85a31b23cd331a775a7c3015423 +Subproject commit c5ac5de5730f79a294b9af186aa069e21291ed55 diff --git a/providers/openfeature-provider-flagd/pyproject.toml b/providers/openfeature-provider-flagd/pyproject.toml index 503879384..d3c885015 100644 --- a/providers/openfeature-provider-flagd/pyproject.toml +++ b/providers/openfeature-provider-flagd/pyproject.toml @@ -31,13 +31,18 @@ Homepage = "https://github.com/open-feature/python-sdk-contrib" [dependency-groups] dev = [ - "asserts>=0.14.0,<0.15.0", "coverage[toml]>=7.10.0,<8.0.0", "grpcio-health-checking>=1.82.1,<2.0.0", "mypy>=1.18.0,<2.0.0", # The tests call `api.set_provider_and_wait`, which arrived in 0.10.0. The # runtime floor stays at 0.8.2 because the provider itself needs nothing newer. "openfeature-sdk>=0.10.0", + # The OpenFeature conformance suite. Ships the feature files, the flag set, + # the control-API client and the Compose harness that owns the container + # stack, and registers its step definitions through a pytest11 entry point -- + # so tests/tck declares a Compose file and nothing else. The `compose` extra + # is what pulls testcontainers in for the harness. + "openfeature-tck[compose]", "poethepoet>=0.37.0", "pytest>=9.0.0,<10.0.0", "pytest-bdd>=8.1.0,<9.0.0", @@ -115,13 +120,54 @@ warn_unused_ignores = false [tool.uv.sources] openfeature-flagd-core = { workspace = true } +openfeature-tck = { workspace = true } [tool.pytest] strict = true [tool.poe.tasks] -test = "pytest tests" -test-cov = "coverage run -m pytest tests" +# `tests/tck` is excluded from the default build on purpose, and a maintainer +# runs `poe test-tck` by hand before merging a change to it. Why a conformance +# suite is not a required gate is Appendix F, "Running the suite in CI" -- +# linked from tools/openfeature-tck/README.md -- and is deliberately not +# restated here. +# +# What is local to this package: a full run is 5 failed, 648 passed, 75 +# skipped, and tests/tck/conftest.py accounts for every failure. +# +# The suite is still *collected* on every default build, so it cannot quietly +# stop importing against the harness while nobody runs it. `--ignore` would +# otherwise leave nothing checking that, since mypy here is configured over +# `src` alone. +# +# `ignore_fail = "return_non_zero"` is what makes that true rather than +# aspirational: poe aborts a sequence at its first failing subtask, and this +# package's default suite is red on this branch -- the TCK requires +# openfeature-sdk >= 0.10 and this provider is not updated for it yet -- so the +# collect step sat behind a step that never succeeds and never ran. Every +# subtask now runs and a non-zero exit still propagates. +test = { sequence = ["test-default", "test-tck-collect"], ignore_fail = "return_non_zero" } +test-cov = { sequence = ["test-cov-default", "test-tck-collect"], ignore_fail = "return_non_zero" } +test-default = "pytest tests --ignore=tests/tck" +test-cov-default = "coverage run -m pytest tests --ignore=tests/tck" +# The sync is part of the run rather than something a maintainer remembers, +# because Appendix F requires the revision check to be in force **where the +# scenarios execute** and not only in the TCK implementation's own tests. This +# is a uv workspace, so `openfeature-tck` is installed editable and the +# canonical assets this suite collects are the gitignored copies under +# `tools/openfeature-tck/src`: a rebase that moves the submodule pin leaves them +# from the previous revision, and a suite run from here would neither notice nor +# say so. That is exactly how an adoption in another language published a full +# pass against stale scenarios. `hatch_build_sync.py` checks the submodule out +# at the pin, refuses to copy if it cannot, and needs nothing but the standard +# library. +# +# Not on `test-tck-collect`, deliberately: that runs on every default build, it +# executes no scenario, and it publishes no result, so the rule does not reach +# it and the default build stays free of a submodule operation. +sync-spec-assets = "python ../../tools/openfeature-tck/hatch_build_sync.py" +test-tck = ["sync-spec-assets", {cmd = "pytest tests/tck"}] +test-tck-collect = "pytest tests/tck --collect-only -q" cov-report = "coverage xml" cov = [ "test-cov", diff --git a/providers/openfeature-provider-flagd/tests/e2e/conftest.py b/providers/openfeature-provider-flagd/tests/e2e/conftest.py deleted file mode 100644 index e47ae120b..000000000 --- a/providers/openfeature-provider-flagd/tests/e2e/conftest.py +++ /dev/null @@ -1,5 +0,0 @@ -from tests.e2e.step.config_steps import * # noqa: F403 -from tests.e2e.step.context_steps import * # noqa: F403 -from tests.e2e.step.event_steps import * # noqa: F403 -from tests.e2e.step.flag_step import * # noqa: F403 -from tests.e2e.step.provider_steps import * # noqa: F403 diff --git a/providers/openfeature-provider-flagd/tests/e2e/file/__init__.py b/providers/openfeature-provider-flagd/tests/e2e/file/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/providers/openfeature-provider-flagd/tests/e2e/file/conftest.py b/providers/openfeature-provider-flagd/tests/e2e/file/conftest.py deleted file mode 100644 index 048108d60..000000000 --- a/providers/openfeature-provider-flagd/tests/e2e/file/conftest.py +++ /dev/null @@ -1,31 +0,0 @@ -import pytest - -from openfeature.contrib.provider.flagd.config import ResolverType -from tests.e2e.testfilter import TestFilter - -resolver = ResolverType.FILE -feature_list = { - "~targetURI", - "~customCert", - "~unixsocket", - "~reconnect", - "~sync", - "~caching", - "~grace", - "~contextEnrichment", - "~deprecated", - "~fractional-v1", - "~fractional-v2", -} - - -def pytest_collection_modifyitems(config, items): - test_filter = TestFilter( - config, feature_list=feature_list, resolver=resolver.value, base_path=__file__ - ) - test_filter.filter_items(items) - - -@pytest.fixture() -def resolver_type() -> ResolverType: - return resolver diff --git a/providers/openfeature-provider-flagd/tests/e2e/file/test_flaqd.py b/providers/openfeature-provider-flagd/tests/e2e/file/test_flaqd.py deleted file mode 100644 index 3feb4c523..000000000 --- a/providers/openfeature-provider-flagd/tests/e2e/file/test_flaqd.py +++ /dev/null @@ -1,5 +0,0 @@ -from pytest_bdd import scenarios - -from tests.e2e.paths import TEST_HARNESS_PATH - -scenarios(f"{TEST_HARNESS_PATH}/gherkin") diff --git a/providers/openfeature-provider-flagd/tests/e2e/flagd_container.py b/providers/openfeature-provider-flagd/tests/e2e/flagd_container.py deleted file mode 100644 index f47433e09..000000000 --- a/providers/openfeature-provider-flagd/tests/e2e/flagd_container.py +++ /dev/null @@ -1,104 +0,0 @@ -import logging -import os -import time -from pathlib import Path - -import grpc -from grpc_health.v1 import health_pb2, health_pb2_grpc -from testcontainers.compose import DockerCompose - -from openfeature.contrib.provider.flagd.config import ResolverType - -logger = logging.getLogger(__name__) - -HEALTH_CHECK = 8014 -LAUNCHPAD = 8080 -FORBIDDEN = 9212 - - -class FlagdContainer: - """Manages the docker-compose environment for flagd e2e tests. - - Uses docker-compose to start both flagd and envoy containers, - so the envoy forbidden endpoint (port 9212) returns a proper HTTP 403. - """ - - def __init__( - self, - feature: str | None = None, - **kwargs, - ) -> None: - self._test_harness_dir = ( - Path(__file__).parents[2] / "openfeature" / "test-harness" - ) - self._version = (self._test_harness_dir / "version.txt").read_text().rstrip() - - image: str = "ghcr.io/open-feature/flagd-testbed" - if feature is not None: - image = f"{image}-{feature}" - - self.flagDir = Path("./flags") - self.flagDir.mkdir(parents=True, exist_ok=True) - - # Set environment variables for docker-compose substitution - os.environ["IMAGE"] = image - os.environ["VERSION"] = f"v{self._version}" - os.environ["FLAGS_DIR"] = str(self.flagDir.absolute()) - - self._compose = DockerCompose( - context=str(self._test_harness_dir), - compose_file_name="docker-compose.yaml", - wait=True, - ) - - def get_port(self, resolver_type: ResolverType) -> int: - if resolver_type == ResolverType.RPC: - return self._compose.get_service_port("flagd", 8013) - else: - return self._compose.get_service_port("flagd", 8015) - - def get_exposed_port(self, port: int) -> int: - """Get mapped port. For FORBIDDEN (9212) returns envoy port, otherwise flagd port.""" - if port == FORBIDDEN: - return self._compose.get_service_port("envoy", FORBIDDEN) - return self._compose.get_service_port("flagd", port) - - def get_launchpad_url(self) -> str: - port = self._compose.get_service_port("flagd", LAUNCHPAD) - return f"http://localhost:{port}" - - def start(self) -> "FlagdContainer": - self._compose.start() - host = self._compose.get_service_host("flagd", HEALTH_CHECK) or "localhost" - port = self._compose.get_service_port("flagd", HEALTH_CHECK) - self._checker(host, port) - return self - - def stop(self) -> None: - self._compose.stop() - - def _checker(self, host: str, port: int) -> None: - # Give an extra second before continuing - time.sleep(1) - # Use the GRPC health check endpoint - with grpc.insecure_channel(f"{host}:{port}") as channel: - health_stub = health_pb2_grpc.HealthStub(channel) - - def health_check_call(stub: health_pb2_grpc.HealthStub): - request = health_pb2.HealthCheckRequest() - try: - resp = stub.Check(request) - return resp.status == health_pb2.HealthCheckResponse.SERVING - except Exception: - return False - - # Check health status every 1 second for 30 seconds - ok = False - for _ in range(30): - ok = health_check_call(health_stub) - if ok: - break - time.sleep(1) - - if not ok: - raise ConnectionError("flagd not ready in time") diff --git a/providers/openfeature-provider-flagd/tests/e2e/inprocess/__init__.py b/providers/openfeature-provider-flagd/tests/e2e/inprocess/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/providers/openfeature-provider-flagd/tests/e2e/inprocess/conftest.py b/providers/openfeature-provider-flagd/tests/e2e/inprocess/conftest.py deleted file mode 100644 index d900fdcae..000000000 --- a/providers/openfeature-provider-flagd/tests/e2e/inprocess/conftest.py +++ /dev/null @@ -1,25 +0,0 @@ -import pytest - -from openfeature.contrib.provider.flagd.config import ResolverType -from tests.e2e.testfilter import TestFilter - -resolver = ResolverType.IN_PROCESS -feature_list = [ - "~targetURI", - "~unixsocket", - "~deprecated", - "~fractional-v1", - "~fractional-v2", -] - - -def pytest_collection_modifyitems(config, items): - test_filter = TestFilter( - config, feature_list=feature_list, resolver=resolver.value, base_path=__file__ - ) - test_filter.filter_items(items) - - -@pytest.fixture() -def resolver_type() -> ResolverType: - return resolver diff --git a/providers/openfeature-provider-flagd/tests/e2e/inprocess/test_flaqd.py b/providers/openfeature-provider-flagd/tests/e2e/inprocess/test_flaqd.py deleted file mode 100644 index 3feb4c523..000000000 --- a/providers/openfeature-provider-flagd/tests/e2e/inprocess/test_flaqd.py +++ /dev/null @@ -1,5 +0,0 @@ -from pytest_bdd import scenarios - -from tests.e2e.paths import TEST_HARNESS_PATH - -scenarios(f"{TEST_HARNESS_PATH}/gherkin") diff --git a/providers/openfeature-provider-flagd/tests/e2e/parsers.py b/providers/openfeature-provider-flagd/tests/e2e/parsers.py deleted file mode 100644 index 9d9560c6a..000000000 --- a/providers/openfeature-provider-flagd/tests/e2e/parsers.py +++ /dev/null @@ -1,7 +0,0 @@ -def to_bool(s: str) -> bool: - return s.lower() == "true" - - -def to_list(s: str) -> list: - values = s.replace('"', "").split(",") - return [s.strip() for s in values] diff --git a/providers/openfeature-provider-flagd/tests/e2e/paths.py b/providers/openfeature-provider-flagd/tests/e2e/paths.py deleted file mode 100644 index 6a5022f1b..000000000 --- a/providers/openfeature-provider-flagd/tests/e2e/paths.py +++ /dev/null @@ -1,2 +0,0 @@ -TEST_HARNESS_PATH = "../../openfeature/test-harness" -SPEC_PATH = "../../openfeature/spec" diff --git a/providers/openfeature-provider-flagd/tests/e2e/rpc/__init__.py b/providers/openfeature-provider-flagd/tests/e2e/rpc/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/providers/openfeature-provider-flagd/tests/e2e/rpc/conftest.py b/providers/openfeature-provider-flagd/tests/e2e/rpc/conftest.py deleted file mode 100644 index b06e71369..000000000 --- a/providers/openfeature-provider-flagd/tests/e2e/rpc/conftest.py +++ /dev/null @@ -1,27 +0,0 @@ -import pytest - -from openfeature.contrib.provider.flagd.config import ResolverType -from tests.e2e.testfilter import TestFilter - -resolver = ResolverType.RPC -feature_list = [ - "~targetURI", - "~unixsocket", - "~sync", - "~metadata", - "~deprecated", - "~fractional-v1", - "~fractional-v2", -] - - -def pytest_collection_modifyitems(config, items): - test_filter = TestFilter( - config, feature_list=feature_list, resolver=resolver.value, base_path=__file__ - ) - test_filter.filter_items(items) - - -@pytest.fixture() -def resolver_type() -> ResolverType: - return resolver diff --git a/providers/openfeature-provider-flagd/tests/e2e/rpc/test_flaqd.py b/providers/openfeature-provider-flagd/tests/e2e/rpc/test_flaqd.py deleted file mode 100644 index 3feb4c523..000000000 --- a/providers/openfeature-provider-flagd/tests/e2e/rpc/test_flaqd.py +++ /dev/null @@ -1,5 +0,0 @@ -from pytest_bdd import scenarios - -from tests.e2e.paths import TEST_HARNESS_PATH - -scenarios(f"{TEST_HARNESS_PATH}/gherkin") diff --git a/providers/openfeature-provider-flagd/tests/e2e/step/_utils.py b/providers/openfeature-provider-flagd/tests/e2e/step/_utils.py deleted file mode 100644 index 613e2f24d..000000000 --- a/providers/openfeature-provider-flagd/tests/e2e/step/_utils.py +++ /dev/null @@ -1,30 +0,0 @@ -import json -import time -import typing - -from asserts import assert_true - - -def str2bool(v): - return v.lower() in ("yes", "true", "t", "1") - - -type_cast = { - "Integer": int, - "Float": float, - "String": str, - "Boolean": str2bool, - "Object": json.loads, -} - - -JsonObject: typing.TypeAlias = dict | list -JsonPrimitive: typing.TypeAlias = str | bool | float | int | JsonObject - - -def wait_for(pred, poll_sec=2, timeout_sec=10): - start = time.time() - while not (ok := pred()) and (time.time() - start < timeout_sec): - time.sleep(poll_sec) - assert_true(pred()) - return ok diff --git a/providers/openfeature-provider-flagd/tests/e2e/step/config_steps.py b/providers/openfeature-provider-flagd/tests/e2e/step/config_steps.py deleted file mode 100644 index fa36af031..000000000 --- a/providers/openfeature-provider-flagd/tests/e2e/step/config_steps.py +++ /dev/null @@ -1,111 +0,0 @@ -import re - -import pytest -from asserts import assert_equal, assert_true -from pytest_bdd import given, parsers, then, when - -from openfeature.contrib.provider.flagd.config import CacheType, Config, ResolverType - - -def camel_to_snake(name): - name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) - return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower() - - -def str2bool(v): - return v.lower() in ("yes", "true", "t", "1") - - -def convert_resolver_type(val: str | ResolverType) -> ResolverType: - if isinstance(val, str): - v = val.lower() - return ResolverType(v) - else: - return ResolverType(val) - - -type_cast = { - "Integer": int, - "Long": int, - "String": str, - "Boolean": str2bool, - "ResolverType": convert_resolver_type, - "CacheType": CacheType, - "StringList": lambda s: ( - [item.strip() for item in s.split(",") if item.strip()] - if s and s.strip() - else [] - ), -} - - -@pytest.fixture() -def option_values() -> dict: - return {} - - -@given( - parsers.re( - r'an option "(?P