From 1c8c49aeb4f7b6a4c6a8bdbcc7ac199a65fe1a6f Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 2 Nov 2026 10:54:08 +0100 Subject: [PATCH] feat(tck): emit a machine-readable conformance report Collapsed from 22 commits when this branch was replayed onto the base's controlApi -> backend_type rename and its adopter tag gate, following the js-sdk-contrib replay of the same branch. Several of the commits fought the rename itself, so one squashed replay meant one conflict pass, and this repository squash-merges anyway. The report follows the rename: it writes backend.backendType through backend_type_of(), so a control that states none is reported as "custom", matching the schema change in spec#425. The report-binding fixture now runs ahead of the base's combined scenario gate, so a scenario skipped by any of the gates still reaches its suite. The original subjects, oldest first: - feat(tck): emit a machine-readable conformance report - feat(tck): identify a Scenario Outline row by its parameters - fix(tck): give a failed capability a reason, and stop claiming untested ones - fix(tck): read the tags of the Examples block a row came from - refactor(tck): carry results in Cucumber Messages - feat(tck): record which Cucumber Messages release produced the stream - fix(tck): say when a control does not report how it drives the backend - refactor(tck): rename @strict-numeric-typing to @numeric-coercion - fix(tck): a reserved capability must not be declarable - feat(tck): report an extension under its own uri prefix - feat(tck): refuse a report that did not run the canonical set - fix(tck): read the canonical count from the assets, not from a test - fix(tck): follow the canonical gherkin/ prefix through the report - fix(tck): stop reporting declaration.notApplicable - docs(tck): refresh the report numbers for 56 scenarios - fix(tck): follow the base's rename and the removal of features_path() - feat(tck)!: controlApi is always written, and the guessing goes away - refactor(tck)!: the environment variables drop their PROVIDER_ prefix - docs(tck): separate "answers the question" from "satisfies the capability" - docs(tck): restate the report documentation on the rewritten README - fix(tck): the mismatch matrix is eight rows, not eleven - docs(tck): follow the base branch's run numbers Signed-off-by: Simon Schrottner --- tools/openfeature-tck/.gitignore | 3 + tools/openfeature-tck/README.md | 81 +- tools/openfeature-tck/hatch_build.py | 17 +- tools/openfeature-tck/hatch_build_sync.py | 68 +- tools/openfeature-tck/pyproject.toml | 34 + .../openfeature/contrib/tools/tck/__init__.py | 7 + .../contrib/tools/tck/canonical.py | 116 ++ .../openfeature/contrib/tools/tck/emitter.py | 657 +++++++++ .../openfeature/contrib/tools/tck/messages.py | 667 +++++++++ .../openfeature/contrib/tools/tck/plugin.py | 32 +- .../openfeature/contrib/tools/tck/report.py | 493 +++++++ .../openfeature/contrib/tools/tck/state.py | 7 + .../contrib/tools/tck/steps/provider_steps.py | 19 + tools/openfeature-tck/tests/conftest.py | 59 +- .../tests/test_canonical_set.py | 374 +++++ .../tests/test_controllable_conformance.py | 6 +- .../tests/test_extension_reporting.py | 652 +++++++++ .../tests/test_http_control.py | 2 +- .../tests/test_in_memory_conformance.py | 9 +- tools/openfeature-tck/tests/test_report.py | 1218 +++++++++++++++++ uv.lock | 13 + 21 files changed, 4515 insertions(+), 19 deletions(-) create mode 100644 tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py create mode 100644 tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py create mode 100644 tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py create mode 100644 tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py create mode 100644 tools/openfeature-tck/tests/test_canonical_set.py create mode 100644 tools/openfeature-tck/tests/test_extension_reporting.py create mode 100644 tools/openfeature-tck/tests/test_report.py diff --git a/tools/openfeature-tck/.gitignore b/tools/openfeature-tck/.gitignore index 41520f2e8..a0ed5358e 100644 --- a/tools/openfeature-tck/.gitignore +++ b/tools/openfeature-tck/.gitignore @@ -5,3 +5,6 @@ src/openfeature/contrib/tools/tck/gherkin/ src/openfeature/contrib/tools/tck/flag_data/ src/openfeature/contrib/tools/tck/control-api.yaml +# Generated alongside them, from the submodule pin, so a conformance report can +# name the spec revision it ran against. +src/openfeature/contrib/tools/tck/spec_revision.json diff --git a/tools/openfeature-tck/README.md b/tools/openfeature-tck/README.md index 39aaa3b06..42202d02a 100644 --- a/tools/openfeature-tck/README.md +++ b/tools/openfeature-tck/README.md @@ -371,6 +371,83 @@ self-tests declare less than they otherwise would: it cannot update its flag set hands each variant back untouched, so it does not attempt numeric coercion at all — a permitted choice rather than a defect, and the reason `@numeric-coercion` is simply not declared there. +## Conformance reports + +Set `TCK_REPORT_DIR` and each suite writes **two** files: an envelope at `/.json`, +conforming to the [report schema][report-schema], and the results it points at, at +`/.ndjson`, which is a [Cucumber Messages][messages] stream. + +```console +$ TCK_REPORT_DIR=./reports pytest +tck [in-memory]: report written to reports/in-memory.json with results in in-memory.ndjson (1 failed, 43 passed, 21 skipped) + +$ jq -c .results reports/in-memory.json +{"format":"cucumber-messages","location":"in-memory.ndjson","digest":"sha256:c7e12a…"} +``` + +It is an environment variable rather than a `TckConfig` field so that emitting a report is a property +of the *run* and not of the code: CI sets it, a developer running the suite locally does not, and no +adopter changes a line to publish one. Unset means no report, which is not an error, and several +suites in one session each write their own pair, so flagd's two resolvers do not collide. + +**A partial run is not a conformance run.** `-k`, `-m`, `--deselect`, or a test module that stopped +calling `scenarios()` on the canonical path each run fewer scenarios, and none of them is an error to +pytest — Go measured the consequence: a green run and a well-formed report covering one scenario out +of the whole canonical set, with nothing in the document saying so. So every run is checked against +the scenarios this distribution ships, and one that did not execute all of them writes no report and +names what it missed. A capability-gated skip **has** run — the question was put and declined — so +declining never trips this, and your own extension scenarios are not counted towards the canonical +set and cannot close a gap in it. `TCK_PARTIAL=1` lets you work on a single scenario without the +guard failing the run; it still writes no report, and Java spells the same escape hatch the same way. + +**Why the results are not our format.** Per-scenario outcomes, tags, Scenario Outline row identity +and the executed feature source are already specified by Cucumber Messages, which is maintained, +cross-language, schema'd and emitted natively by cucumber-jvm; defining them again would create a +second format to version and two places for the same fact to disagree. So the envelope says what was +tested and what the provider claims, and the payload says what happened. The payload is referenced +rather than inlined because it carries the feature sources and is far larger than the envelope, and +`results.digest` is a SHA-256 over the exact bytes written, so a consumer can tell that what it +fetched is what the envelope described. Two things Messages cannot carry stay in the envelope: +`declaration`, which is an *input* to reading the results rather than a summary of them — only it +says whether a skipped scenario was declined — and the tested subject, for which no standard results +format has a slot. + +**Reading the payload.** Appendix F requires a scenario skipped for an undeclared capability to be +reported as skipped with the reason and never as passed, and a consumer cannot check that against a +summary line, so the stream carries every scenario the run collected, gate-skipped ones included, as +Cucumber's own `SKIPPED`. Each is a `TestCase` referring to a `Pickle`, and a test case is as bad as +its worst step. Every test case carries two hook steps as well as its Gherkin steps, because pytest +runs a scenario in three phases and only the middle one executes steps: the before-hook is where a +capability skip's reason lands and the after-hook where a teardown failure does. The capability +responsible for a skip follows from the pickle's tags and the envelope's `declaration`, which is why +it is not transported once per scenario. And a pickle's `astNodeIds` are `[scenario id, table row +id]`, resolving in the `GherkinDocument` to exactly the cells the feature file wrote — which is what +tells the eight rows of the type-mismatch matrix apart, one of which differs in outcome from its seven +siblings, exactly rather than by a naming convention every implementation would have to reproduce. + +So the payload is not a transcription of pytest's summary. The one scenario the Python SDK cannot +satisfy is marked `xfail`, so pytest counts it as expected and exits zero; the provider still did not +satisfy it, and the stream says `FAILED`. The acknowledgement goes in the envelope's +`knownDeviations` instead — an expected failure is a recorded deviation, not an excused one. + +**What identifies a report.** `tck.specRevision` comes from `spec_revision.json`, which the asset +sync generates from the submodule at build time, because the submodule is not in the wheel and an +installed copy has nothing left to ask; a build that cannot reach git records `unknown` rather than +inventing a commit. There is no asset tree hash: the payload's `Source` messages carry the executed +feature files verbatim, which answers "did these two runs ask the same questions" directly rather +than by proxy. `provider.name` is what the provider reports through its own metadata, not +`TckConfig.name`, which is chosen to read well in a failure message — `flagd-rpc` — and is therefore +reported as the *configuration*. `backend.backendType` comes from the control's `backend_type`, `"custom"` when it states none, and +the whole `backend` block is always written, both being in the schema's `required` arrays: nothing +is inferred from the control's class, which is the point. + +One gap is this implementation's rather than the suite's: **pytest-bdd emits no Cucumber Messages.** +It ships the legacy Cucumber JSON format and nothing for the ndjson protocol, so `messages.py` builds +the stream from the official types and re-parses the feature files for the AST node ids a pickle +refers to. If pytest-bdd ever emits Messages itself, that module should shrink to a shim. Whether a +report belongs inside a provider's released artifact is open on +[open-feature/spec#424](https://github.com/open-feature/spec/issues/424). + ## Contributing The Gherkin, the canonical flag set and the control-API document are **not owned by this @@ -379,7 +456,7 @@ sdist at build time. So adopting needs no submodule and contributing does: ```bash git submodule update --init tools/openfeature-tck/spec -poe test # syncs the assets first; 227 passed, 41 skipped, 2 xfailed, no Docker +poe test # syncs the assets first; 295 passed, 41 skipped, 2 xfailed, no Docker ``` The copies under `src/` are gitignored, generated and carry a `DO-NOT-EDIT.txt`: a change goes to @@ -403,6 +480,8 @@ drifted from one. The sync itself still only warns in a checkout whose superproj and copies unverified; the test run then fails on the unreadable pin, so sync from a shell that can see the superproject. +[report-schema]: https://github.com/open-feature/spec/blob/main/specification/assets/provider-tck/report/conformance-report.schema.json +[messages]: https://github.com/cucumber/messages [appendix-a]: https://github.com/open-feature/spec/blob/main/specification/appendix-a-included-utilities.md [appendix-f]: https://github.com/open-feature/spec/blob/main/specification/appendix-f-provider-conformance.md [java-tck]: https://github.com/open-feature/java-sdk-contrib/tree/main/tools/tck diff --git a/tools/openfeature-tck/hatch_build.py b/tools/openfeature-tck/hatch_build.py index 5bff5a40e..19b0b26fd 100644 --- a/tools/openfeature-tck/hatch_build.py +++ b/tools/openfeature-tck/hatch_build.py @@ -18,7 +18,14 @@ # the single definition of what gets copied where -- would not be importable. sys.path.insert(0, str(Path(__file__).parent)) -from hatch_build_sync import FILES, PACKAGE_REL, SPEC_ASSETS, TREES, sync +from hatch_build_sync import ( + FILES, + PACKAGE_REL, + REVISION_FILE, + SPEC_ASSETS, + TREES, + sync, +) class SpecAssetsCopyHook(BuildHookInterface): @@ -26,7 +33,13 @@ class SpecAssetsCopyHook(BuildHookInterface): def initialize(self, version: str, build_data: dict) -> None: root = Path(self.root) - copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES] + # The generated revision file travels with the assets it describes. It + # has to be built here rather than read at run time, because the + # submodule that knows the answer is not in the wheel and a conformance + # report has to name the revision it ran against. + copies = [root / PACKAGE_REL / dest for _, dest in TREES + FILES] + [ + root / PACKAGE_REL / REVISION_FILE + ] # Building from a checkout: refresh from the submodule, so what ships is # always the revision the pin names. Building from an sdist: there is no diff --git a/tools/openfeature-tck/hatch_build_sync.py b/tools/openfeature-tck/hatch_build_sync.py index d21b2ea36..215b6ae3a 100644 --- a/tools/openfeature-tck/hatch_build_sync.py +++ b/tools/openfeature-tck/hatch_build_sync.py @@ -15,6 +15,7 @@ from __future__ import annotations +import json import os import shutil import subprocess @@ -25,7 +26,8 @@ ROOT = Path(__file__).parent SPEC_DIRNAME = "spec" SPEC_ROOT = (ROOT / SPEC_DIRNAME).resolve() -SPEC_ASSETS = (SPEC_ROOT / "specification/assets/provider-tck").resolve() +ASSETS_PATH_IN_SPEC = "specification/assets/provider-tck" +SPEC_ASSETS = (SPEC_ROOT / ASSETS_PATH_IN_SPEC).resolve() PACKAGE_REL = Path("src/openfeature/contrib/tools/tck") DEST_BASE = ROOT / PACKAGE_REL @@ -243,6 +245,28 @@ def checkout_pinned_spec(git: GitRunner = _run_git) -> str | None: return pinned +REVISION_FILE = "spec_revision.json" +"""Which revision of the specification the copied assets came from. + +Recorded at build time because the answer is only available at build time: the +submodule that holds it is not in the wheel, and a conformance report that cannot +name the revision it ran against cannot be compared with another. It is generated +by the same command that copies the assets, which is what keeps the two from +disagreeing. + +Not committed, for the same reason the assets are not: the submodule pin is the +single record of which revision this package targets. +""" + +UNKNOWN_REVISION = "unknown" +"""Seven characters, the minimum the report schema accepts. + +A build that cannot reach git says it does not know rather than inventing a +commit, and still produces a document that validates. Which happens for real: +building from a source tarball has no ``.git`` to ask. +""" + + def sync() -> None: checkout_pinned_spec() @@ -267,6 +291,48 @@ def sync() -> None: dest.unlink() shutil.copy2(SPEC_ASSETS / src_name, dest) + write_revision() + + +def write_revision() -> None: + """Record the spec commit these copies came from. + + The asset tree hash that used to accompany it is gone. It was carried so a + consumer could tell whether two runs executed the same questions; the + conformance report's results are now a Cucumber Messages stream, which + carries the executed feature source itself and answers that directly rather + than by proxy. + """ + commit = _git("rev-parse", "HEAD") or UNKNOWN_REVISION + (DEST_BASE / REVISION_FILE).write_text( + json.dumps({"specRevision": commit}, indent=2) + "\n", + encoding="utf-8", + ) + + +def _git(*args: str) -> str: + """Run git inside the submodule, returning its output or an empty string. + + A build must not hard-fail because git is absent or the checkout is not a + repository -- both are ordinary when building from an unpacked sdist. The + failure is reported as a warning and the identity degrades to ``unknown``, + which is legible in the resulting report rather than silently wrong. + """ + command = ["git", "-C", str(SPEC_ROOT), *args] + try: + completed = subprocess.run( # noqa: S603 + command, capture_output=True, check=True, text=True + ) + except (OSError, subprocess.CalledProcessError) as error: + warnings.warn( + f"could not determine the spec revision ({' '.join(command)}: {error}); " + f"conformance reports from this build will not name the revision they " + f"ran against", + stacklevel=2, + ) + return "" + return completed.stdout.strip() + if __name__ == "__main__": sync() diff --git a/tools/openfeature-tck/pyproject.toml b/tools/openfeature-tck/pyproject.toml index 2cb6f12ff..594dafbba 100644 --- a/tools/openfeature-tck/pyproject.toml +++ b/tools/openfeature-tck/pyproject.toml @@ -22,6 +22,21 @@ dependencies = [ # Same runner the flagd provider and the flagd testkit already use, so an # adopting module gains no new test framework. "pytest-bdd>=8.1.0,<9.0.0", + # The conformance report's results are a Cucumber Messages stream rather + # than a format this package defines. These two are the reference + # implementations of the halves of that protocol: cucumber-messages is the + # official Python types, published from the same repository as the protocol + # itself, and gherkin-official is the parser that produces the + # gherkinDocument and pickle messages. pytest-bdd already depends on + # gherkin-official, so only the first is genuinely new -- and it has no + # dependencies of its own. + # + # pytest-bdd ships no Messages emitter (its cucumber_json.py is the legacy + # JSON format), so the stream is assembled here; assembling it from typed + # messages rather than hand-written dicts is what keeps it from drifting + # away from the protocol. + "cucumber-messages>=34.0.0,<35.0.0", + "gherkin-official>=29.0.0", ] requires-python = ">=3.10" @@ -69,6 +84,10 @@ artifacts = [ "src/openfeature/contrib/tools/tck/gherkin/", "src/openfeature/contrib/tools/tck/flag_data/", "src/openfeature/contrib/tools/tck/control-api.yaml", + # Which spec revision those assets came from, generated beside them. The + # submodule is not in the wheel, so a conformance report emitted by an + # installed copy has no other way to name the revision it ran against. + "src/openfeature/contrib/tools/tck/spec_revision.json", ] [tool.hatch.build.hooks.custom] @@ -92,6 +111,21 @@ fixed_format_cache = true pretty = true strict = true disallow_any_generics = false +# cucumber-messages and gherkin-official ship no py.typed. Both are annotated +# internally, so following them gives real types for the messages this package +# builds rather than the Any a plain `ignore_missing_imports` would hand back -- +# which is the point of using the typed library at all. +follow_untyped_imports = true + +[[tool.mypy.overrides]] +# gherkin-official has no annotations at all, so following it turns every call +# into a `no-untyped-call` error rather than into a type. pytest-bdd silences +# the same import the same way. cucumber-messages is the opposite case -- fully +# annotated, only missing py.typed -- and is followed, which is where the value +# of using it rather than hand-written dicts actually lands. +module = ["gherkin.*"] +follow_untyped_imports = false +ignore_missing_imports = true # testcontainers 4.14 ships no py.typed for `testcontainers.compose`, so the one # lazy import in compose.py cannot be checked against it. Scoped to that module diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py index 5937f09df..dca837bd2 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/__init__.py @@ -76,6 +76,7 @@ def tck_config(): import importlib.resources +from .canonical import PARTIAL_ENV from .capability import ( DECLARABLE_CAPABILITIES, INEXPRESSIBLE_CAPABILITIES, @@ -112,12 +113,14 @@ def tck_config(): HttpControl, ) from .inprocess import InProcessControl +from .messages import MESSAGES_FORMAT from .provider import ( CHANGING_FLAG_KEY, ControllableInMemoryProvider, canonical_flag_set, canonical_flags_json, ) +from .report import REPORT_DIR_ENV, SCHEMA_VERSION from .state import TckState __all__ = [ @@ -130,7 +133,11 @@ def tck_config(): "DEFAULT_STARTUP_TIMEOUT", "EXTENSIONS_DIRECTORY", "INEXPRESSIBLE_CAPABILITIES", + "MESSAGES_FORMAT", + "PARTIAL_ENV", + "REPORT_DIR_ENV", "RESERVED_CAPABILITIES", + "SCHEMA_VERSION", "BackendControl", "BackendEndpoint", "BackendType", diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py new file mode 100644 index 000000000..33b34164d --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/canonical.py @@ -0,0 +1,116 @@ +"""The canonical scenario set, and whether a run actually executed it. + +A conformance suite that goes green on scenarios it did not run is worse than no +suite. The capability gate already rules out the loud version of that -- an +undeclared capability is reported as skipped, with its reason, never as passed -- +but it says nothing about the quiet version, where the scenarios were never asked +for in the first place. A ``-k`` expression, a ``-m`` filter, a ``--deselect``, a +test module that stopped calling ``scenarios()`` on the canonical path: each of +those runs less of the suite and none of them is an error. The Go implementation +measured it. ``-run`` on a single scenario passed green and emitted a well-formed +report covering one scenario out of the whole canonical set, and nothing in the +document said so. + +So the run is checked against what this distribution *ships* rather than against +what it was asked to run. The expectation is compiled from the packaged feature +files with the same Gherkin compiler that produces the results payload, which +means it is one entry per Scenario Outline row -- the granularity the runner +generates and therefore the only one a comparison can be made at. + +Two properties this has to have, and both are about what may close a gap. + +**A skip counts; an absence does not.** A scenario the capability gate skipped +did run: it was asked, and the report accounts for it with a reason. A scenario +that was never collected is missing, and no declaration makes it otherwise. + +**An extension cannot close a gap.** An adopter's own scenarios are matched by +neither uri nor path against the packaged set, and the executed side of the +comparison is filtered to files that are genuinely inside this distribution -- +not merely to files reported under the canonical prefix, so that the check does +not rest on the same derivation it is meant to corroborate. +""" + +from __future__ import annotations + +import functools +import os +import typing + +from .extensions import canonical_root, is_canonical, uri_for +from .messages import FeatureCatalog, ScenarioKey, ScenarioRun + +__all__ = [ + "PARTIAL_ENV", + "canonical_scenarios", + "describe", + "missing_canonical", + "partial_run_allowed", +] + +PARTIAL_ENV = "TCK_PARTIAL" +"""Set to acknowledge that a run is deliberately not a conformance run. + +For working on one scenario with ``-k`` without the guard failing the run. It +never makes a partial run publishable: no report is written for a suite that did +not execute the canonical set, with or without it. The Java TCK spells the same +escape hatch the same way, so the two are one thing to know rather than two. +""" + +_TRUTHY = {"1", "true", "yes", "on"} + + +def partial_run_allowed(environment: typing.Mapping[str, str] | None = None) -> bool: + """Whether the run has declared itself partial.""" + source = os.environ if environment is None else environment + return source.get(PARTIAL_ENV, "").strip().lower() in _TRUTHY + + +@functools.cache +def canonical_scenarios() -> frozenset[ScenarioKey]: + """Every scenario the packaged feature files define, row by row. + + Empty when the assets are not reachable as files -- an installation from a + zipimport, say. Everything built on this then degrades to "cannot tell", + which is the honest answer and never a false accusation. + + Cached because it is the same answer for the whole process and parsing it is + the same work the results payload already does. + """ + root = canonical_root() + if root is None or not root.is_dir(): + return frozenset() + + catalog = FeatureCatalog() + for path in sorted(root.rglob("*.feature")): + uri = uri_for(path) + if uri is not None: + catalog.load_file(uri, path) + return catalog.scenario_keys + + +def missing_canonical(runs: typing.Iterable[ScenarioRun]) -> tuple[ScenarioKey, ...]: + """The canonical scenarios this suite did not execute, in reporting order. + + ``runs`` is everything the suite accounted for, extensions included; only + the ones whose feature file is genuinely one of the packaged assets are + counted, so an adopter's scenario can neither fill a gap nor be blamed for + one. + """ + expected = canonical_scenarios() + if not expected: + return () + executed = { + (run.identity.uri, run.identity.name, run.identity.example) + for run in runs + if is_canonical(run.identity.path) + } + return tuple(sorted(expected - executed)) + + +def describe(key: ScenarioKey) -> str: + """One missing scenario, named the way a failure message should name it.""" + uri, name, row = key + if not row: + return f"{uri}: {name}" + cells = " ".join(f"{header}={cell}" for header, cell in row) + return f"{uri}: {name} [{cells}]" diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py new file mode 100644 index 000000000..4d5a4a00c --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/emitter.py @@ -0,0 +1,657 @@ +"""The pytest half of the conformance report: turning a run into the two documents. + +Kept apart from :mod:`report`, which knows what an envelope *is*, and from +:mod:`messages`, which knows what a Cucumber Messages stream is; neither knows +anything about pytest. Everything here is translation -- a pytest node into a +scenario, a :class:`pytest.TestReport` into a step status, the end of a session +into a pair of files on disk. + +Two translations matter. + +**Skips.** pytest reports a skip honestly, unlike some runners, and Cucumber's +``SKIPPED`` says the same thing, so a capability-gated scenario reaches the +stream as skipped without anything having to be decided. What the stream does not +say is *why* -- and it does not need to, because the envelope carries the +provider's declaration and the stream carries the scenario's tags, so the reason +for the skip follows from the two. The skip message is carried anyway, on the +setup hook's result, because a person reading the stream should not have to +perform that derivation. + +**Expected failures.** A scenario marked ``xfail`` is one pytest reports as +skipped and finishes green on. The provider still did not satisfy it, so the +stream reports it as failed. The acknowledgement belongs in the envelope's +``knownDeviations``, where it is a claim about the provider rather than a +softening of the result. +""" + +from __future__ import annotations + +import os +import time +import typing +from pathlib import Path + +import pytest + +from .canonical import ( + PARTIAL_ENV, + canonical_scenarios, + describe, + missing_canonical, + partial_run_allowed, +) +from .config import TckConfig +from .extensions import ( + collision_problem, + reserved_prefix_problem, + uri_collisions, + uri_for, +) +from .messages import ( + FeatureCatalog, + ScenarioIdentity, + ScenarioRun, + Status, + StepRun, + feature_uri, + worse, + write_stream, +) +from .report import ( + REPORT_DIR_ENV, + TCK_DISTRIBUTION, + TCK_IMPLEMENTATION, + PhaseOutcome, + ReportCollector, + Results, + SuiteReport, + distribution_version, + envelope_file_name, + normalise_tags, + stream_file_name, + write_envelope, +) + +__all__ = [ + "COLLECTOR_KEY", + "ReportEmitter", + "classify_phase", + "scenario_identity", + "scenario_run", +] + +COLLECTOR_KEY = pytest.StashKey[ReportCollector]() +"""Where the session's collector lives, so a fixture can reach it from a request.""" + +_EXAMPLE_PARAM = "_pytest_bdd_example" +"""The parameter pytest-bdd renders a Scenario Outline over. + +An implementation detail of pytest-bdd, named here rather than spelled inline so +that a version bump that renames it fails in one place. The alternative -- asking +the scenario template for its examples -- would have to work out which row *this* +node is, which is the question the callspec already answers. +""" + +_MAX_REASON = 500 +"""How much of a failure message the stream carries. + +A message is for a person reading a comparison page, not for debugging: whoever +ran the suite has the traceback. Whole tracebacks in a published document also +leak local paths. +""" + +_SKIPPED = pytest.skip.Exception +"""What ``pytest.skip`` raises, named so a step hook can recognise it.""" + +_MAX_MISSING = 10 +"""How many missing canonical scenarios a failure names before summarising. + +Enough to act on and not so many that the reason is lost above them. A run with +one scenario selected is missing every other one the set has, and listing all of +them says nothing the count did not. +""" + + +def scenario_identity(node: pytest.Item) -> ScenarioIdentity | None: + """Describe a pytest node as a Gherkin scenario, or return ``None``. + + ``__scenario__`` is what pytest-bdd hangs on the function it generates, so + its presence is also the test for "is this a TCK scenario at all" -- and it + is readable at collection, without running a single fixture, which is what + lets a scenario skipped before its first step still be accounted for. + """ + scenario = getattr(getattr(node, "function", None), "__scenario__", None) + if scenario is None: + return None + + feature = getattr(scenario, "feature", None) + tags: set[str] = set(getattr(scenario, "tags", None) or ()) + tags |= set(getattr(feature, "tags", None) or ()) + rule = getattr(scenario, "rule", None) + if rule is not None: + tags |= set(getattr(rule, "tags", None) or ()) + tags |= _examples_tags(node, scenario) + + filename = str(getattr(feature, "filename", "")) + path = Path(filename) + relative = str(getattr(feature, "rel_filename", "") or path.name) + + return ScenarioIdentity( + # Derived from where the file is, falling back to what pytest-bdd called + # it. pytest-bdd names a feature by its parent directory joined to its + # own name, which two files can share: an extension at + # extensions/gherkin/errors.feature arrives under exactly the uri + # the canonical errors.feature already occupies, and the payload carries + # one source per uri. + uri=uri_for(path) or feature_uri(relative), + path=path, + name=str(getattr(scenario, "name", "")), + example=_example_of(node), + tags=normalise_tags(tags), + ) + + +def _examples_tags(node: pytest.Item, scenario: object) -> set[str]: + """The tags of the Examples block *this row* came from. + + Gherkin allows an Examples block to carry its own tags, so two rows of one + Scenario Outline can differ in which capability gates them. Those tags are + not on the scenario, the feature or the rule, so a stream built from those + three alone would show a row the capability gate skipped as carrying no + capability at all -- and the envelope's declaration would then not explain + the skip, which is the one derivation the format asks a consumer to make. + + Resolved by intersecting the tags the scenario's Examples blocks declare with + the markers pytest actually put on this node: pytest-bdd attaches an Examples + block's tags as marks on that block's parameter sets, so the intersection + names this row's blocks without having to work out which block a row came + from, and admits nothing that is not a Gherkin tag of this scenario. + + No canonical feature file uses per-Examples tags today, so this is latent -- + but it is latent in the direction of under-reporting a skip, which is the one + failure mode the format exists to rule out. + """ + declared: set[str] = set() + for examples in getattr(scenario, "examples", None) or (): + declared |= set(getattr(examples, "tags", None) or ()) + if not declared: + return set() + return declared & {marker.name for marker in node.iter_markers()} + + +def _example_of(node: pytest.Item) -> tuple[tuple[str, str], ...]: + """The Examples row this node came from, keyed by column header. + + No longer reported -- Cucumber Messages identifies an outline row by the AST + node id of the table row a pickle was compiled from, which is exact and which + every runner that emits Messages already carries. This survives as the *join + key*: it is the one description of a row that both a pytest-bdd node and a + Gherkin pickle can produce independently, so it is how a node is matched to + its pickle. Matching on the pickle's name would not work, because the + compiler interpolates the row's parameters into it and pytest-bdd does not. + + pytest-bdd renders an outline by parametrizing the generated test over one + dict per row, keyed by the Examples column header, and pytest hangs it on the + node's callspec. A scenario that is not an outline is not parametrized and + has no callspec at all, which is why the empty tuple is the answer for one -- + and it matches the empty row of a pickle with a single AST node id. + + Values are passed through as the parser produced them: Gherkin cells are + strings, and both sides of the join have to agree on ``"1"`` rather than one + of them guessing it was meant as a number. + """ + params = getattr(getattr(node, "callspec", None), "params", None) + if not isinstance(params, dict): + return () + row = params.get(_EXAMPLE_PARAM) + if not isinstance(row, dict): + return () + # Column order, as the feature file wrote it, because dicts preserve + # insertion order and pytest-bdd builds this one from the header row. + return tuple((str(header), str(cell)) for header, cell in row.items()) + + +def _group_of(node: pytest.Item) -> str: + """Which module a scenario was generated into. + + pytest-bdd's ``scenarios()`` injects its tests into the module that called + it, and a module resolves one ``tck_config``, so the module is what says + which suite a scenario belongs to. Two modules sharing a ``tck_config`` from + a conftest are two groups pointing at one suite, which is exactly right. + """ + return node.nodeid.partition("::")[0] + + +class ReportEmitter: + """Collects outcomes for the session and writes one report per suite. + + A plugin object rather than module-level hook functions because + ``pytest_runtest_logreport`` is handed a report and nothing else: the state + it has to reach has to come from somewhere, and an instance is a less + surprising somewhere than a module global. + """ + + def __init__(self, config: pytest.Config) -> None: + self.collector = ReportCollector() + self._step_started: dict[str, int] = {} + config.stash[COLLECTOR_KEY] = self.collector + + @pytest.hookimpl(trylast=True) + def pytest_collection_modifyitems(self, items: list[pytest.Item]) -> None: + """Enumerate every TCK scenario the session will run. + + At collection rather than as each runs, so that the stream accounts for + scenarios that never got as far as running a fixture. + + ``trylast`` so that pytest's own deselection -- ``-k``, ``-m``, + ``--deselect`` -- has already removed what it is going to remove. + Enumerating before it does would make a filtered run report every + deselected scenario as collected but never run, which is a true + statement about a list nobody asked for and drowns the one message that + matters: which canonical scenarios are missing. + """ + for item in items: + identity = scenario_identity(item) + if identity is not None: + self.collector.collect(item.nodeid, _group_of(item), identity) + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + self.collector.observe(report.nodeid, _phase_outcome(report)) + + # -- what each Gherkin step did ------------------------------------------ + # + # pytest reports a scenario, not its steps. Cucumber Messages records a + # result per step, and inventing one -- marking all eight steps failed + # because the scenario failed -- would be saying something untrue about the + # seven that passed and the ones that were never reached. pytest-bdd's step + # hooks are the only place the truth is available. + + def pytest_bdd_before_step( + self, request: pytest.FixtureRequest, step: object + ) -> None: + self._step_started[request.node.nodeid] = time.time_ns() + + def pytest_bdd_after_step(self, request: pytest.FixtureRequest) -> None: + self._finish_step(request, Status.passed) + + def pytest_bdd_step_error( + self, request: pytest.FixtureRequest, exception: BaseException + ) -> None: + # A step that calls ``pytest.skip`` raises through the same hook as one + # that failed, and the two are not the same result. Told apart by the + # exception type rather than by the message, which is prose. + if isinstance(exception, _SKIPPED): + self._finish_step(request, Status.skipped, exception) + return + self._finish_step(request, Status.failed, exception) + + def pytest_bdd_step_func_lookup_error( + self, request: pytest.FixtureRequest, exception: BaseException + ) -> None: + # UNDEFINED rather than FAILED: the step was never run, because nothing + # claimed to know how to run it. That is a defect in an adoption rather + # than a finding about the provider, and the stream says which. + self._step_started.setdefault(request.node.nodeid, time.time_ns()) + self._finish_step(request, Status.undefined, exception) + + def _finish_step( + self, + request: pytest.FixtureRequest, + status: Status, + exception: BaseException | None = None, + ) -> None: + node_id = request.node.nodeid + finished = time.time_ns() + self.collector.observe_step( + node_id, + StepRun( + status=status, + message=_reason(str(exception)) if exception is not None else "", + exception_type=type(exception).__name__ + if exception is not None + else "", + started_ns=self._step_started.pop(node_id, finished), + finished_ns=finished, + ), + ) + + def pytest_sessionfinish(self, session: pytest.Session) -> None: + if session.config.getoption("collectonly", False): + # Nothing ran, and nothing was meant to. Every check below asks what + # a run executed, and the answer "nothing" is not a finding here. + return + + problems = self.collector.resolve(scenario_run) + + # Whether a suite may be published is a property of the run rather than + # of the report, so it is established whether or not one was asked for. + # Both checks are made on every suite rather than short-circuited, so a + # suite with two faults hears about both. + unpublishable: set[int] = set() + for suite in self.collector.suites: + sound = self._identities_are_sound(session, suite) + complete = self._canonical_set_ran(session, suite) + if not sound or not complete: + unpublishable.add(id(suite)) + + directory = os.environ.get(REPORT_DIR_ENV, "").strip() + if not directory: + return + for problem in problems: + self._fail(session, f"tck: {problem}") + self.write(session, Path(directory), unpublishable) + + def write( + self, + session: pytest.Session, + directory: Path, + unpublishable: typing.AbstractSet[int] = frozenset(), + ) -> None: + """Write every suite's pair of files, failing the session if one cannot be. + + A run that asked for a report and silently did not get one is how a + publishing pipeline ends up serving a stale result forever, so both a + write failure and an incomplete document are loud and change the exit + status rather than being logged and forgotten. + """ + written: dict[str, str] = {} + for suite in self.collector.suites: + if id(suite) in unpublishable: + continue + name = suite.config.name + file_name = envelope_file_name(name) + if written.get(file_name, name) != name: + self._fail( + session, + f"tck: suites {written[file_name]!r} and {name!r} both " + f"write {file_name}; give them names that do not collide", + ) + continue + written[file_name] = name + self._write_suite(session, directory, suite) + + def _canonical_set_ran(self, session: pytest.Session, suite: SuiteReport) -> bool: + """Whether this suite executed every scenario the TCK ships. + + The check a conformance claim rests on that no amount of reading the + report can supply: the results payload says what happened to the + scenarios that ran, and says nothing at all about the ones that did not. + + A capability-gated skip counts -- it was asked, and the report accounts + for it with a reason. An extension scenario does not count and cannot + close a gap. :data:`~.canonical.PARTIAL_ENV` downgrades the failure to a + note for someone working on a single scenario; it does not make the run + publishable, because the report is withheld either way. + """ + missing = missing_canonical(suite.runs.values()) + if not missing: + return True + + name = suite.config.name + total = len(canonical_scenarios()) + headline = ( + f"tck [{name}]: {len(missing)} of {total} canonical scenarios " + f"did not run, so this run cannot support a conformance claim and no " + f"report is written for it. The canonical set is fixed by the " + f"specification; running less of it is not a configuration. Decline " + f"capabilities your provider does not have through TckConfig instead, " + f"which reports the scenarios as skipped with their reason" + ) + if partial_run_allowed(): + self._say( + session, + f"{headline}. {PARTIAL_ENV} is set, so the run is not failed for it", + ) + else: + self._fail( + session, + f"{headline}. To filter anyway while working on one scenario, set " + f"{PARTIAL_ENV}=1 and accept that the run is not a conformance run", + ) + for key in missing[:_MAX_MISSING]: + self._say(session, f" - {describe(key)}") + if len(missing) > _MAX_MISSING: + self._say(session, f" ... and {len(missing) - _MAX_MISSING} more") + return False + + def _write_suite( + self, session: pytest.Session, directory: Path, suite: SuiteReport + ) -> None: + name = suite.config.name + runs = suite.sorted_runs + + catalog = FeatureCatalog() + try: + for run in runs: + catalog.load(run.identity) + except OSError as error: + self._fail( + session, + f"tck [{name}]: could not read the feature files the run " + f"executed, so the results payload cannot name them: {error}", + ) + return + + unmatched = [ + run.identity for run in runs if catalog.pickle_for(run.identity) is None + ] + for identity in unmatched: + # The one failure mode this format exists to rule out: a scenario + # that ran and is missing from the results. Reported per scenario + # rather than as a count, because which one it is is the whole point. + self._fail( + session, + f"tck [{name}]: {identity.uri} scenario " + f"{identity.name!r}{_row(identity)} matched no Gherkin pickle, so " + f"the results payload does not account for it", + ) + + stream_path = directory / stream_file_name(name) + try: + digest = write_stream( + stream_path, + catalog, + runs, + implementation=TCK_IMPLEMENTATION, + implementation_version=distribution_version(TCK_DISTRIBUTION), + ) + envelope = suite.build(Results(location=stream_path.name, digest=digest)) + path = write_envelope(directory, name, envelope) + except OSError as error: + self._fail( + session, + f"tck [{name}]: could not write the conformance report " + f"to {directory}: {error}", + ) + return + + counts = ", ".join( + f"{count} {status}" for status, count in sorted(suite.counts().items()) + ) + self._say( + session, + f"tck [{name}]: report written to {path} with results in " + f"{stream_path.name} ({counts})", + ) + + def _identities_are_sound( + self, session: pytest.Session, suite: SuiteReport + ) -> bool: + """Whether every feature file this suite ran is named in the payload as itself. + + Two ways it might not be, and both are silent without this. A file that + is not one of the packaged assets must not be reported under their uri + prefix, or a consumer reading ``gherkin/errors.feature`` in the stream + has no way to tell that the specification did not write it. And two + files must not share a uri, or the stream carries one source for both + and the second file's scenarios are reported against the first's. + + A suite that fails either writes no report. Refusing is the point: a + document that presents an adopter's feature file as the specification's + is worse than no document, because it is the one thing a consumer cannot + check. + """ + name = suite.config.name + runs = suite.sorted_runs + + reserved = [ + problem + for run in runs + if (problem := reserved_prefix_problem(run.identity.uri, run.identity.path)) + ] + for problem in dict.fromkeys(reserved): + self._fail(session, f"tck [{name}]: {problem}") + + collisions = uri_collisions( + (run.identity.uri, run.identity.path) for run in runs + ) + for uri, paths in sorted(collisions.items()): + self._fail(session, f"tck [{name}]: {collision_problem(uri, paths)}") + + return not reserved and not collisions + + def _say(self, session: pytest.Session, message: str) -> None: + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line(message) + + def _fail(self, session: pytest.Session, message: str) -> None: + self._say(session, message) + session.exitstatus = pytest.ExitCode.INTERNAL_ERROR + + +def _row(identity: ScenarioIdentity) -> str: + if not identity.example: + return "" + row = " ".join(f"{header}={cell}" for header, cell in identity.example) + return f" [{row}]" + + +def _phase_outcome(report: pytest.TestReport) -> PhaseOutcome: + """Reduce a pytest phase report to what the conformance report needs.""" + xfail_reason: str | None = getattr(report, "wasxfail", None) + message = _skip_reason(report) if report.skipped else _failure_reason(report) + return PhaseOutcome( + when=report.when or "", + outcome=report.outcome, + xfail_reason=xfail_reason, + message=message, + start=getattr(report, "start", 0.0), + stop=getattr(report, "stop", 0.0), + ) + + +def scenario_run( + identity: ScenarioIdentity, + phases: list[PhaseOutcome], + steps: list[StepRun], +) -> ScenarioRun: + """Assemble one scenario's execution from what pytest reported about it. + + The scenario's own status is the most serious of its phases', so a scenario + whose steps passed and whose teardown then blew up is a failed scenario: the + phase that reports last must not be the one that decides. + """ + starts = [phase.start for phase in phases if phase.start] + stops = [phase.stop for phase in phases if phase.stop] + run = ScenarioRun( + identity=identity, + steps=list(steps), + started_ns=int(min(starts, default=0.0) * 1_000_000_000), + finished_ns=int(max(stops, default=0.0) * 1_000_000_000), + ) + for phase in phases: + status, message = classify_phase(phase) + result = StepRun( + status=status, + message=message, + started_ns=int(phase.start * 1_000_000_000), + finished_ns=int(phase.stop * 1_000_000_000), + ) + if phase.when == "setup": + run.setup = result + elif phase.when == "teardown": + run.teardown = result + upgraded = worse(run.status, status) + if upgraded is not run.status: + # The message belongs to whichever phase decided the verdict, so a + # teardown failure does not inherit the reason a passing call gave. + run.message = message + run.status = upgraded + return run + + +def classify_phase(phase: PhaseOutcome) -> tuple[Status, str]: + """Map one pytest phase onto a Cucumber status. + + The one decision that is not a rename: an expected failure is still a + failure. pytest reports an ``xfail`` as skipped and exits zero; the provider + did not satisfy the scenario, and a stream calling it anything else would + hide exactly the deviation the marker was added to keep visible. The + acknowledgement goes in the envelope's ``knownDeviations`` instead, which is + where a claim about the provider belongs. + """ + if phase.outcome == "skipped" and phase.xfail_reason is not None: + return Status.failed, _reason(f"expected failure: {phase.xfail_reason}") + if phase.outcome == "failed": + return Status.failed, phase.message or "failed" + if phase.outcome == "skipped": + return Status.skipped, phase.message or "skipped" + return Status.passed, "" + + +def _skip_reason(report: pytest.TestReport) -> str: + longrepr = report.longrepr + if isinstance(longrepr, tuple) and len(longrepr) == 3: + return _reason(str(longrepr[2]).removeprefix("Skipped: ")) + return _reason(str(longrepr)) if longrepr else "" + + +def _failure_reason(report: pytest.TestReport) -> str: + message = getattr(getattr(report.longrepr, "reprcrash", None), "message", "") + if not message: + message = str(report.longrepr) if report.longrepr else "" + return _reason(message) + + +def _reason(message: str) -> str: + collapsed = " ".join(message.split()) + if len(collapsed) <= _MAX_REASON: + return collapsed + return collapsed[: _MAX_REASON - 1].rstrip() + "…" + + +def observe_provider_name( + config: pytest.Config, tck_config: TckConfig, provider_name: str | None +) -> None: + """Record what the provider called itself, for the suite the run is in. + + The provider's own metadata name rather than the suite name, because the two + answer different questions: the suite name is chosen to read well in a + failure message, which makes it the configuration and it is reported as one. + """ + collector: ReportCollector | None = config.stash.get(COLLECTOR_KEY, None) + if collector is not None and provider_name: + collector.suite_for(tck_config).observe_provider_name(provider_name) + + +def bind_scenario(request: pytest.FixtureRequest) -> None: + """Tell the collector which suite this scenario's module is testing. + + Called from an autouse fixture that the capability gate depends on, so that a + scenario the gate stops has still contributed its suite. Only one scenario of + a module has to get this far, but the gate skips whole capabilities at a + time, and a module all of whose scenarios were skipped would otherwise have + no report to be written to. + """ + collector: ReportCollector | None = request.config.stash.get(COLLECTOR_KEY, None) + if collector is None or scenario_identity(request.node) is None: + # Checked before asking for the config so that a test which is not a TCK + # scenario instantiates nothing, which is the same bargain the capability + # gate makes. + return + try: + tck_config = typing.cast(TckConfig, request.getfixturevalue("tck_config")) + except pytest.FixtureLookupError: + return + collector.bind(request.node.nodeid, tck_config) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py new file mode 100644 index 000000000..c89aaa278 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/messages.py @@ -0,0 +1,667 @@ +"""The results payload: one run of the suite written as Cucumber Messages. + +The conformance report used to define its own per-scenario result list -- an +outcome enum, a tag list, and a field naming which Scenario Outline row an entry +came from. All three already exist in `Cucumber Messages`_, the ndjson protocol +Cucumber itself emits: it is maintained, schema'd, cross-language, and it carries +things a bespoke format would have had to invent, including the executed feature +source. So the report no longer describes results. It points at a stream of them. + +Nothing here knows about pytest. It takes a set of scenario outcomes and a set of +feature files and produces the stream; :mod:`.emitter` is what turns a pytest +session into the former. + +**Where the messages come from.** Two libraries, each doing the half it owns. + +``gherkin-official`` -- the reference Gherkin parser, already a dependency of +pytest-bdd -- produces the ``gherkinDocument`` and ``pickle`` payloads. Those +payloads *are* Messages: emitting Messages ndjson is what that library exists +for, so its output is used as it comes rather than round-tripped through +another representation that could quietly drop a field it does not model. + +``cucumber-messages`` -- the official Python types, from the same repository as +the protocol -- builds the execution half: ``Meta``, ``TestCase``, +``TestCaseStarted``, ``TestStepFinished`` and the rest. Hand-writing those dicts +would work until the protocol moved. + +**Why the feature files are parsed again.** pytest-bdd parses them with +``gherkin-official`` too, but converts the result into dataclasses of its own +that do not carry the AST node ids. Those ids are what a pickle refers to, and +what makes one Scenario Outline row distinguishable from another, so the stream +needs a parse whose ids it owns. Four small files, parsed once per session. + +.. _Cucumber Messages: https://github.com/cucumber/messages +""" + +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +import platform +import typing +from dataclasses import dataclass, field +from pathlib import Path, PurePath + +import cucumber_messages as cucumber +from gherkin.ast_builder import AstBuilder +from gherkin.parser import Parser +from gherkin.pickles.compiler import Compiler +from gherkin.stream.id_generator import IdGenerator + +from .capability import Capability, capability_for_tag + +__all__ = [ + "MESSAGES_FORMAT", + "FeatureCatalog", + "ScenarioIdentity", + "ScenarioKey", + "ScenarioRun", + "Status", + "StepRun", + "worse", + "write_stream", +] + +ScenarioKey = tuple[str, str, tuple[tuple[str, str], ...]] +"""What names one scenario: its feature file's uri, its name, and its Examples row. + +The one description of a scenario that a pytest-bdd node and a Gherkin pickle +can each produce without consulting the other, which is what makes it usable +both as the join key inside :class:`FeatureCatalog` and as the currency of a +comparison between the scenarios that ran and the ones that were shipped. +""" + +MESSAGES_FORMAT = "cucumber-messages" +"""The ``results.format`` value the envelope carries for this payload.""" + +Status = cucumber.TestStepResultStatus +"""The protocol's own status vocabulary, used rather than one of ours. + +Cucumber's seven statuses already draw the distinctions a conformance run needs, +and the four-value outcome enum this replaced drew a different set: it split +"did not run" into a capability the provider did not declare and one that cannot +apply to it, and merged "failed" with "the step was never reached". The first +distinction is not a property of the run at all -- it follows from the report's +declaration and the scenario's tags -- so it belongs in the envelope, once, and +not in every scenario. +""" + +_SEVERITY = { + Status.unknown: 0, + Status.passed: 1, + Status.skipped: 2, + Status.pending: 3, + Status.undefined: 4, + Status.ambiguous: 5, + Status.failed: 6, +} +"""How Cucumber orders its statuses, which is how a test case takes one. + +A test case is as bad as its worst step -- that is the rule a consumer applies to +derive a scenario's outcome from the stream, and this module applies the same one +so that what the stream says and what this package believes cannot diverge. +""" + + +def worse(left: Status, right: Status) -> Status: + """The more serious of two statuses.""" + return right if _SEVERITY[right] > _SEVERITY[left] else left + + +_MEDIA_TYPE = cucumber.SourceMediaType.text_x_cucumber_gherkin_plain + +_SETUP_HOOK_ID = "tck-setup" +_TEARDOWN_HOOK_ID = "tck-teardown" +"""The two hooks every test case carries, as the protocol models them. + +pytest runs a scenario in three phases and only the middle one executes Gherkin +steps: the capability gate skips during setup, and a provider that fails to shut +down fails during teardown. Neither has a pickle step to attach a result to, so +without hooks a gated skip would have to borrow the first step's result and a +teardown failure would be invisible behind a row of passed steps. Cucumber +represents exactly this with a ``Hook`` and a ``TestStep`` that references it, +which is what these are. +""" + +_MESSAGES_DISTRIBUTION = "cucumber-messages" +_UNKNOWN_VERSION = "unknown" + + +@dataclass(frozen=True) +class ScenarioIdentity: + """What a scenario is, independent of how it turned out. + + Established at collection, from the pytest node alone, so that a scenario + skipped before a single step ran is identified exactly as fully as one that + passed. That is what lets the stream account for every scenario rather than + only for the ones that got far enough to be interesting. + """ + + uri: str + """The feature file as Cucumber names it, e.g. ``gherkin/errors.feature``. + + Slash-separated on every platform, and the same string in ``Source``, + ``GherkinDocument`` and ``Pickle``, which is what ties the three together. + """ + + path: Path + """Where that file actually is, so its source can be read and parsed.""" + + name: str + """The scenario name as the feature file spells it. + + For a Scenario Outline this is the template name, shared by every row -- + which is why it is not on its own an identity. + """ + + tags: tuple[str, ...] + + example: tuple[tuple[str, str], ...] = () + """The Examples row, as header/cell pairs, for a scenario from an outline. + + Not reported: Messages carries row identity as the pickle's AST node ids, + which is where four independent implementations of a bespoke ``example`` + field were each converging by hand. It survives here only as the join key + that matches a pytest node to its pickle -- pytest-bdd parametrises the + generated test over the row, and the row is the one thing both sides of that + join can see. + """ + + def capabilities(self) -> tuple[Capability, ...]: + """The capabilities this scenario's tags gate it behind.""" + gated = (capability_for_tag(tag) for tag in self.tags) + return tuple(capability for capability in gated if capability is not None) + + +@dataclass(frozen=True) +class StepRun: + """What happened to one step, as the protocol records it.""" + + status: Status + message: str = "" + exception_type: str = "" + started_ns: int = 0 + finished_ns: int = 0 + + +@dataclass +class ScenarioRun: + """One scenario's execution: its verdict, and what each phase did.""" + + identity: ScenarioIdentity + status: Status = Status.unknown + message: str = "" + setup: StepRun | None = None + teardown: StepRun | None = None + steps: list[StepRun] = field(default_factory=list) + """Step results in execution order, as far as execution got. + + Shorter than the pickle's step list whenever a scenario stopped early, which + is the normal case for a failure and the whole list for a skip. The stream + pads the difference with ``SKIPPED``, which is what Cucumber means by it. + """ + + started_ns: int = 0 + finished_ns: int = 0 + + +@dataclass(frozen=True) +class _Pickle: + """One compiled pickle, reduced to what the stream needs to refer to it.""" + + id: str + step_ids: tuple[str, ...] + payload: dict[str, typing.Any] + + +class FeatureCatalog: + """The feature files a run executed, parsed into Messages and indexed. + + Indexed by what both sides of the join can see: the file, the scenario name + as the feature file spells it, and the Examples row. A pytest-bdd node knows + those three; a pickle can be made to yield them by following its AST node ids + back to the scenario and the table row it was compiled from. Matching on the + pickle's own name would not do, because the compiler interpolates outline + parameters into it and pytest-bdd does not. + """ + + def __init__(self) -> None: + # One generator across the whole session, so ids are unique across + # feature files rather than only within one -- a stream is a single id + # space and two files numbering from zero would collide. + self._ids = IdGenerator() + self._sources: dict[str, str] = {} + self._documents: dict[str, dict[str, typing.Any]] = {} + self._pickles: dict[str, list[_Pickle]] = {} + self._index: dict[ScenarioKey, _Pickle] = {} + + @property + def uris(self) -> list[str]: + return sorted(self._documents) + + @property + def scenario_keys(self) -> frozenset[ScenarioKey]: + """Every scenario the loaded feature files define, row by row. + + What a feature file *asks*, as opposed to what a run executed. Taken + from the compiled pickles rather than from the AST so that a Scenario + Outline contributes one entry per row, which is what the runner + generates and therefore what a comparison has to be made in. + """ + return frozenset(self._index) + + def load(self, identity: ScenarioIdentity) -> None: + """Parse the feature file this scenario came from, once.""" + self.load_file(identity.uri, identity.path) + + def load_file(self, uri: str, path: Path) -> None: + """Parse one feature file under the uri it will be reported by, once. + + Separate from :meth:`load` because the canonical set has to be parsed + with no run in hand: the question "did every shipped scenario execute" + is asked of feature files, not of outcomes. + """ + if uri in self._documents: + return + source = path.read_text(encoding="utf-8") + document: dict[str, typing.Any] = Parser( + ast_builder=AstBuilder(self._ids) + ).parse(source) + document["uri"] = uri + pickles: list[dict[str, typing.Any]] = Compiler(self._ids).compile(document) + + self._sources[uri] = source + self._documents[uri] = document + self._pickles[uri] = [ + _Pickle( + id=str(pickle["id"]), + step_ids=tuple(str(step["id"]) for step in pickle.get("steps") or ()), + payload=pickle, + ) + for pickle in pickles + ] + self._index_pickles(uri, document, pickles) + + def _index_pickles( + self, + uri: str, + document: dict[str, typing.Any], + pickles: list[dict[str, typing.Any]], + ) -> None: + names, rows = _ast_index(document) + for pickle, entry in zip(pickles, self._pickles[uri], strict=True): + ast_node_ids = [str(node) for node in pickle["astNodeIds"]] + name = names.get(ast_node_ids[0], str(pickle["name"])) + row = rows.get(ast_node_ids[1], ()) if len(ast_node_ids) > 1 else () + self._index.setdefault((uri, name, row), entry) + + def pickle_for(self, identity: ScenarioIdentity) -> _Pickle | None: + """The pickle this scenario was compiled from, or ``None`` if unmatched. + + ``None`` is a defect rather than a possibility to tolerate: a scenario + that ran and has no pickle cannot appear in the stream, which is the one + failure mode the report exists to rule out. The caller fails the run. + """ + return self._index.get((identity.uri, identity.name, identity.example)) + + def source_envelopes(self) -> typing.Iterator[dict[str, typing.Any]]: + """The ``Source``, ``GherkinDocument`` and ``Pickle`` messages, in order. + + The source comes first because everything after it refers to it, and it + is the reason this format beats recording an asset revision: a consumer + can read the questions that were actually asked rather than trusting a + commit hash to stand for them. + """ + for uri in self.uris: + yield _envelope( + cucumber.Envelope( + source=cucumber.Source( + data=self._sources[uri], media_type=_MEDIA_TYPE, uri=uri + ) + ) + ) + yield {"gherkinDocument": self._documents[uri]} + for entry in self._pickles[uri]: + yield {"pickle": entry.payload} + + +def _ast_index( + document: dict[str, typing.Any], +) -> tuple[dict[str, str], dict[str, tuple[tuple[str, str], ...]]]: + """Map AST node ids onto scenario names and Examples rows. + + Walks the parsed document rather than the pickles, because the pickle is + where the outline has already been expanded: the scenario name it carries has + the row's parameters substituted into it, and the row itself has become a + list of interpolated step texts. The AST still has both separately, which is + what a pytest-bdd node can be compared against. + """ + names: dict[str, str] = {} + rows: dict[str, tuple[tuple[str, str], ...]] = {} + + def visit(children: typing.Iterable[dict[str, typing.Any]]) -> None: + for child in children: + if "rule" in child: + visit(child["rule"].get("children") or ()) + continue + scenario = child.get("scenario") + if scenario is None: + continue + names[str(scenario["id"])] = str(scenario["name"]) + for examples in scenario.get("examples") or (): + header = examples.get("tableHeader") + if header is None: + continue + headers = [str(cell["value"]) for cell in header["cells"]] + for row in examples.get("tableBody") or (): + cells = [str(cell["value"]) for cell in row["cells"]] + rows[str(row["id"])] = tuple(zip(headers, cells, strict=False)) + + feature = document.get("feature") + if feature is not None: + visit(feature.get("children") or ()) + return names, rows + + +def write_stream( + path: Path, + catalog: FeatureCatalog, + runs: typing.Sequence[ScenarioRun], + implementation: str, + implementation_version: str, +) -> str: + """Write the stream and return its digest as ``sha256:``. + + The digest is returned rather than recomputed by the caller so that what is + hashed is exactly the bytes that were written, which is the only version of + that claim worth putting in the envelope. + """ + digest = hashlib.sha256() + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="\n") as stream: + for envelope in _stream(catalog, runs, implementation, implementation_version): + line = ( + json.dumps(envelope, ensure_ascii=False, separators=(",", ":")) + "\n" + ) + stream.write(line) + digest.update(line.encode("utf-8")) + return f"sha256:{digest.hexdigest()}" + + +def _stream( + catalog: FeatureCatalog, + runs: typing.Sequence[ScenarioRun], + implementation: str, + implementation_version: str, +) -> typing.Iterator[dict[str, typing.Any]]: + started = min((run.started_ns for run in runs), default=0) + finished = max((run.finished_ns for run in runs), default=started) + + yield _envelope( + cucumber.Envelope(meta=_meta(implementation, implementation_version)) + ) + yield _envelope( + cucumber.Envelope( + test_run_started=cucumber.TestRunStarted( + id=_RUN_ID, timestamp=_timestamp(started) + ) + ) + ) + yield from _hook_envelopes() + yield from catalog.source_envelopes() + + for index, run in enumerate(runs): + entry = catalog.pickle_for(run.identity) + if entry is None: + # Ruled out by the caller before it gets here; skipping rather than + # raising keeps a defect in the report from destroying the run's own + # exit status, which says something the report cannot. + continue + yield from _test_case_envelopes(f"test-case-{index}", entry, run) + + yield _envelope( + cucumber.Envelope( + test_run_finished=cucumber.TestRunFinished( + success=all(run.status is not Status.failed for run in runs), + timestamp=_timestamp(finished), + test_run_started_id=_RUN_ID, + ) + ) + ) + + +_RUN_ID = "tck-run" + + +def _test_case_envelopes( + test_case_id: str, entry: _Pickle, run: ScenarioRun +) -> typing.Iterator[dict[str, typing.Any]]: + """One scenario: its test case, and what each of its steps did.""" + started_id = f"{test_case_id}-started" + setup_step_id = f"{test_case_id}-setup" + teardown_step_id = f"{test_case_id}-teardown" + + test_steps = [ + cucumber.TestStep(id=setup_step_id, hook_id=_SETUP_HOOK_ID), + *( + cucumber.TestStep(id=f"{test_case_id}-{position}", pickle_step_id=step_id) + for position, step_id in enumerate(entry.step_ids) + ), + cucumber.TestStep(id=teardown_step_id, hook_id=_TEARDOWN_HOOK_ID), + ] + yield _envelope( + cucumber.Envelope( + test_case=cucumber.TestCase( + id=test_case_id, + pickle_id=entry.id, + test_run_started_id=_RUN_ID, + test_steps=test_steps, + ) + ) + ) + yield _envelope( + cucumber.Envelope( + test_case_started=cucumber.TestCaseStarted( + attempt=0, + id=started_id, + test_case_id=test_case_id, + timestamp=_timestamp(run.started_ns), + ) + ) + ) + + for step_id, result in zip( + (step.id for step in test_steps), _step_runs(entry, run), strict=True + ): + yield from _step_envelopes(started_id, step_id, result) + + yield _envelope( + cucumber.Envelope( + test_case_finished=cucumber.TestCaseFinished( + test_case_started_id=started_id, + timestamp=_timestamp(run.finished_ns), + will_be_retried=False, + ) + ) + ) + + +def _step_runs(entry: _Pickle, run: ScenarioRun) -> list[StepRun]: + """Every step of the pickle and both hooks, including what never ran. + + A scenario that stopped early -- the ordinary shape of both a failure and a + skip -- has fewer recorded results than the pickle has steps, and the + remainder are reported ``SKIPPED``, which is what Cucumber means by it and + what makes the stream account for the whole scenario rather than the part of + it that executed. + + The last thing this does is make sure the scenario's own verdict survives the + trip. A consumer reads a test case's outcome as the worst of its steps, so a + verdict no step accounts for would be lost: a strict ``xfail`` that passed is + a failed scenario every one of whose steps passed, and there are other ways + for a runner to fail a test case between its steps. Whatever is left over is + attached to the after-hook, which is where a test case failing outside its + own steps belongs. + """ + unreached = StepRun( + status=Status.skipped, + started_ns=run.finished_ns, + finished_ns=run.finished_ns, + ) + steps = list(run.steps[: len(entry.step_ids)]) + steps += [unreached] * (len(entry.step_ids) - len(steps)) + setup = run.setup or StepRun( + status=Status.passed, + started_ns=run.started_ns, + finished_ns=run.started_ns, + ) + teardown = run.teardown or StepRun( + status=Status.passed, + started_ns=run.finished_ns, + finished_ns=run.finished_ns, + ) + + reported = Status.unknown + for result in (setup, *steps, teardown): + reported = worse(reported, result.status) + if worse(reported, run.status) is not reported: + teardown = StepRun( + status=run.status, + message=run.message, + started_ns=teardown.started_ns, + finished_ns=teardown.finished_ns, + ) + return [setup, *steps, teardown] + + +def _step_envelopes( + started_id: str, step_id: str, result: StepRun +) -> typing.Iterator[dict[str, typing.Any]]: + yield _envelope( + cucumber.Envelope( + test_step_started=cucumber.TestStepStarted( + test_case_started_id=started_id, + test_step_id=step_id, + timestamp=_timestamp(result.started_ns), + ) + ) + ) + exception = ( + cucumber.Exception(type=result.exception_type, message=result.message or None) + if result.exception_type + else None + ) + yield _envelope( + cucumber.Envelope( + test_step_finished=cucumber.TestStepFinished( + test_case_started_id=started_id, + test_step_id=step_id, + test_step_result=cucumber.TestStepResult( + duration=_duration(result.finished_ns - result.started_ns), + status=result.status, + message=result.message or None, + exception=exception, + ), + timestamp=_timestamp(result.finished_ns), + ) + ) + ) + + +def _hook_envelopes() -> typing.Iterator[dict[str, typing.Any]]: + for hook_id, hook_type, name in ( + ( + _SETUP_HOOK_ID, + cucumber.HookType.before_test_case, + "tck setup: capability gate, provider registration", + ), + ( + _TEARDOWN_HOOK_ID, + cucumber.HookType.after_test_case, + "tck teardown: provider shutdown", + ), + ): + yield _envelope( + cucumber.Envelope( + hook=cucumber.Hook( + id=hook_id, + name=name, + type=hook_type, + source_reference=cucumber.SourceReference( + uri="openfeature/contrib/tools/tck/plugin.py" + ), + ) + ) + ) + + +def _meta(implementation: str, implementation_version: str) -> cucumber.Meta: + """Who produced the stream, and against which protocol version.""" + return cucumber.Meta( + cpu=cucumber.Product(name=platform.machine() or _UNKNOWN_VERSION), + implementation=cucumber.Product( + name=implementation, version=implementation_version + ), + os=cucumber.Product(name=platform.system() or _UNKNOWN_VERSION), + protocol_version=_protocol_version(), + runtime=cucumber.Product( + name=platform.python_implementation(), version=platform.python_version() + ), + ) + + +def messages_protocol_version() -> str: + """The Messages release this stream was produced against. + + Exposed because the report envelope has to record it too. Messages is + versioned and the implementations pin different releases -- this one is on + 34.2.0 while the Go TCK builds against v21 -- so a consumer holding two + reports cannot assume one schema validates both. Sharing this one function + with the stream's own Meta message means the envelope and the stream cannot + disagree about which release produced it. + """ + return _protocol_version() + + +def _protocol_version() -> str: + """The Messages version this stream is written against. + + Read from the installed library rather than written down, because the library + is what decides: a version pinned here would go on claiming 34.2.0 after a + dependency bump moved the types underneath it. + """ + try: + return importlib.metadata.version(_MESSAGES_DISTRIBUTION) + except importlib.metadata.PackageNotFoundError: + return _UNKNOWN_VERSION + + +def _envelope(envelope: cucumber.Envelope) -> dict[str, typing.Any]: + converted: dict[str, typing.Any] = cucumber.message_converter.to_dict(envelope) + return converted + + +def _timestamp(nanoseconds: int) -> cucumber.Timestamp: + return cucumber.Timestamp( + seconds=nanoseconds // 1_000_000_000, nanos=nanoseconds % 1_000_000_000 + ) + + +def _duration(nanoseconds: int) -> cucumber.Duration: + nanoseconds = max(nanoseconds, 0) + return cucumber.Duration( + seconds=nanoseconds // 1_000_000_000, nanos=nanoseconds % 1_000_000_000 + ) + + +def feature_uri(relative_filename: str) -> str: + """Normalise pytest-bdd's relative feature path into a Cucumber uri. + + pytest-bdd builds it with ``os.path.join``, so on Windows it arrives + backslash-separated. A uri is slash-separated everywhere, and the same string + has to appear in the ``Source``, the ``GherkinDocument`` and every ``Pickle`` + or nothing ties them together -- so a report emitted on Windows would + otherwise not be comparable with one emitted on Linux. + """ + return PurePath(relative_filename).as_posix() diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py index f0e784844..a65709a9c 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/plugin.py @@ -30,6 +30,7 @@ ) from .compose import ComposeBackend, RunningBackend, run_compose_backend from .config import TckConfig +from .emitter import ReportEmitter, bind_scenario, observe_provider_name from .extensions import canonical_tags, is_canonical, uri_for from .state import TckState @@ -46,18 +47,25 @@ def pytest_configure(config: pytest.Config) -> None: - """Register the capability tags as markers. + """Register the capability tags as markers, and the report emitter. pytest-bdd turns every Gherkin tag into a marker with ``getattr(pytest.mark, tag)`` without registering it, which raises ``PytestUnknownMarkWarning`` for each one -- noise at best, and a hard failure in a project configured with ``-W error``. + + The emitter is registered unconditionally even though it writes nothing + unless :data:`~.report.REPORT_DIR_ENV` is set. Accumulating the outcomes + costs a dictionary entry per scenario, and deciding at the end of the session + rather than at the start is one fewer way for a run to discover too late that + it was not recording. """ for capability in Capability: config.addinivalue_line( "markers", f"{capability.value}: OpenFeature provider TCK capability {capability.tag}", ) + config.pluginmanager.register(ReportEmitter(config), "openfeature-tck-report") def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: @@ -215,7 +223,9 @@ def tck_config(tck_backend: RunningBackend) -> TckConfig: @pytest.fixture -def tck_state(tck_config: TckConfig) -> typing.Iterator[TckState]: +def tck_state( + request: pytest.FixtureRequest, tck_config: TckConfig +) -> typing.Iterator[TckState]: """Per-scenario state, carried between step definitions.""" # Resetting here rather than in an autouse fixture ties the reset to the # scenarios that actually use the TCK, and guarantees it happens after the @@ -224,11 +234,22 @@ def tck_state(tck_config: TckConfig) -> typing.Iterator[TckState]: tck_config.control.prepare_scenario() state = TckState(config=tck_config) yield state + # The provider is identified in the report by what it called itself, and the + # only thing that ever holds an instance is the scenario that made one. + observe_provider_name(request.config, tck_config, state.provider_name) state.teardown() @pytest.fixture(autouse=True) -def _tck_scenario_gate(request: pytest.FixtureRequest) -> None: +def _tck_report_binding(request: pytest.FixtureRequest) -> None: + """Attribute this scenario to its suite before anything can skip it.""" + bind_scenario(request) + + +@pytest.fixture(autouse=True) +def _tck_scenario_gate( + request: pytest.FixtureRequest, _tck_report_binding: None +) -> None: """Autouse wrapper around the two gates a scenario passes before it runs. Plain functions behind a one-line fixture, so the decisions they make can be @@ -237,6 +258,11 @@ def _tck_scenario_gate(request: pytest.FixtureRequest) -> None: The order lives in :func:`apply_scenario_gates` for the same reason: it is a decision, and one a test should be able to make without a session. + + ``_tck_report_binding`` is requested rather than left to autouse ordering so + that the scenario has reached its suite before this fixture can skip it. A + scenario skipped here is exactly the one the conformance report must account + for, and one that never reached a suite could not be reported at all. """ apply_scenario_gates(request) diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py new file mode 100644 index 000000000..556c079a2 --- /dev/null +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/report.py @@ -0,0 +1,493 @@ +"""The conformance report envelope: what was tested, what it claims, where the results are. + +A run of the suite produces a pass or a fail on a terminal, which is enough for +the person who started it and useless to anyone else. The report is the same run +written down in a form something other than a human can read -- a comparison +page, an aggregator, a release gate -- against a schema owned by the +specification rather than by this package, so that four languages emit the same +document. + +This document no longer describes the results. It is an envelope that identifies +the subject and points at a :mod:`Cucumber Messages <.messages>` stream beside +it. The per-scenario outcome list, the outcome enum and the field naming which +Scenario Outline row an entry came from have all been deleted, because Messages +already carries every one of them -- along with the executed feature source, +which no bespoke format had. + +Two things stay here, because Messages has no slot for either. + +The **declaration** is an input to reading the results rather than a summary of +them. A skipped scenario in the stream says the question was not put to this +provider; only the declaration says whether that is because the provider +declines the capability. Given the declaration and a scenario's tags -- both +present -- the reason for a skip follows, so it no longer has to be transported +once per scenario. + +The **tested subject**: no standard results format has a slot for "the provider +under test". Messages records the runtime and the OS, which is what produced the +answers, not what was being asked about. + +See https://github.com/open-feature/spec/issues/424 for the format and +``specification/assets/provider-tck/report/`` for the schema. +""" + +from __future__ import annotations + +import importlib.metadata +import importlib.resources +import json +import re +import typing +from dataclasses import dataclass, field +from pathlib import Path + +from .config import TckConfig +from .control import backend_type_of +from .messages import ( + MESSAGES_FORMAT, + ScenarioIdentity, + ScenarioRun, + StepRun, + messages_protocol_version, +) + +__all__ = [ + "REPORT_DIR_ENV", + "SCHEMA_VERSION", + "PhaseOutcome", + "ReportCollector", + "Results", + "SuiteReport", + "envelope_file_name", + "stream_file_name", +] + +REPORT_DIR_ENV = "TCK_REPORT_DIR" +"""Names the directory a conformance report is written to. + +An environment variable rather than a :class:`~.config.TckConfig` field, so that +emitting a report is a property of the *run* and not of the code: CI sets it, a +developer running the suite locally does not, and no adopter changes a line to +publish one. Each suite writes two files -- ``/.json``, the envelope, +and ``/.ndjson``, the results the envelope points at -- so several +suites in one pytest session, flagd's RPC and in-process resolvers say, each +produce their own pair without colliding. + +Unset means no report, which is the default and is not an error. +""" + +SCHEMA_VERSION = "1" +"""The major version of the report schema this emitter produces.""" + +TCK_IMPLEMENTATION = "python-sdk-contrib/tools/openfeature-tck" +"""Which TCK implementation produced the report, as the schema spells it.""" + +PROVIDER_LANGUAGE = "python" + +SDK_DISTRIBUTION = "openfeature-sdk" +TCK_DISTRIBUTION = "openfeature-tck" + +UNKNOWN = "unknown" +"""Stands in for an identity that could not be read. + +Seven characters, which is the schema's minimum for ``tck.specRevision``, so a +build that could not reach git still emits a document that validates and says +plainly that it does not know rather than inventing a commit. +""" + +_PACKAGE = "openfeature.contrib.tools.tck" + +_REVISION_FILE = "spec_revision.json" +"""Written at build time from the spec submodule; see ``hatch_build_sync.py``. + +Read from a data file rather than the submodule because the submodule is not in +the published wheel: an adopter installing this package has no ``spec/`` +directory to interrogate, and the revision the assets came from is exactly what +the report has to name. +""" + +_TAG_PATTERN = re.compile(r"^[a-z0-9-]+$") +"""What the schema accepts as a tag, minus the leading at-sign. + +Tags that do not match are dropped rather than emitted, because an invalid +document helps nobody; the canonical feature files carry none, so this only bites +a feature file that has been forked, which is itself worth noticing. +""" + +_UNSAFE_IN_FILENAME = re.compile(r"[^A-Za-z0-9._-]") + + +@dataclass(frozen=True) +class Results: + """Where the executed results are, and what covers them.""" + + location: str + digest: str + format: str = MESSAGES_FORMAT + + def as_json(self) -> dict[str, typing.Any]: + # The format's version is recorded alongside its name because Cucumber + # Messages is versioned and the four implementations pin different + # releases. Without it a consumer validating this stream has to guess + # which schema to use, and guessing wrong is worse than not checking: a + # later schema accepts messages this producer could not have emitted, + # and an earlier one rejects messages that are perfectly valid. + document = { + "format": self.format, + "formatVersion": messages_protocol_version(), + "location": self.location, + } + if self.digest: + document["digest"] = self.digest + return document + + +@dataclass(frozen=True) +class PhaseOutcome: + """One pytest phase report, reduced to what the conformance report needs. + + Reduced rather than kept, because a :class:`pytest.TestReport` holds a + formatted traceback and holding a session's worth of them to classify at the + end would be a memory leak with a nice name. + """ + + when: str + """``setup``, ``call`` or ``teardown``.""" + + outcome: str + """``passed``, ``failed`` or ``skipped``, as pytest decided.""" + + xfail_reason: str | None = None + """Set when pytest marked this an expected failure.""" + + message: str = "" + """The skip reason, or the failure's headline, already trimmed.""" + + start: float = 0.0 + stop: float = 0.0 + + +Resolver = typing.Callable[ + [ScenarioIdentity, "list[PhaseOutcome]", "list[StepRun]"], ScenarioRun +] +"""Turns what pytest reported about one scenario into what the stream records. + +A callable rather than a method, because the mapping is entirely about pytest -- +which phase means what, and that an expected failure is still a failure -- and +this module deliberately knows nothing about pytest. +""" + + +@dataclass +class SuiteReport: + """What one suite -- one :class:`~.config.TckConfig` -- accumulates as it runs. + + Runs are keyed by pytest node id rather than appended to a list, which is + what makes "every scenario appears exactly once" a property of the structure + instead of a promise made by the code that fills it. + """ + + config: TckConfig + provider_name: str | None = None + runs: dict[str, ScenarioRun] = field(default_factory=dict) + + def observe_provider_name(self, name: str) -> None: + """Remember what the provider called itself through its own metadata. + + Last one wins, and they should all agree: a suite tests one provider. + """ + if name: + self.provider_name = name + + def record(self, node_id: str, run: ScenarioRun) -> None: + self.runs[node_id] = run + + @property + def sorted_runs(self) -> list[ScenarioRun]: + """The runs in a stable order: feature, then scenario, then row. + + Sorted by the whole identity, the Examples row included, so that two rows + of one outline reach the stream in the feature file's terms rather than + in whichever order the dictionary happened to be filled. + """ + return sorted( + self.runs.values(), + key=lambda run: ( + run.identity.uri, + run.identity.name, + run.identity.example, + ), + ) + + def counts(self) -> dict[str, int]: + """Status tallies, for a log line and for the tests that check them.""" + tally: dict[str, int] = {} + for run in self.runs.values(): + key = run.status.value.lower() + tally[key] = tally.get(key, 0) + 1 + return tally + + def build(self, results: Results) -> dict[str, typing.Any]: + """Assemble the envelope around a results payload already written.""" + document: dict[str, typing.Any] = { + "schemaVersion": SCHEMA_VERSION, + "provider": { + # What the provider calls itself, not the suite name: the suite + # name is chosen to read well in a failure message -- "flagd-rpc" + # -- which makes it the configuration, and it is reported as one. + # A provider with two materially different modes therefore + # produces two reports that are not interchangeable. + "name": self.provider_name or self.config.name, + "language": PROVIDER_LANGUAGE, + "configuration": self.config.name, + }, + "sdk": { + "name": SDK_DISTRIBUTION, + "version": distribution_version(SDK_DISTRIBUTION), + }, + "tck": { + "implementation": TCK_IMPLEMENTATION, + "version": distribution_version(TCK_DISTRIBUTION), + "specRevision": spec_revision(), + }, + "declaration": self._declaration(), + "results": results.as_json(), + } + + # Always emitted, never conditionally: the schema requires `backend` + # at the top level and `backendType` within it, because a provider with + # no backend still had its flag state manipulated somehow and how is + # what the rest of the document is worth. + document["backend"] = self._backend() + deviations = [deviation.as_json() for deviation in self.config.known_deviations] + if deviations: + # Omitted rather than emitted empty: stating no deviations is a + # claim, and an emitter that always emitted the field would make that + # claim on every provider's behalf whether or not it had checked. + document["knownDeviations"] = deviations + return document + + def _declaration(self) -> dict[str, typing.Any]: + """What the provider claims, which is what makes a skip legible. + + A skip in the payload says only that the question was not put to this + provider; this says whether that is because the capability was not + claimed. Given the two, the reason a scenario was skipped follows from + its own tags, which is why one skip carrying its reason is the whole + mechanism and nothing here restates it. + + The set is not filtered. A reserved capability -- one no scenario + carries, which the schema forbids in this block -- cannot be in a + ``TckConfig`` at all: it is refused at construction, and the default + capability set excludes it. Dropping one silently at emission time would + make a rejected configuration look like an accepted one, and leave the + adopter who wrote it believing the declaration they read back was the + declaration they asked for. + """ + return {"declared": self.config.sorted_capabilities} + + def _backend(self) -> dict[str, typing.Any]: + """What was driven, and which path drove it. + + ``backendType`` comes from the control through :func:`backend_type_of`, + never from its class: nothing outside a control can tell whether it + spoke the normative HTTP API or reached into this process, so a control + that does not say is reported as ``"custom"`` rather than guessed at. + + ``description`` is free text for a person and the schema leaves it + optional, so an empty one is left out rather than emitted blank. + """ + backend: dict[str, typing.Any] = { + "backendType": backend_type_of(self.config.control) + } + description = self.config.control.description + if description: + backend["description"] = description + return backend + + +class ReportCollector: + """Session-wide accumulator: which scenario belongs to which suite, and how it went. + + One pytest session can run several suites -- the TCK's own tests run two, and + a provider with more than one resolver runs one per resolver -- so outcomes + are attributed to a suite rather than to the session, and each suite writes + its own pair of files. + + Scenarios are enumerated at collection and resolved into runs only at the end + of the session. The order matters. A scenario skipped by a marker never runs + a fixture, so a design that learned of a scenario when its fixtures ran would + leave it out of the stream entirely -- and a report that silently omits what + it skipped satisfies "a skip is never reported as passed" while still + misleading the person reading it. + """ + + def __init__(self) -> None: + # Suites are keyed by the identity of their TckConfig, so two suites that + # happen to share a name stay distinct here; that collision is caught + # where it actually bites, when their file names turn out to be equal. + self._suites: dict[int, SuiteReport] = {} + self._suite_by_group: dict[str, SuiteReport] = {} + self._collected: dict[str, tuple[str, ScenarioIdentity]] = {} + self._phases: dict[str, list[PhaseOutcome]] = {} + self._steps: dict[str, list[StepRun]] = {} + + def collect(self, node_id: str, group: str, identity: ScenarioIdentity) -> None: + """Note that this scenario exists, and which group of tests it came from. + + The group is the module the scenario was generated into. pytest-bdd's + ``scenarios()`` injects its tests into the module that called it, and a + module resolves one ``tck_config``, so the module is what says which + suite a scenario belongs to -- and it says so without running anything. + """ + self._collected[node_id] = (group, identity) + + @property + def identities(self) -> list[ScenarioIdentity]: + """Every collected scenario, so the feature files can be parsed once.""" + return [identity for _, identity in self._collected.values()] + + def observe(self, node_id: str, phase: PhaseOutcome) -> None: + """Record one phase's result for a scenario, if it is one of ours.""" + if node_id in self._collected: + self._phases.setdefault(node_id, []).append(phase) + + def observe_step(self, node_id: str, step: StepRun) -> None: + """Record what one Gherkin step did, in execution order. + + Recorded as it happens rather than reconstructed afterwards, because + pytest reports a scenario and not its steps: only the runner knows which + step of eight failed, and a stream that marked all eight failed would be + saying something untrue about seven of them. + """ + self._steps.setdefault(node_id, []).append(step) + + def bind(self, node_id: str, config: TckConfig) -> None: + """Learn which suite a group of scenarios is testing. + + Called from a fixture, because the ``TckConfig`` is a fixture value and + there is no way to know it without asking for it. Only one scenario of a + group has to get this far for the whole group to be attributed. + """ + entry = self._collected.get(node_id) + if entry is not None: + self._suite_by_group[entry[0]] = self.suite_for(config) + + def suite_for(self, config: TckConfig) -> SuiteReport: + return self._suites.setdefault(id(config), SuiteReport(config=config)) + + @property + def suites(self) -> list[SuiteReport]: + return list(self._suites.values()) + + def resolve(self, resolver: Resolver) -> list[str]: + """Turn the collected phases into runs, and report what could not be. + + Returns the problems, one string each, and they are meant to be shouted + about rather than logged: a scenario that ran but is missing from the + stream is the one failure mode this format exists to rule out. + """ + problems: list[str] = [] + for node_id, (group, identity) in sorted(self._collected.items()): + suite = self._suite_by_group.get(group) + if suite is None: + problems.append( + f"{node_id}: no TckConfig was resolved for {group}, so its " + f"outcome belongs to no suite and is missing from every report" + ) + continue + phases = self._phases.get(node_id) + if not phases: + problems.append( + f"{node_id}: was collected but never ran, so the report for " + f"{suite.config.name!r} does not account for it" + ) + continue + suite.record( + node_id, resolver(identity, phases, self._steps.get(node_id, [])) + ) + return problems + + +def normalise_tags(tags: typing.Iterable[str]) -> tuple[str, ...]: + """Turn Gherkin tags as pytest-bdd holds them into the form the schema wants. + + pytest-bdd strips the leading at-sign; the schema requires it back. + """ + return tuple(sorted(f"@{tag}" for tag in tags if _TAG_PATTERN.match(tag))) + + +def report_stem(suite_name: str) -> str: + """Turn a suite name into a file name stem. + + Suite names are chosen to read well in a failure message rather than to be + path-safe, so anything not obviously safe becomes a hyphen. Without this a + suite named ``flagd/rpc`` would quietly write outside the directory it was + given. + """ + cleaned = _UNSAFE_IN_FILENAME.sub("-", suite_name).strip("-.") + return cleaned or "report" + + +def envelope_file_name(suite_name: str) -> str: + return f"{report_stem(suite_name)}.json" + + +def stream_file_name(suite_name: str) -> str: + """The results payload, beside the envelope that points at it. + + A sibling rather than a subdirectory so that ``results.location`` is a bare + file name, which is a relative reference that survives the whole pair being + moved, uploaded or served from somewhere other than where it was written. + """ + return f"{report_stem(suite_name)}.ndjson" + + +def write_envelope( + directory: Path, suite_name: str, document: dict[str, typing.Any] +) -> Path: + """Write one envelope, returning where it went.""" + directory.mkdir(parents=True, exist_ok=True) + path = directory / envelope_file_name(suite_name) + path.write_text( + json.dumps(document, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + return path + + +def distribution_version(distribution: str) -> str: + """Read an installed distribution's version. + + Read rather than declared, because a declared version is a second place to + be wrong: the report would go on claiming 0.8.2 after a dependency bump moved + the actual code underneath it. + """ + try: + return importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError: + return UNKNOWN + + +def spec_revision() -> str: + """Return the spec commit these feature files came from. + + Captured at build time rather than read here, because the submodule that + holds the answer is not in the wheel. A build that could not reach git says + so with :data:`UNKNOWN` instead of inventing a commit, and an installation + old enough to predate the generated file degrades the same way rather than + failing to emit a report at all. + + The asset tree hash that used to accompany it is gone. It was carried so that + a consumer could tell whether two runs executed the same questions; the + results payload now carries the executed feature source itself, which answers + that directly rather than by proxy. + """ + reference = importlib.resources.files(_PACKAGE) / _REVISION_FILE + try: + data = json.loads(reference.read_text(encoding="utf-8")) + except (OSError, ValueError): + return UNKNOWN + if not isinstance(data, dict): + return UNKNOWN + revision = data.get("specRevision") + return revision if isinstance(revision, str) and len(revision) >= 7 else UNKNOWN diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/state.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/state.py index 2d59d19b8..40dd1599e 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/state.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/state.py @@ -249,6 +249,13 @@ class TckState: they verify is the provider's own ``shutdown``, ``initialize`` and ``get_metadata`` rather than the SDK's handling of them. """ + provider_name: str | None = None + """What the provider called itself through its own metadata. + + Observed rather than configured, because it is what the conformance report + identifies the provider by: ``TckConfig.name`` is chosen to read well in a + failure message, which makes it the *configuration* rather than the provider. + """ flag_key: str | None = None flag_type: FlagType | None = None default_value: typing.Any = None diff --git a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/provider_steps.py b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/provider_steps.py index 7a5446943..3aae80dec 100644 --- a/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/provider_steps.py +++ b/tools/openfeature-tck/src/openfeature/contrib/tools/tck/steps/provider_steps.py @@ -3,6 +3,7 @@ from __future__ import annotations import concurrent.futures +import contextlib import threading import time from collections.abc import Callable @@ -11,6 +12,7 @@ from openfeature import api from openfeature.evaluation_context import EvaluationContext +from openfeature.provider import FeatureProvider from ..control import serve_configuration from ..state import LifecycleRecord, TckState @@ -40,6 +42,7 @@ def a_stable_provider(tck_state: TckState) -> None: if provider is None: msg = "TckConfig.new_provider returned None" raise AssertionError(msg) + _observe_metadata_name(tck_state, provider) try: _call_within( @@ -89,6 +92,7 @@ def an_unavailable_provider(tck_state: TckState) -> None: if provider is None: msg = "TckConfig.new_unavailable_provider returned None" raise AssertionError(msg) + _observe_metadata_name(tck_state, provider) # The waiting variant for the same reason as the stable provider. The SDK's # registry already converts a raising initialize into PROVIDER_ERROR, so the @@ -218,6 +222,21 @@ def the_provider_metadata_name_should_not_be_empty(tck_state: TckState) -> None: raise AssertionError(msg) +def _observe_metadata_name(tck_state: TckState, provider: FeatureProvider) -> None: + """Note what the provider calls itself, for the conformance report. + + Before registration rather than after, so that a provider which fails to + initialise -- the ``@unavailable`` case, and any genuine failure -- is still + identified in the report by its own name. Metadata is a pure accessor by + contract, but a provider that raises from it must not take the scenario down + with it: the name is for a report, and no scenario asserts on it. + """ + with contextlib.suppress(Exception): + name = provider.get_metadata().name + if name: + tck_state.provider_name = name + + def _record_lifecycle_call( tck_state: TckState, operation: str, call: Callable[[], object] ) -> None: diff --git a/tools/openfeature-tck/tests/conftest.py b/tools/openfeature-tck/tests/conftest.py index a5e6726f5..d5c09ea5a 100644 --- a/tools/openfeature-tck/tests/conftest.py +++ b/tools/openfeature-tck/tests/conftest.py @@ -3,35 +3,76 @@ A conformance suite that quietly goes green on scenarios it did not run is worse than no suite at all -- and the same is true of one that quietly goes green on a scenario it *did* run and fail. So the one scenario the Python SDK cannot -currently satisfy is marked ``xfail(strict=True)`` here, which: +currently satisfy is recorded twice, in two forms that answer different +questions. -* keeps it visible in the report, as XFAIL with the reason attached; -* fails the suite if it ever *passes*, so the marker is removed the moment the - SDK is fixed rather than lingering as a lie. +``xfail(strict=True)`` keeps the *run* honest: the scenario is expected to fail, +and the suite fails if it ever passes, so the marker is removed the moment the +SDK is fixed rather than lingering as a lie. -This lives in the TCK's own self-test rather than in the shared package. It is a -fact about the SDK under test, not part of the conformance definition, and -Appendix F deliberately leaves a general "known deviations" concept as an open -question (spec#417, Q4). If that concept lands, this moves into it. +:class:`KnownDeviation` keeps the *report* honest. The results payload reports +the scenario as failed regardless of the marker -- an expected failure is still a +failure, and softening it there would hide exactly what the marker exists to keep +visible -- and the envelope carries the acknowledgement beside it, with the issue +it is tracked under. That is what lets a consumer tell a known and tracked gap +from a surprise without the result itself being weakened. + +The two are declared together here so they cannot drift: the reason on the marker +and the summary in the report are the same sentence. """ from __future__ import annotations import pytest +from openfeature.contrib.tools.tck import KnownDeviation + # The Scenario Outline row that asks for boolean-flag as an Integer. _BOOL_AS_INT = ( "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" ) +_ISSUE = "https://github.com/open-feature/python-sdk/issues/619" + _REASON = ( "python-sdk: a boolean satisfies an Integer request. The client type-checks with " "isinstance(value, int) and bool is a subclass of int in Python, so boolean-flag " "requested as an Integer returns True with reason STATIC and no error code, where " "the specification requires the code default and TYPE_MISMATCH. " - "See https://github.com/open-feature/python-sdk/issues/619" + f"See {_ISSUE}" ) +KNOWN_DEVIATIONS = (KnownDeviation(issue=_ISSUE, summary=_REASON),) +"""What the report acknowledges. + +No ``capability``: the scenario carries no capability tag, because returning the +code default on a type mismatch is mandatory. ``@numeric-coercion`` is a +neighbouring question -- whether 0.5 satisfies an integer request -- and this +provider answers *that* one the way the tag asks, by refusing it, so attributing +the deviation there would be wrong twice over. + +Which is not the same as satisfying the capability, and the distinction matters +now that Appendix F has corrected its note on it. The tag is withheld here +because this provider does not coerce at all: it passes the lossy row by +rejecting every float, which is the shortcut the two lossless rows exist to +catch, and it fails both of those. That is the withholding the appendix still +calls right -- a provider that cannot attempt the behaviour -- rather than the +one it now rules out, where a provider attempts it and gets a direction wrong. +The boolean-as-Integer gap is a third thing again: mandatory, ungated, and the +reason this entry names no capability. +""" + + +@pytest.fixture(scope="session") +def tck_known_deviations() -> tuple[KnownDeviation, ...]: + """The deviations a suite in this package declares. + + A fixture rather than an import so that the marker below and the report's + acknowledgement are written down once, in one place, and a suite picks it up + the same way it picks up everything else it is given. + """ + return KNOWN_DEVIATIONS + def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: for item in items: diff --git a/tools/openfeature-tck/tests/test_canonical_set.py b/tools/openfeature-tck/tests/test_canonical_set.py new file mode 100644 index 000000000..40f7027a0 --- /dev/null +++ b/tools/openfeature-tck/tests/test_canonical_set.py @@ -0,0 +1,374 @@ +"""That a run which executed less than the canonical set says so, and publishes nothing. + +The capability gate rules out the loud way a conformance suite can go green on +scenarios it did not run: an undeclared capability is reported as skipped, with +its reason. Nothing ruled out the quiet way, where the scenarios were never +collected at all. ``-k``, ``-m``, ``--deselect``, a test module that stopped +calling ``scenarios()`` on the canonical path -- each runs less of the suite, and +none of them is an error to pytest. + +Go measured the consequence: ``-run`` on a single scenario passed green and +emitted a well-formed report covering one scenario out of the whole canonical +set. +Nothing in that document said so, and nothing reading it could have known. + +So the properties here are about what a report is allowed to be written from. +Everything that has to be checked end to end is, because the question is about a +whole pytest session rather than about what a function returns. +""" + +from __future__ import annotations + +import dataclasses +import json +import os +import re +import subprocess +import sys +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.tck import ( + EXTENSIONS_DIRECTORY, + PARTIAL_ENV, + REPORT_DIR_ENV, +) +from openfeature.contrib.tools.tck.canonical import ( + canonical_scenarios, + describe, + missing_canonical, + partial_run_allowed, +) +from openfeature.contrib.tools.tck.extensions import ( + CANONICAL_DIRECTORY, + is_canonical_uri, +) +from openfeature.contrib.tools.tck.messages import ( + ScenarioIdentity, + ScenarioRun, +) + +SUITE_NAME = "guarded" + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so a partial run can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + InProcessControl, + TckConfig, + feature_paths, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="guarded", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS, Capability.OBJECT}, + ) + + +scenarios(*feature_paths()) +''' + +_CONFTEST_MODULE = """\ +import pytest + +DEVIATION = "[boolean-flag-Integer-1]" + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name.endswith(DEVIATION): + item.add_marker(pytest.mark.xfail(reason="python-sdk#619")) +""" + +# Three scenarios of an adopter's own, so that a run which drops one canonical +# scenario still executes more scenarios than the canonical set contains. The +# count is what makes "an extension cannot close a gap" checkable rather than +# asserted. +_VENDOR_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + + Scenario: A vendor rule resolves again + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + + Scenario: And once more + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" +""" + +_EXTENSION_SCENARIOS = 3 +"""How many scenarios ``_VENDOR_FEATURE`` contributes.""" + +_OUTCOMES = re.compile(r"(\d+) (passed|failed|skipped|xfailed|xpassed)\b") +"""The per-outcome counts of a ``-q`` summary line, which together are what ran.""" + +# The canonical scenario the filtered runs below leave out. Named rather than +# counted, so a change to the canonical assets cannot leave these passing while +# they select nothing. +EXCLUDED_SELECTOR = "unknown_flag_key" +EXCLUDED_SCENARIO = "An unknown flag key returns the code default" + +# How many canonical scenarios there are, read from the packaged assets through +# the same parser the guard itself uses. Appendix F makes that a rule: an +# expectation about the canonical set that restates a number goes stale the next +# time the specification adds a scenario, and a conformance suite whose own tests +# have to be edited to follow the assets is a suite that will be edited to agree +# with them. +CANONICAL_COUNT = len(canonical_scenarios()) + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of the generated adoption.""" + + reports: Path + result: subprocess.CompletedProcess[str] + + @property + def stdout(self) -> str: + return self.result.stdout + + @property + def envelopes(self) -> list[Path]: + return sorted(self.reports.glob("*.json")) + + +def _run( + tmp_path: Path, + *arguments: str, + partial: bool = False, + extension: bool = False, +) -> Run: + directory = tmp_path / "adoption" + directory.mkdir(parents=True, exist_ok=True) + (directory / "test_guarded.py").write_text(_SUITE_MODULE, encoding="utf-8") + (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") + if extension: + extensions = directory / EXTENSIONS_DIRECTORY + extensions.mkdir(parents=True, exist_ok=True) + (extensions / "vendor.feature").write_text(_VENDOR_FEATURE, encoding="utf-8") + + reports = tmp_path / "reports" + environment = dict(os.environ) + environment[REPORT_DIR_ENV] = str(reports) + if partial: + environment[PARTIAL_ENV] = "1" + else: + environment.pop(PARTIAL_ENV, None) + + result = subprocess.run( # noqa: S603 + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + str(directory), + *arguments, + ], + capture_output=True, + text=True, + env=environment, + check=False, + ) + return Run(reports=reports, result=result) + + +def _identity(uri: str, name: str, path: Path) -> ScenarioIdentity: + return ScenarioIdentity(uri=uri, path=path, name=name, tags=()) + + +# -- a run that executed the whole set --------------------------------------- + + +@pytest.fixture(scope="module") +def complete(tmp_path_factory: pytest.TempPathFactory) -> Run: + """The unfiltered run, which is what everything else is measured against.""" + return _run(tmp_path_factory.mktemp("complete"), extension=True) + + +def test_the_whole_canonical_set_still_writes_a_report(complete: Run) -> None: + """Including the scenarios the capability gate skipped. + + A gated skip *ran*: it was asked, and the report accounts for it with a + reason. Treating it as missing would make the guard contradict the one rule + the suite is built around. + """ + assert complete.result.returncode == 0, complete.stdout + assert [path.name for path in complete.envelopes] == [f"{SUITE_NAME}.json"] + assert "canonical scenarios did not run" not in complete.stdout + # This adoption declares neither @stale nor @lifecycle, so some canonical + # scenarios were skipped -- which is the case being asserted about. + assert " skipped" in complete.stdout + + +# -- and one that did not ---------------------------------------------------- + + +def test_a_filtered_run_fails_and_writes_no_report(tmp_path: Path) -> None: + """The Go hazard, at the point it would have produced the document. + + A well-formed report covering one scenario of the canonical set is worse than + no report, because nothing in it says which of the rest were never asked. + """ + run = _run(tmp_path, "-k", EXCLUDED_SELECTOR) + + assert run.result.returncode != 0, run.stdout + assert ( + f"{CANONICAL_COUNT - 1} of {CANONICAL_COUNT} canonical scenarios did not run" + in run.stdout + ) + assert not run.envelopes, "a partial run must publish nothing" + assert not list(run.reports.glob("*.ndjson")) + # The message names scenarios rather than only counting them, and says how + # to filter deliberately. + assert EXCLUDED_SCENARIO not in run.stdout, "that one is the scenario that ran" + assert PARTIAL_ENV in run.stdout + + +def test_acknowledging_a_partial_run_does_not_make_it_publishable( + tmp_path: Path, +) -> None: + """``TCK_PARTIAL`` buys a green run, never a document. + + Someone working on one scenario should not have to fight the guard; nobody + should be able to turn a partial run into a conformance claim. Those are + different requests, and only the first is granted. Java's TCK spells the + same escape hatch the same way. + """ + run = _run(tmp_path, "-k", EXCLUDED_SELECTOR, partial=True) + + assert run.result.returncode == 0, run.stdout + assert "canonical scenarios did not run" in run.stdout + assert f"{PARTIAL_ENV} is set" in run.stdout + assert not run.envelopes, "acknowledged or not, it is not a conformance run" + + +def test_an_extension_cannot_close_a_gap(tmp_path: Path) -> None: + """Three extension scenarios do not make up for one canonical one. + + The adoption below runs more scenarios than the canonical set contains and is + still one short: an adopter's scenarios are theirs, and counting them towards + the specification's set would let any gap be filled by adding a feature file. + """ + run = _run(tmp_path, "-k", f"not {EXCLUDED_SELECTOR}", extension=True) + + assert run.result.returncode != 0, run.stdout + assert f"1 of {CANONICAL_COUNT} canonical scenarios did not run" in run.stdout + assert f"{CANONICAL_DIRECTORY}/errors.feature: {EXCLUDED_SCENARIO}" in run.stdout + assert not run.envelopes + + +# -- what the canonical set is ----------------------------------------------- + + +def test_the_canonical_set_is_read_from_the_packaged_assets(complete: Run) -> None: + """One entry per Scenario Outline row, which is what a runner generates. + + The number is not written down here. It is read off the unfiltered run -- + pytest-bdd's own collection of the same packaged feature files, less the + three scenarios the adopter's extension contributes -- because an + expectation that restated a count would pin this file rather than the + assets, and go stale the next time the specification adds a scenario. + """ + scenarios = canonical_scenarios() + + summary = [line for line in complete.stdout.splitlines() if line.strip()][-1] + collected = sum(int(count) for count, _ in _OUTCOMES.findall(summary)) + + assert len(scenarios) == collected - _EXTENSION_SCENARIOS, summary + assert all(is_canonical_uri(uri) for uri, _, _ in scenarios) + # An outline contributes rows, not a single templated entry. + assert any(row for _, _, row in scenarios) + + +def test_a_run_of_nothing_is_missing_everything() -> None: + assert set(missing_canonical([])) == canonical_scenarios() + + +def test_an_extension_run_is_neither_counted_nor_blamed(tmp_path: Path) -> None: + """Matched by path rather than by uri. + + The uri is derived, and a check that rested on the same derivation it exists + to corroborate would be checking its own arithmetic. + """ + vendor = ScenarioRun( + identity=_identity( + "extensions/vendor.feature", + "A vendor rule resolves", + tmp_path / EXTENSIONS_DIRECTORY / "vendor.feature", + ) + ) + # Even one that claims a canonical uri outright -- which the report refuses + # separately -- cannot reduce the missing set. + impostor = ScenarioRun( + identity=_identity( + f"{CANONICAL_DIRECTORY}/errors.feature", + EXCLUDED_SCENARIO, + tmp_path / CANONICAL_DIRECTORY / "errors.feature", + ) + ) + assert set(missing_canonical([vendor, impostor])) == canonical_scenarios() + + +def test_a_missing_scenario_is_described_by_its_row() -> None: + assert describe((f"{CANONICAL_DIRECTORY}/x.feature", "A scenario", ())) == ( + f"{CANONICAL_DIRECTORY}/x.feature: A scenario" + ) + assert describe( + (f"{CANONICAL_DIRECTORY}/x.feature", "An outline", (("key", "a"),)) + ) == (f"{CANONICAL_DIRECTORY}/x.feature: An outline [key=a]") + + +@pytest.mark.parametrize( + ("value", "allowed"), + [("1", True), ("true", True), ("TRUE", True), ("yes", True), ("on", True)], +) +def test_the_acknowledgement_is_read_generously(value: str, allowed: bool) -> None: + assert partial_run_allowed({PARTIAL_ENV: value}) is allowed + + +@pytest.mark.parametrize("value", ["", "0", "false", "no", " ", "maybe"]) +def test_anything_else_is_not_an_acknowledgement(value: str) -> None: + """Including a typo: the default has to be the safe one.""" + assert partial_run_allowed({PARTIAL_ENV: value}) is False + assert partial_run_allowed({}) is False + + +def test_the_envelope_of_a_complete_run_is_unchanged(complete: Run) -> None: + """The guard withholds a report; it does not add anything to one.""" + envelope: dict[str, typing.Any] = json.loads( + complete.envelopes[0].read_text(encoding="utf-8") + ) + assert set(envelope) == { + "schemaVersion", + "provider", + "sdk", + "tck", + "backend", + "declaration", + "results", + } diff --git a/tools/openfeature-tck/tests/test_controllable_conformance.py b/tools/openfeature-tck/tests/test_controllable_conformance.py index c5f0ebb93..fb5da14ab 100644 --- a/tools/openfeature-tck/tests/test_controllable_conformance.py +++ b/tools/openfeature-tck/tests/test_controllable_conformance.py @@ -20,13 +20,14 @@ from openfeature.contrib.tools.tck import ( Capability, InProcessControl, + KnownDeviation, TckConfig, feature_paths, ) @pytest.fixture(scope="session") -def tck_config() -> TckConfig: +def tck_config(tck_known_deviations: tuple[KnownDeviation, ...]) -> TckConfig: """Declare the provider under test and what it can do. ``STALE`` and ``UNAVAILABLE_INIT`` stay undeclared: there is still no @@ -83,6 +84,9 @@ class inherits it rather than choosing it: ``ControllableInMemoryProvider`` Capability.LARGE_INTEGERS, Capability.STANDARD_REASONS, }, + # The same SDK bug, against the same issue: it is a defect in the client + # rather than in either provider, so both suites acknowledge it. + known_deviations=tck_known_deviations, ) diff --git a/tools/openfeature-tck/tests/test_extension_reporting.py b/tools/openfeature-tck/tests/test_extension_reporting.py new file mode 100644 index 000000000..1c241b956 --- /dev/null +++ b/tools/openfeature-tck/tests/test_extension_reporting.py @@ -0,0 +1,652 @@ +"""What a conformance report says about an adopter's own scenarios. + +That an extension *runs* inside the canonical suite, and cannot take a canonical +feature file's identity, is checked in ``test_extensions`` without a report in +sight. What is left is what the report does with it, and it is the half a +consumer actually reads: + +* an extension scenario appears in the same suite's report as the canonical ones, + which is not presentational -- a report is written per ``TckConfig``, so + appearing in one is appearing in the same provider registration; +* it appears under the ``extensions/`` uri prefix and never the reserved + ``gherkin/`` one, which is the only thing telling a consumer whose question a + scenario was; +* an adoption with no extension writes the report it wrote before, field for + field; +* and the two cases the uri derivation cannot rule out produce **no report at + all** rather than a plausible one. + +The last is where reporting adds a rule rather than a field. A document that +presents an adopter's feature file as the specification's -- or one file's +scenarios against another file's source -- is worse than no document, because it +is the one thing a consumer cannot check from the outside. Java measured the +loud form of it: a same-named feature file in a second classpath root silently +*replaced* the canonical one, and the run went green having asked the adopter's +questions. + +All of it is read back from the emitted documents the way a consumer reads them, +against real pytest sessions in subprocesses, because every property is about +what a whole session wrote. +""" + +from __future__ import annotations + +import collections +import dataclasses +import json +import os +import subprocess +import sys +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.tck import ( + EXTENSIONS_DIRECTORY, + REPORT_DIR_ENV, + canonical_root, +) +from openfeature.contrib.tools.tck.extensions import ( + CANONICAL_DIRECTORY, + is_canonical_uri, +) + +CANONICAL_FEATURE = "errors.feature" +"""The canonical file the shadowing fixture copies, chosen because it is the one +whose scenarios an extension could most plausibly want to restate.""" + + +def _canonical_root() -> Path: + """The packaged canonical directory, or a failure that says how to get one. + + ``canonical_root()`` answers ``None`` when the assets are not on a + filesystem, which is the honest answer for a zipimport and a missing build + step everywhere else. + """ + root = canonical_root() + assert root is not None, ( + "the packaged canonical features must be on a filesystem for this file " + "to have anything to say; run `poe sync-spec-assets` first" + ) + return root + + +CANONICAL_ROOT = _canonical_root() + + +BASELINE_DIRECTORY = "baseline" +"""Where the adoption that has no extensions of its own lives. + +A subdirectory rather than a second module beside the first, because all three +adoptions now write the same line and what distinguishes them is where they are +written. ``feature_paths()`` looks for an ``extensions`` directory beside the +calling module, so a module one level down is an adopter with none. + +There used to be a second public call, ``features_path()``, which returned the +canonical set alone and was what ``before`` used. It is gone: the two differed +by one character and the shorter one silently dropped the extensions directory. +""" + +VENDOR_URI = "extensions/vendor.feature" +VENDOR_SCENARIO = "A vendor rule resolves through the suite's own provider" + + +# -- the generated adoption -------------------------------------------------- + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so extensions can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + InProcessControl, + TckConfig, + feature_paths, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="{name}", + control=control, + new_provider=control.new_provider, + capabilities={{Capability.EVENTS, Capability.OBJECT}}, + ) + + +scenarios(*feature_paths()) +''' + +# The step the adopter writes, in the adopter's own conftest.py and nowhere else. +# It asks the TCK's own per-scenario state what provider this scenario is running +# against, which is what makes "the same backend lifecycle" checkable rather than +# asserted: a second harness would have a second provider, or none. +_CONFTEST_MODULE = """\ +import pytest +from pytest_bdd import then + +from openfeature.contrib.tools.tck import TckState + +DEVIATION = "[boolean-flag-Integer-1]" + + +@then("the vendor rule ran against the provider the suite registered") +def vendor_rule_ran(tck_state: TckState) -> None: + assert tck_state.client is not None, "no provider was registered" + assert tck_state.provider_name == "In-Memory Provider", tck_state.provider_name + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name.endswith(DEVIATION): + item.add_marker(pytest.mark.xfail(reason="python-sdk#619")) +""" + +# Deliberately reuses the canonical step vocabulary and adds exactly one step of +# its own, which is the shape an adopter's feature file actually takes. +_VENDOR_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves through the suite's own provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" + And the vendor rule ran against the provider the suite registered +""" + +# The same Feature name and the same Scenario name as the file above, with a +# different step list -- so that a run reporting one against the other's source +# would be wrong in a way nothing downstream could notice. +_NESTED_FEATURE = """\ +Feature: Vendor rules + + Background: + Given a stable provider + + Scenario: A vendor rule resolves through the suite's own provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" +""" + +# An adopter's own directory named `gherkin`, which is the one way a +# non-canonical file can still reach the reserved prefix. Its scenario is +# deliberately trivial: the point is the file name, not what it asks. +_RESERVED_FEATURE = """\ +Feature: A feature file in a directory named gherkin + + Scenario: A flag resolves + Given a stable provider + Given a String-flag with key "string-flag" and a default value "bye" + When the flag was evaluated with details + Then the resolved details value should be "hi" +""" + +_RESERVED_SUITE = '''\ +"""An adoption that hands scenarios() a directory of its own named gherkin.""" + +import pathlib + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + InProcessControl, + TckConfig, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="reserved", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS}, + ) + + +scenarios(str(pathlib.Path(__file__).parent / "gherkin")) +''' + + +# -- reading a run back ------------------------------------------------------ + + +@dataclasses.dataclass(frozen=True) +class Case: + """One scenario as the stream reports it.""" + + uri: str + name: str + row: tuple[tuple[str, str], ...] + status: str + + @property + def identity(self) -> tuple[str, str, tuple[tuple[str, str], ...]]: + return (self.uri, self.name, self.row) + + +_SEVERITY = [ + "UNKNOWN", + "PASSED", + "SKIPPED", + "PENDING", + "UNDEFINED", + "AMBIGUOUS", + "FAILED", +] +"""Cucumber's own ordering: a test case is as bad as its worst step.""" + + +@dataclasses.dataclass(frozen=True) +class Report: + """One suite's pair of documents, read the way a consumer reads them.""" + + envelope: dict[str, typing.Any] + sources: dict[str, str] + cases: list[Case] + + @property + def canonical(self) -> list[Case]: + return [case for case in self.cases if is_canonical_uri(case.uri)] + + @property + def extensions(self) -> list[Case]: + return [case for case in self.cases if not is_canonical_uri(case.uri)] + + def named(self, name: str) -> list[Case]: + return [case for case in self.cases if case.name == name] + + @property + def identities(self) -> set[tuple[str, str, tuple[tuple[str, str], ...]]]: + return {case.identity for case in self.cases} + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of a generated adoption.""" + + directory: Path + reports: Path + result: subprocess.CompletedProcess[str] + + def report(self, name: str) -> Report: + path = self.reports / f"{name}.json" + assert path.exists(), ( + f"no report at {path}; pytest exited {self.result.returncode}\n" + f"{self.result.stdout}\n{self.result.stderr}" + ) + envelope = json.loads(path.read_text(encoding="utf-8")) + return _read(envelope, self.reports / envelope["results"]["location"]) + + +def _read(envelope: dict[str, typing.Any], stream_path: Path) -> Report: + """Assemble a stream into scenarios by following the protocol's own links.""" + sources: dict[str, str] = {} + names: dict[str, str] = {} + rows: dict[str, tuple[tuple[str, str], ...]] = {} + pickles: dict[str, dict[str, typing.Any]] = {} + test_cases: dict[str, dict[str, typing.Any]] = {} + started: dict[str, str] = {} + results: dict[str, list[str]] = collections.defaultdict(list) + + for line in stream_path.read_text(encoding="utf-8").splitlines(): + message = json.loads(line) + kind = next(iter(message)) + body = message[kind] + if kind == "source": + sources[body["uri"]] = body["data"] + elif kind == "gherkinDocument": + _index_document(body, names, rows) + elif kind == "pickle": + pickles[body["id"]] = body + elif kind == "testCase": + test_cases[body["id"]] = body + elif kind == "testCaseStarted": + started[body["id"]] = body["testCaseId"] + elif kind == "testStepFinished": + results[body["testCaseStartedId"]].append(body["testStepResult"]["status"]) + + cases = [] + for started_id, case_id in started.items(): + pickle = pickles[test_cases[case_id]["pickleId"]] + ast = pickle["astNodeIds"] + cases.append( + Case( + uri=pickle["uri"], + name=names[ast[0]], + row=rows.get(ast[1], ()) if len(ast) > 1 else (), + status=max(results[started_id], key=_SEVERITY.index), + ) + ) + return Report(envelope=envelope, sources=sources, cases=cases) + + +def _index_document( + document: dict[str, typing.Any], + names: dict[str, str], + rows: dict[str, tuple[tuple[str, str], ...]], +) -> None: + def visit(children: typing.Iterable[dict[str, typing.Any]]) -> None: + for child in children: + if "rule" in child: + visit(child["rule"].get("children", ())) + continue + scenario = child.get("scenario") + if scenario is None: + continue + names[scenario["id"]] = scenario["name"] + for examples in scenario.get("examples", ()): + header = examples.get("tableHeader") + if header is None: + continue + headers = [cell["value"] for cell in header["cells"]] + for row in examples.get("tableBody", ()): + cells = [cell["value"] for cell in row["cells"]] + rows[row["id"]] = tuple(zip(headers, cells, strict=True)) + + feature = document.get("feature") + if feature is not None: + visit(feature.get("children", ())) + + +def _pytest(directory: Path, reports: Path) -> subprocess.CompletedProcess[str]: + environment = dict(os.environ) + environment[REPORT_DIR_ENV] = str(reports) + return subprocess.run( # noqa: S603 + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + str(directory), + ], + capture_output=True, + text=True, + env=environment, + check=False, + ) + + +def _suite(name: str) -> str: + return _SUITE_MODULE.format(name=name) + + +def _run( + tmp_path_factory: pytest.TempPathFactory, + modules: dict[str, str], + features: dict[str, str] | None = None, +) -> Run: + """Write an adoption, run it, and hand back what it wrote. + + Both mappings are keyed by a path relative to the adoption directory, so a + fixture can put a module or a feature file wherever the property under test + needs it -- including inside a directory named ``gherkin``, which is the + case that has to be refused. + """ + directory = tmp_path_factory.mktemp("adoption") + for relative, body in {**modules, **(features or {})}.items(): + path = directory / Path(relative) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + + reports = tmp_path_factory.mktemp("reports") + return Run(directory=directory, reports=reports, result=_pytest(directory, reports)) + + +@pytest.fixture(scope="module") +def adoption(tmp_path_factory: pytest.TempPathFactory) -> Run: + """One session running three adoptions of the same provider. + + All three write the same line, ``scenarios(*feature_paths())``, and differ + by where they are written. ``before`` sits one directory down with no + ``extensions`` beside it, which is an adopter who has none; ``after`` has an + ordinary extension beside it; ``shadowed`` is in a directory of its own with + an extension that is a verbatim copy of a canonical feature file placed + under a directory named ``gherkin`` -- so that both routes to a canonical + identity, the file's own name and its parent's, are taken at once. + + One session rather than three, because a subprocess pytest run is by far the + most expensive thing in this file and the three suites are independent: each + resolves its own ``TckConfig`` and writes its own pair of documents. + """ + canonical = (CANONICAL_ROOT / CANONICAL_FEATURE).read_text(encoding="utf-8") + return _run( + tmp_path_factory, + { + f"{BASELINE_DIRECTORY}/test_before.py": _suite("before"), + "test_after.py": _suite("after"), + "conftest.py": _CONFTEST_MODULE, + "shadow/test_shadow.py": _suite("shadowed"), + }, + { + f"{EXTENSIONS_DIRECTORY}/vendor.feature": _VENDOR_FEATURE, + f"shadow/{EXTENSIONS_DIRECTORY}/{CANONICAL_DIRECTORY}/" + f"{CANONICAL_FEATURE}": canonical, + }, + ) + + +# -- an extension is reported by the suite it ran in ------------------------- + + +def test_an_extension_scenario_is_in_the_same_suites_report(adoption: Run) -> None: + """One suite, one report, both sets of scenarios in it. + + The report is written per ``TckConfig``, so an extension scenario appearing + in the same report as the canonical ones is not a presentational detail: it + is the same suite, which is the same provider registration and the same + backend control. + """ + assert adoption.result.returncode == 0, adoption.result.stdout + report = adoption.report("after") + + vendor = report.named(VENDOR_SCENARIO) + assert len(vendor) == 1, report.extensions + assert vendor[0].status == "PASSED" + assert vendor[0].uri == VENDOR_URI + + assert report.canonical, "the canonical scenarios must have run too" + assert {case.status for case in report.canonical} <= {"PASSED", "SKIPPED", "FAILED"} + + +def test_the_extension_feature_is_reported_under_its_own_prefix( + adoption: Run, +) -> None: + """Which is what keeps an adopter's claim apart from the specification's. + + ``extensions/`` is the prefix the Go and JavaScript suites mount extensions + under too, so a consumer holding reports from several languages applies one + rule. + """ + report = adoption.report("after") + assert [case.uri for case in report.extensions] == [VENDOR_URI] + assert report.sources[VENDOR_URI] == _VENDOR_FEATURE + assert all( + case.uri.startswith(f"{CANONICAL_DIRECTORY}/") for case in report.canonical + ) + + +def test_the_envelope_is_unaffected_by_an_extension(adoption: Run) -> None: + """The report schema has no slot for extensions and needs none. + + Everything an extension adds is a scenario in the results payload, where a + uri already distinguishes it. An envelope field would be a second place for + the same fact to live. + """ + envelope = adoption.report("after").envelope + assert envelope["provider"]["configuration"] == "after" + assert set(envelope) == { + "schemaVersion", + "provider", + "sdk", + "tck", + "backend", + "declaration", + "results", + } + + +# -- and changes nothing for an adopter who has none ------------------------- + + +def test_the_canonical_scenarios_are_the_ones_that_always_ran( + adoption: Run, +) -> None: + """An extension adds; it does not alter. + + ``before`` has no ``extensions`` directory beside it and sees no extension. + Every scenario it ran, ``after`` ran too -- same rows, same outcomes, same + sources -- and the only difference between the two is what the extension + added. An adopter who has no extensions is the same comparison with the + right-hand side empty, which is what ``feature_paths()`` returning the + canonical path alone makes true by construction rather than by luck. + """ + assert adoption.result.returncode == 0, adoption.result.stdout + before = adoption.report("before") + after = adoption.report("after") + + assert not before.extensions, ( + "a module with no extensions directory beside it must see no extension" + ) + assert {case.identity: case.status for case in after.canonical} == { + case.identity: case.status for case in before.cases + } + assert after.identities - before.identities == {(VENDOR_URI, VENDOR_SCENARIO, ())} + assert { + uri: source for uri, source in after.sources.items() if is_canonical_uri(uri) + } == before.sources + + +def test_an_extension_does_not_change_the_envelope_a_suite_writes( + adoption: Run, +) -> None: + """Everything but the two fields that necessarily differ. + + ``results`` names a file and digests its bytes, and the bytes carry + timestamps; ``configuration`` is the suite name, which is what tells the two + generated suites apart in the first place. + """ + before = dict(adoption.report("before").envelope) + after = dict(adoption.report("after").envelope) + for envelope in (before, after): + del envelope["results"] + envelope["provider"] = { + key: value + for key, value in envelope["provider"].items() + if key != "configuration" + } + assert after == before + + +# -- and cannot stand in for a canonical scenario ---------------------------- + + +def test_an_extension_cannot_replace_a_canonical_feature_file( + adoption: Run, +) -> None: + """The hazard Java measured, checked at the point it would have bitten. + + A verbatim copy of ``errors.feature`` under ``extensions/gherkin/`` + reaches pytest-bdd as ``gherkin/errors.feature`` -- the canonical uri. The + uri the payload reports is derived from where the file is instead, so the + canonical source is still the packaged one, the copy is reported as an + extension, and both ran. + """ + report = adoption.report("shadowed") + canonical_uri = f"{CANONICAL_DIRECTORY}/{CANONICAL_FEATURE}" + extension_uri = f"extensions/{CANONICAL_DIRECTORY}/{CANONICAL_FEATURE}" + + packaged = (CANONICAL_ROOT / CANONICAL_FEATURE).read_text(encoding="utf-8") + assert report.sources[canonical_uri] == packaged + assert report.sources[extension_uri] == packaged + + # Both files ran: the copy neither replaced the canonical one nor was + # silently dropped for colliding with it. + canonical = { + case.identity for case in report.canonical if case.uri == canonical_uri + } + copied = {case.identity for case in report.cases if case.uri == extension_uri} + assert canonical, "the canonical feature file did not run" + assert len(copied) == len(canonical) + assert not any(is_canonical_uri(case.uri) for case in report.extensions) + + +def test_a_shadowing_extension_does_not_disturb_the_canonical_run( + adoption: Run, +) -> None: + """The canonical scenarios are the same ones, with the same outcomes. + + Compared against a run that has no extension at all rather than against a + number written down here, so a change to the canonical assets cannot leave + this passing while the copy quietly displaces something. + """ + shadow = adoption.report("shadowed") + baseline = adoption.report("before") + assert {case.identity: case.status for case in shadow.canonical} == { + case.identity: case.status for case in baseline.cases + } + + +def test_a_directory_named_gherkin_is_refused( + tmp_path_factory: pytest.TempPathFactory, +) -> None: + """The one collision the naming convention cannot rule out on its own. + + An adopter can still hand ``scenarios()`` a directory of their own named + ``gherkin``, and its files are then named exactly as canonical ones would + be. No report is written for that suite: a document presenting an adopter's + feature file as the specification's is worse than no document, because it is + the one thing a consumer cannot check. + """ + run = _run( + tmp_path_factory, + {"test_reserved.py": _RESERVED_SUITE}, + {f"{CANONICAL_DIRECTORY}/local.feature": _RESERVED_FEATURE}, + ) + assert run.result.returncode != 0, run.result.stdout + assert ( + f"{CANONICAL_DIRECTORY}/local.feature is not a canonical feature file" + in run.result.stdout + ) + assert EXTENSIONS_DIRECTORY in run.result.stdout, "the message must say the fix" + assert not list(run.reports.glob("*.json")), "no report may be written" + + +def test_two_extension_files_cannot_share_one_uri( + tmp_path_factory: pytest.TempPathFactory, +) -> None: + """Deriving the uri from the location narrows the collision; it does not end it. + + An ``extensions`` directory nested inside another one reaches the same + uri as its namesake at the root, and so would two test modules sharing one + ``tck_config``. A Messages stream carries one source per uri, so the second + file's scenarios would be reported against the first file's pickles wherever + the names matched -- which here they do, deliberately. Refused rather than + resolved. + """ + nested = f"{EXTENSIONS_DIRECTORY}/nested/{EXTENSIONS_DIRECTORY}/vendor.feature" + run = _run( + tmp_path_factory, + {"test_collide.py": _suite("collide"), "conftest.py": _CONFTEST_MODULE}, + { + f"{EXTENSIONS_DIRECTORY}/vendor.feature": _VENDOR_FEATURE, + nested: _NESTED_FEATURE, + }, + ) + assert run.result.returncode != 0, run.result.stdout + assert f"{VENDOR_URI} is the uri of 2 different feature files" in run.result.stdout + assert not list(run.reports.glob("*.json")), "no report may be written" diff --git a/tools/openfeature-tck/tests/test_http_control.py b/tools/openfeature-tck/tests/test_http_control.py index 3969ccce2..507947d84 100644 --- a/tools/openfeature-tck/tests/test_http_control.py +++ b/tools/openfeature-tck/tests/test_http_control.py @@ -345,7 +345,7 @@ def test_the_control_reports_which_api_it_drives_the_backend_through() -> None: Every operation on this class is an HTTP request to the normative control API, so this is the one control that can answer the question flatly, and leaving it to default to ``"custom"`` would understate the run. A report - whose ``backend.controlApi`` says otherwise for a provider with a real + whose ``backend.backendType`` says otherwise for a provider with a real backend is claiming something it should not. """ with _StubControlApi() as stub: diff --git a/tools/openfeature-tck/tests/test_in_memory_conformance.py b/tools/openfeature-tck/tests/test_in_memory_conformance.py index 77d1ffaec..b972e55e8 100644 --- a/tools/openfeature-tck/tests/test_in_memory_conformance.py +++ b/tools/openfeature-tck/tests/test_in_memory_conformance.py @@ -23,6 +23,7 @@ from openfeature.contrib.tools.tck import ( BackendType, Capability, + KnownDeviation, TckConfig, canonical_flag_set, feature_paths, @@ -76,7 +77,7 @@ def _new_provider() -> FeatureProvider: @pytest.fixture(scope="session") -def tck_config() -> TckConfig: +def tck_config(tck_known_deviations: tuple[KnownDeviation, ...]) -> TckConfig: """Declare the provider under test and what it can do. Each omission is a fact about the provider rather than a convenience: @@ -178,6 +179,11 @@ def tck_config() -> TckConfig: neither of which is declared here, so they skip with that reason -- which is the capability working as intended rather than a gap: a reason cannot be observed without the behaviour that produces it. + + ``known_deviations`` is the one thing here that is not a claim about what + this provider supports: it is the acknowledgement of a scenario the SDK + fails, which the results payload still reports as a failure. See + ``conftest.py``. """ return TckConfig( name="in-memory", @@ -192,6 +198,7 @@ def tck_config() -> TckConfig: Capability.LARGE_INTEGERS, Capability.STANDARD_REASONS, }, + known_deviations=tck_known_deviations, ) diff --git a/tools/openfeature-tck/tests/test_report.py b/tools/openfeature-tck/tests/test_report.py new file mode 100644 index 000000000..80346c0aa --- /dev/null +++ b/tools/openfeature-tck/tests/test_report.py @@ -0,0 +1,1218 @@ +"""What the conformance report must never do. + +The report exists because a runner's summary cannot be checked by anything +downstream. So the tests that matter here are not about JSON shape; they are +about the two properties a consumer is entitled to assume, neither of which is +guaranteed by the code that happens to assemble the documents: + +* a scenario skipped for an undeclared capability is never reported as passed, + and the reason it was skipped is recoverable; +* every scenario the run collected appears exactly once, which is what makes the + first property checkable rather than merely asserted -- a document that quietly + dropped what it skipped would satisfy the letter of it and still mislead. + +Both are now checked against the *results payload* rather than against the +envelope, because that is where the results moved: a run writes an envelope and a +`Cucumber Messages`_ stream beside it, and the envelope says only what was tested, +what the provider claims and where the results are. The assertions are written +the way a consumer reads the stream -- a test case is as bad as its worst step -- +so that what these tests check is what a consumer would see rather than an +internal representation. + +Both are checked against a real pytest session in a subprocess, because both are +properties of how the suite runs rather than of how the documents are assembled. +That session is also the only place a skip, a pass and a failure occur together, +and the only place the payload can be seen to disagree with the runner's summary +-- which it does, deliberately, for a known deviation. + +.. _Cucumber Messages: https://github.com/cucumber/messages +""" + +from __future__ import annotations + +import collections +import dataclasses +import hashlib +import json +import os +import subprocess +import sys +import typing +from pathlib import Path + +import pytest + +from openfeature.contrib.tools.tck import ( + DECLARABLE_CAPABILITIES, + EXTENSIONS_DIRECTORY, + RESERVED_CAPABILITIES, + BackendType, + Capability, + KnownDeviation, + TckConfig, + canonical_root, +) +from openfeature.contrib.tools.tck.emitter import ( + classify_phase, + scenario_run, +) +from openfeature.contrib.tools.tck.extensions import CANONICAL_DIRECTORY +from openfeature.contrib.tools.tck.messages import ( + MESSAGES_FORMAT, + ScenarioIdentity, + Status, + StepRun, + _Pickle, + _step_runs, + feature_uri, +) +from openfeature.contrib.tools.tck.report import ( + REPORT_DIR_ENV, + PhaseOutcome, + Results, + SuiteReport, + envelope_file_name, + normalise_tags, + spec_revision, + stream_file_name, +) + +# The generated suite's name is deliberately not path-safe. +SUITE_NAME = "report/fixture" +SUITE_FILE = "report-fixture.json" + +UNKNOWN_KEY_SCENARIO = "An unknown flag key returns the code default" + +# The type-mismatch matrix: eight Examples rows under one scenario name, one of +# which the Python SDK fails. It is the case row identity exists for. +MISMATCH_SCENARIO = "Requesting the wrong type returns the code default" + +# The row that fails, spelled as the feature file spells it -- strings, because +# Gherkin has no types and "1" is not 1. +DEVIATING_ROW = {"key": "boolean-flag", "requested": "Integer", "default": "1"} + + +def _canonical_root() -> Path: + """The packaged canonical directory, or a failure that says how to get one. + + ``canonical_root()`` answers ``None`` when the assets are not on a + filesystem, which is the honest answer for a zipimport and a missing build + step everywhere else. + """ + root = canonical_root() + assert root is not None, ( + "the packaged canonical features must be on a filesystem for this file " + "to have anything to say; run `poe sync-spec-assets` first" + ) + return root + + +CANONICAL_ROOT = _canonical_root() + + +DEVIATION_ISSUE = "https://github.com/open-feature/python-sdk/issues/619" + +# How Cucumber orders its statuses. A test case is as bad as its worst step, and +# this is the rule a consumer applies to derive a scenario's outcome from the +# stream -- so it is the rule these tests apply too. +SEVERITY = [ + "UNKNOWN", + "PASSED", + "SKIPPED", + "PENDING", + "UNDEFINED", + "AMBIGUOUS", + "FAILED", +] + +_SUITE_MODULE = '''\ +"""A one-fixture adoption, generated so the report can be checked end to end.""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + InProcessControl, + KnownDeviation, + TckConfig, + feature_paths, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="{name}", + control=control, + new_provider=control.new_provider, + capabilities={capabilities}, + known_deviations={deviations}, + ) + + +scenarios(*feature_paths()) +''' + +CAPABILITIES = "{Capability.EVENTS, Capability.OBJECT, Capability.LARGE_INTEGERS}" +"""What the main generated suite declares: enough to produce a skip and a pass. + +``LARGE_INTEGERS`` rather than ``NUMERIC_COERCION`` for the same reason the +in-memory self-tests declare the one and not the other -- a Python ``int`` is +unbounded, and the provider behind ``InProcessControl`` hands each variant back +untouched rather than coercing it. Three declared capabilities are what these +tests need; which three has to stay an honest claim about the provider. +""" + +DEVIATIONS = ( + "(KnownDeviation(" + f'issue="{DEVIATION_ISSUE}", ' + 'summary="a boolean satisfies an Integer request"),)' +) + +# One scenario skipped outright and one known deviation marked xfail, so the run +# produces a skip, a pass and a failure and finishes green while the payload +# does not. +_CONFTEST_MODULE = """\ +import pytest + +SKIPPED = "test_an_unknown_flag_key_returns_the_code_default" +DEVIATION = "test_requesting_the_wrong_type_returns_the_code_default[boolean-flag-Integer-1]" + + +def pytest_collection_modifyitems(items): + for item in items: + if item.name == SKIPPED: + item.add_marker(pytest.mark.skip(reason="deliberately not run here")) + elif item.name == DEVIATION: + item.add_marker(pytest.mark.xfail(reason="python-sdk#619", strict=True)) +""" + + +# A Scenario Outline whose second Examples block carries a tag of its own, which +# no canonical feature file does yet. Written here so that the one case where two +# rows of an outline are gated differently is covered. +_TAGGED_FEATURE = """\ +Feature: Per-Examples tags + + Background: + Given a stable provider + + Scenario Outline: Requesting the wrong type returns the code default + Given a -flag with key "" and a default value "" + When the flag was evaluated with details + Then the resolved details value should be "" + And the reason should be "ERROR" + And the error-code should be "TYPE_MISMATCH" + And no exception should have been thrown + + Examples: ungated + | key | requested | default | + | string-flag | Boolean | false | + | string-flag | Integer | 1 | + + @object + Examples: gated behind a capability this suite does not declare + | key | requested | default | + | string-flag | Float | 0.1 | +""" + +_TAGGED_SUITE = '''\ +"""A suite whose extension feature file tags one Examples block of an outline. + +The canonical set runs alongside it, because a suite that leaves the canonical +set out writes no report at all -- see ``_canonical_set_ran``. +""" + +import pytest +from pytest_bdd import scenarios + +from openfeature.contrib.tools.tck import ( + Capability, + InProcessControl, + TckConfig, + feature_paths, +) + + +@pytest.fixture(scope="session") +def tck_config(): + control = InProcessControl() + return TckConfig( + name="per-examples", + control=control, + new_provider=control.new_provider, + capabilities={Capability.EVENTS}, + ) + + +scenarios(*feature_paths()) +''' + + +# -- reading the payload back the way a consumer would ----------------------- + + +@dataclasses.dataclass(frozen=True) +class Case: + """One scenario as the stream reports it, assembled from its messages.""" + + uri: str + name: str + """The scenario name from the *AST*, so an outline's rows share it.""" + + row: tuple[tuple[str, str], ...] + """The Examples row, resolved from the pickle's AST node ids.""" + + tags: frozenset[str] + status: str + """The worst of the test case's steps, which is Cucumber's rule.""" + + setup_message: str + """What the before-hook said, which is where a skip's reason lands.""" + + step_statuses: tuple[str, ...] + + @property + def identity(self) -> tuple[str, str, tuple[tuple[str, str], ...]]: + return (self.uri, self.name, self.row) + + +@dataclasses.dataclass(frozen=True) +class Stream: + """A parsed Cucumber Messages stream.""" + + kinds: collections.Counter[str] + sources: dict[str, str] + cases: list[Case] + + def named(self, name: str) -> list[Case]: + return [case for case in self.cases if case.name == name] + + @property + def statuses(self) -> collections.Counter[str]: + return collections.Counter(case.status for case in self.cases) + + +@dataclasses.dataclass +class _Index: + """The stream's messages, keyed the way the protocol says they relate.""" + + kinds: collections.Counter[str] = dataclasses.field( + default_factory=collections.Counter + ) + sources: dict[str, str] = dataclasses.field(default_factory=dict) + names: dict[str, str] = dataclasses.field(default_factory=dict) + rows: dict[str, tuple[tuple[str, str], ...]] = dataclasses.field( + default_factory=dict + ) + pickles: dict[str, typing.Any] = dataclasses.field(default_factory=dict) + test_cases: dict[str, typing.Any] = dataclasses.field(default_factory=dict) + hooks: dict[str, str] = dataclasses.field(default_factory=dict) + started: dict[str, str] = dataclasses.field(default_factory=dict) + results: dict[str, list[tuple[str, typing.Any]]] = dataclasses.field( + default_factory=dict + ) + + def add(self, kind: str, body: typing.Any) -> None: + self.kinds[kind] += 1 + if kind == "source": + self.sources[body["uri"]] = body["data"] + elif kind == "gherkinDocument": + _index_document(body, self.names, self.rows) + elif kind == "pickle": + self.pickles[body["id"]] = body + elif kind == "testCase": + self.test_cases[body["id"]] = body + for step in body["testSteps"]: + if "hookId" in step: + self.hooks[step["id"]] = step["hookId"] + elif kind == "testCaseStarted": + self.started[body["id"]] = body["testCaseId"] + elif kind == "testStepFinished": + self.results.setdefault(body["testCaseStartedId"], []).append( + (body["testStepId"], body["testStepResult"]) + ) + + +def _read_stream(path: Path) -> Stream: + """Assemble the stream into test cases the way a consumer has to. + + Deliberately written against the protocol rather than against this package: + a pickle's ``astNodeIds`` are followed back into the ``GherkinDocument`` to + recover the scenario name and the Examples row, and a test case's status is + computed as the worst of its steps. If the stream does not actually support + those two operations, these tests fail -- which is the point. + """ + index = _Index() + for line in path.read_text(encoding="utf-8").splitlines(): + message = json.loads(line) + kind = next(iter(message)) + index.add(kind, message[kind]) + + cases = [ + _case(index, started_id, case_id) + for started_id, case_id in index.started.items() + ] + return Stream(kinds=index.kinds, sources=index.sources, cases=cases) + + +def _case(index: _Index, started_id: str, case_id: str) -> Case: + pickle = index.pickles[index.test_cases[case_id]["pickleId"]] + ast = pickle["astNodeIds"] + steps = index.results[started_id] + setup: dict[str, typing.Any] = next( + ( + result + for step_id, result in steps + if index.hooks.get(step_id, "").endswith("setup") + ), + {}, + ) + return Case( + uri=pickle["uri"], + name=index.names[ast[0]], + row=index.rows.get(ast[1], ()) if len(ast) > 1 else (), + tags=frozenset(tag["name"] for tag in pickle.get("tags", ())), + status=max((result["status"] for _, result in steps), key=SEVERITY.index), + setup_message=setup.get("message", ""), + step_statuses=tuple( + result["status"] for step_id, result in steps if step_id not in index.hooks + ), + ) + + +def _index_document( + document: dict[str, typing.Any], + names: dict[str, str], + rows: dict[str, tuple[tuple[str, str], ...]], +) -> None: + def visit(children: typing.Iterable[dict[str, typing.Any]]) -> None: + for child in children: + if "rule" in child: + visit(child["rule"].get("children", ())) + continue + scenario = child.get("scenario") + if scenario is None: + continue + names[scenario["id"]] = scenario["name"] + for examples in scenario.get("examples", ()): + header = examples.get("tableHeader") + if header is None: + continue + headers = [cell["value"] for cell in header["cells"]] + for row in examples.get("tableBody", ()): + cells = [cell["value"] for cell in row["cells"]] + rows[row["id"]] = tuple(zip(headers, cells, strict=True)) + + feature = document.get("feature") + if feature is not None: + visit(feature.get("children", ())) + + +@dataclasses.dataclass(frozen=True) +class Run: + """One subprocess run of the generated suite: both documents it wrote.""" + + directory: Path + result: subprocess.CompletedProcess[str] + envelope: dict[str, typing.Any] + stream: Stream + stream_path: Path + + @property + def declared(self) -> set[str]: + return set(self.envelope["declaration"]["declared"]) + + +# -- helpers ----------------------------------------------------------------- + + +class _StubControl: + """A minimal control, driving an in-process backend.""" + + @property + def description(self) -> str: + return "a stub" + + @property + def backend_type(self) -> BackendType: + return "in-process" + + def prepare_scenario(self) -> None: + return None + + def change_flag(self) -> None: + return None + + +class _HttpControl(_StubControl): + @property + def backend_type(self) -> BackendType: + return "http" + + +class _SilentControl: + """A control that does not say how it drove the backend.""" + + @property + def description(self) -> str: + return "a silent stub" + + def prepare_scenario(self) -> None: + return None + + def change_flag(self) -> None: + return None + + +def _config_leaving_capabilities_to_their_default() -> TckConfig: + """A config that does not narrow ``capabilities``, so the field default runs. + + Written out rather than routed through ``_config``, which supplies a narrow + set of its own: the default is the whole point of this one. It needs an + unavailable-provider factory because ``@unavailable`` is declarable and the + default therefore declares it. + """ + settings: dict[str, typing.Any] = { + "name": "stub", + "control": _StubControl(), + "new_provider": lambda: None, + "new_unavailable_provider": lambda: None, + } + return TckConfig(**settings) + + +def _config(**overrides: typing.Any) -> TckConfig: + settings: dict[str, typing.Any] = { + "name": "stub", + "control": _StubControl(), + "new_provider": lambda: None, + "capabilities": {Capability.EVENTS}, + } + settings.update(overrides) + return TckConfig(**settings) + + +def _identity(*tags: str, name: str = "a scenario") -> ScenarioIdentity: + return ScenarioIdentity( + uri=f"{CANONICAL_DIRECTORY}/events.feature", + path=CANONICAL_ROOT / "events.feature", + name=name, + tags=tags, + ) + + +def _results() -> Results: + return Results(location="stub.ndjson", digest="sha256:" + "0" * 64) + + +def _examples_from_the_feature_file(feature: str, outline: str) -> list[dict[str, str]]: + """Read an outline's Examples tables straight out of the Gherkin. + + Hand-read rather than taken from a parser, because a parser is what produced + the values under test: asking it what it should have said would check + nothing. It is a small reader for a small shape -- the tables in these files + are plain pipe-delimited rows -- and it exists so that "the stream says what + the table said" is checked against the table. + """ + source = CANONICAL_ROOT / f"{feature}.feature" + lines = source.read_text(encoding="utf-8").splitlines() + rows: list[dict[str, str]] = [] + headers: list[str] = [] + inside = False + + for line in lines: + stripped = line.strip() + if stripped.startswith(("Scenario:", "Scenario Outline:")): + inside = stripped.split(":", 1)[1].strip() == outline + headers = [] + elif not inside: + continue + elif stripped.startswith("Examples"): + headers = [] + elif stripped.startswith("|"): + cells = [cell.strip() for cell in stripped.strip("|").split("|")] + if headers: + rows.append(dict(zip(headers, cells, strict=True))) + else: + headers = cells + + assert rows, f"no Examples rows found for {outline!r} in {feature}.feature" + return rows + + +def _phase(outcome: str, when: str = "call", **extra: typing.Any) -> PhaseOutcome: + return PhaseOutcome(when=when, outcome=outcome, **extra) + + +def _pytest( + *arguments: str, report_dir: Path | None = None +) -> subprocess.CompletedProcess[str]: + environment = dict(os.environ) + environment.pop(REPORT_DIR_ENV, None) + if report_dir is not None: + environment[REPORT_DIR_ENV] = str(report_dir) + return subprocess.run( # noqa: S603 + [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", *arguments], + capture_output=True, + text=True, + env=environment, + check=False, + ) + + +def _write_suite( + directory: Path, + name: str = SUITE_NAME, + capabilities: str = CAPABILITIES, + deviations: bool = True, +) -> Path: + directory.mkdir(parents=True, exist_ok=True) + (directory / "test_suite.py").write_text( + _SUITE_MODULE.format( + name=name, + capabilities=capabilities, + deviations=DEVIATIONS if deviations else "()", + ), + encoding="utf-8", + ) + if deviations: + (directory / "conftest.py").write_text(_CONFTEST_MODULE, encoding="utf-8") + return directory + + +def _run_suite( + tmp_path_factory: pytest.TempPathFactory, + file_name: str = SUITE_FILE, + **suite: typing.Any, +) -> Run: + """Run one generated suite in a subprocess and read what it wrote.""" + directory = _write_suite(tmp_path_factory.mktemp("suite"), **suite) + reports = tmp_path_factory.mktemp("reports") + result = _pytest(str(directory), report_dir=reports) + + path = reports / file_name + assert path.exists(), ( + f"no report at {path}; pytest exited {result.returncode}\n" + f"{result.stdout}\n{result.stderr}" + ) + envelope = json.loads(path.read_text(encoding="utf-8")) + stream_path = path.parent / envelope["results"]["location"] + return Run( + directory=directory, + result=result, + envelope=envelope, + stream=_read_stream(stream_path), + stream_path=stream_path, + ) + + +@pytest.fixture(scope="module") +def run(tmp_path_factory: pytest.TempPathFactory) -> Run: + """One real run of the generated suite, with a report asked for.""" + return _run_suite(tmp_path_factory) + + +@pytest.fixture(scope="module") +def narrow_run(tmp_path_factory: pytest.TempPathFactory) -> Run: + """A run of a suite that leaves a whole Scenario Outline gated. + + ``@object`` is undeclared so that every row of one outline is skipped by the + capability gate, which is the case that has to keep saying which row it + skipped. + """ + return _run_suite( + tmp_path_factory, + file_name="narrow.json", + name="narrow", + capabilities="{Capability.LARGE_INTEGERS}", + deviations=False, + ) + + +# -- the two properties that matter ------------------------------------------ + + +def test_a_capability_skip_is_never_reported_as_passed(run: Run) -> None: + """The rule Appendix F states, checked against the payload, not the runner.""" + gated = [case for case in run.stream.cases if case.tags - run.declared] + assert gated, "the generated suite is meant to have scenarios behind those" + + for case in gated: + assert case.status == "SKIPPED", case + # Cucumber's SKIPPED is per step, so "never as passed" has to hold of + # every step and not merely of the rolled-up verdict. + assert set(case.step_statuses) == {"SKIPPED"}, case + assert case.setup_message, f"a skip must say why: {case}" + + +def test_the_reason_for_a_gated_skip_follows_from_the_two_documents(run: Run) -> None: + """Which is why the per-scenario reason no longer has to be transported. + + The envelope says what the provider declares; the payload says which tags + each scenario carries and that it was skipped. A consumer with both can name + the capability responsible without the emitter having written it down once + per scenario, and that derivation is what the declaration exists for. + """ + skipped = [case for case in run.stream.cases if case.status == "SKIPPED"] + gated = [case for case in skipped if case.tags - run.declared] + assert gated, "no skip was attributable to an undeclared capability" + + # The derivation is checked against the reason the runner actually gave: for + # every skip the two documents attribute to a capability, that capability is + # the one the gate named. If they disagreed, the declaration would be the + # wrong thing to read a skip against. + for case in gated: + responsible = case.tags - run.declared + assert any(tag in case.setup_message for tag in responsible), case + + # And it distinguishes: the one scenario skipped for another reason carries + # no undeclared tag, so it is not attributed to a capability at all. + others = [case for case in skipped if case not in gated] + assert others, "the generated suite is meant to skip one scenario outright" + for case in others: + assert not case.tags - run.declared, case + assert "capability" not in case.setup_message, case + + +def test_every_collected_scenario_appears_exactly_once(run: Run) -> None: + """The property that makes the rule above checkable rather than promised. + + A test case is identified by its uri, its scenario name **and its Examples + row**, all three recovered from the stream by following a pickle's AST node + ids. Name alone is shared by every row of a Scenario Outline, so keying on it + would let the eight rows of the type-mismatch matrix collapse into one and this + test would not notice. + + Counted against pytest's own collection rather than against a number written + down here, so that adding a scenario to the specification cannot leave this + passing while the payload loses one. + """ + identities = [case.identity for case in run.stream.cases] + assert len(identities) == len(set(identities)), "a scenario is reported twice" + + collected = _pytest("--collect-only", str(run.directory)) + assert len(identities) == sum( + 1 for line in collected.stdout.splitlines() if "::test_" in line + ) + + +def test_the_statuses_account_for_every_scenario(run: Run) -> None: + counts = run.stream.statuses + assert set(counts) <= set(SEVERITY), "a status outside the protocol's own" + assert sum(counts.values()) == len(run.stream.cases) + # A skip, a pass and a failure all occur, which is what makes the run worth + # asserting against at all. + assert set(counts) == {"PASSED", "SKIPPED", "FAILED"}, counts + + +def test_the_payload_does_not_repeat_the_runner_summary(run: Run) -> None: + """A known deviation is a failure in the payload even when pytest is green. + + The suite marks the one scenario the Python SDK cannot satisfy as an expected + failure, so pytest exits zero. The provider still did not satisfy it, and a + payload that agreed with the summary would hide exactly what the marker was + added to keep visible. The acknowledgement goes in the envelope instead. + """ + assert run.result.returncode == 0, run.result.stdout + failed = [case for case in run.stream.cases if case.status == "FAILED"] + assert len(failed) == 1 + assert failed[0].row == tuple(DEVIATING_ROW.items()) + + deviations = run.envelope["knownDeviations"] + assert [deviation["issue"] for deviation in deviations] == [DEVIATION_ISSUE] + + +def test_a_scenario_skipped_for_another_reason_is_still_reported(run: Run) -> None: + """A run that chose not to execute a scenario still accounts for it. + + Skipped, and present, even though a marker skip never runs a fixture -- and + with its own reason rather than the capability gate's, which is what tells + the two apart now that the payload has one status for both. + """ + matching = run.stream.named(UNKNOWN_KEY_SCENARIO) + assert len(matching) == 1 + assert matching[0].status == "SKIPPED" + assert "deliberately not run here" in matching[0].setup_message + # Nothing about the provider's declaration explains this one, which is how a + # consumer tells it from a capability skip. + assert not matching[0].tags - run.declared + + +# -- which row of an outline ------------------------------------------------- + + +def test_an_outline_row_is_identified_by_its_ast_node_id(run: Run) -> None: + """The eight rows of the type-mismatch matrix are told apart, and only here. + + All eight share one scenario name, which is the feature file's name and must + stay that way: it is what a report from Go or JavaScript carries for the same + row. What tells them apart is the pickle's second ``astNodeIds`` entry, the + id of the table row it was compiled from, which resolves in the + ``GherkinDocument`` to exactly the cells the feature file wrote. That is the + identity four implementations were each reinventing as a bespoke ``example`` + field before this format carried it. + """ + rows = run.stream.named(MISMATCH_SCENARIO) + expected = _examples_from_the_feature_file("errors", MISMATCH_SCENARIO) + # The literal is what keeps this from passing on two empty lists, and it + # moves when the assets do: the matrix lost the three string rows to + # @string-typing at spec d47a66eb, eleven down to eight. + assert len(rows) == len(expected) == 8 + + observed = [dict(case.row) for case in rows] + assert len(observed) == len({case.row for case in rows}), "two rows collapsed" + assert sorted(map(sorted, (row.items() for row in observed))) == sorted( + map(sorted, (row.items() for row in expected)) + ) + + # Verbatim strings, because Gherkin has no types: a "1" in a table is the + # one-character cell the feature file contains. + for row in observed: + assert all(isinstance(value, str) for value in row.values()), row + + +def test_a_scenario_that_is_not_an_outline_has_no_row(run: Run) -> None: + """One AST node id, so there is no row to resolve and nothing to say.""" + plain = run.stream.named(UNKNOWN_KEY_SCENARIO) + assert len(plain) == 1 + assert plain[0].row == () + + +def test_a_capability_skipped_outline_row_is_still_identified( + narrow_run: Run, +) -> None: + """A skipped row is exactly as ambiguous as a failed one. + + Row identity comes from the pickle rather than from the run, so it does not + depend on the scenario having executed -- which is what lets a row the + capability gate stopped before its first step be told apart from its siblings + just as well as one that failed. + """ + outline = "Requesting a structured flag as a scalar returns the code default" + expected = _examples_from_the_feature_file("errors", outline) + rows = narrow_run.stream.named(outline) + assert len(rows) == len(expected) + + for case in rows: + assert case.status == "SKIPPED", case + assert case.row, f"a skipped outline row must say which row: {case}" + assert sorted(map(sorted, (dict(case.row).items() for case in rows))) == sorted( + map(sorted, (row.items() for row in expected)) + ) + + +def test_a_row_gated_by_its_examples_block_is_the_only_one_skipped( + tmp_path: Path, +) -> None: + """Gherkin lets one Examples block of an outline carry its own tags. + + Two rows of one Scenario Outline can therefore differ in which capability + gates them. Those tags are on neither the scenario, the feature nor the rule, + and a payload built from those three would show the skipped row as carrying + no capability -- leaving the envelope's declaration unable to explain the + skip, which is the one derivation this format asks a consumer to make. + + No canonical feature file does this yet, so the feature file is written here + -- as an extension, because a suite that leaves the canonical set out writes + no report to read back. + """ + directory = tmp_path / "suite" + extensions = directory / EXTENSIONS_DIRECTORY + extensions.mkdir(parents=True) + (extensions / "tagged.feature").write_text(_TAGGED_FEATURE, encoding="utf-8") + (directory / "test_tagged.py").write_text(_TAGGED_SUITE, encoding="utf-8") + + reports = tmp_path / "reports" + result = _pytest(str(directory), report_dir=reports) + path = reports / "per-examples.json" + assert path.exists(), f"pytest exited {result.returncode}\n{result.stdout}" + + envelope = json.loads(path.read_text(encoding="utf-8")) + stream = _read_stream(path.parent / envelope["results"]["location"]) + by_row = { + dict(case.row)["requested"]: case + for case in stream.cases + if case.uri == "extensions/tagged.feature" + } + + # Every row is still reported: nothing about gating one row of an outline may + # drop its siblings from the payload. + assert set(by_row) == {"Boolean", "Integer", "Float"}, by_row + assert by_row["Boolean"].status == "PASSED" + assert by_row["Integer"].status == "PASSED" + + gated = by_row["Float"] + assert gated.status == "SKIPPED", gated + assert Capability.OBJECT.tag in gated.tags, gated + assert Capability.OBJECT.tag not in envelope["declaration"]["declared"] + + +# -- the payload, and the envelope that points at it ------------------------- + + +def test_the_envelope_points_at_the_payload_it_describes(run: Run) -> None: + """Referenced rather than inlined, and covered by a digest. + + A stream carries the feature sources and is far larger than the envelope, so + a consumer deciding whether it cares about a report should not have to fetch + a whole run to find out -- and needs to be able to tell that what it did + fetch is what the envelope described. + """ + results = run.envelope["results"] + assert results["format"] == MESSAGES_FORMAT + # A bare file name, so the reference survives the pair being moved together. + assert results["location"] == run.stream_path.name + assert Path(results["location"]).parent == Path() + + digest = hashlib.sha256(run.stream_path.read_bytes()).hexdigest() + assert results["digest"] == f"sha256:{digest}" + + +def test_the_payload_carries_the_source_of_every_feature_it_ran(run: Run) -> None: + """Which is what replaced recording an asset tree hash. + + A hash said only whether two runs executed the same assets. The source says + what the assets were, so a consumer can read the questions that were actually + asked rather than trusting a recorded revision to stand for them. + """ + uris = {case.uri for case in run.stream.cases} + assert uris, "no test case named a feature file" + assert set(run.stream.sources) == uris + + for uri, data in run.stream.sources.items(): + on_disk = CANONICAL_ROOT / Path(uri).name + assert data == on_disk.read_text(encoding="utf-8"), uri + + assert "assetsTree" not in run.envelope["tck"] + + +def test_the_payload_is_a_well_formed_messages_stream(run: Run) -> None: + """The message types a consumer needs are all present, once each per scenario.""" + kinds = run.stream.kinds + cases = len(run.stream.cases) + assert kinds["meta"] == 1 + assert kinds["testRunStarted"] == 1 + assert kinds["testRunFinished"] == 1 + assert kinds["source"] == kinds["gherkinDocument"] == len(run.stream.sources) + assert kinds["pickle"] == cases + assert kinds["testCase"] == kinds["testCaseStarted"] == cases + assert kinds["testCaseFinished"] == cases + assert kinds["testStepStarted"] == kinds["testStepFinished"] + + +def test_the_declaration_is_an_input_not_a_summary(run: Run) -> None: + """Which is why it cannot be derived from the payload and is stated here. + + The payload cannot express it, because a skip in it says only that the + question was not put to this provider. What the declaration adds is whether + that is because the capability was not claimed -- and that is all it adds: + one skip carrying its reason is the whole mechanism, so the block holds the + declared set and nothing restating it. + """ + declaration = run.envelope["declaration"] + assert set(declaration) == {"declared"} + assert declaration["declared"] == [ + Capability.EVENTS.tag, + Capability.LARGE_INTEGERS.tag, + Capability.OBJECT.tag, + ] + assert Capability.STALE.tag not in declaration["declared"] + + +def test_the_provider_and_its_configuration_are_reported_separately(run: Run) -> None: + assert run.envelope["provider"]["name"] == "In-Memory Provider" + assert run.envelope["provider"]["configuration"] == SUITE_NAME + assert run.envelope["provider"]["language"] == "python" + + +def test_the_envelope_names_what_ran_it(run: Run) -> None: + assert run.envelope["schemaVersion"] == "1" + assert ( + run.envelope["tck"]["implementation"] + == "python-sdk-contrib/tools/openfeature-tck" + ) + assert run.envelope["sdk"]["name"] == "openfeature-sdk" + assert run.envelope["sdk"]["version"] + assert len(run.envelope["tck"]["specRevision"]) >= 7 + assert run.envelope["backend"]["backendType"] == "in-process" + + +def test_the_envelope_carries_no_results_of_its_own(run: Run) -> None: + """The fields Cucumber Messages made redundant, checked to be gone. + + Not a shape test for its own sake: while both existed there were two places + for the same fact to disagree, which is the whole reason the per-scenario + list was deleted rather than kept alongside the payload. + """ + assert "scenarios" not in run.envelope + assert "capabilities" not in run.envelope + + +def test_the_spec_revision_comes_from_the_build() -> None: + """Generated beside the assets, because the submodule is not in the wheel.""" + assert len(spec_revision()) >= 7 + + +# -- opting in --------------------------------------------------------------- + + +def test_no_report_is_written_without_the_environment_variable( + tmp_path: Path, +) -> None: + """The default, and not an error: emitting is a property of the run.""" + directory = _write_suite(tmp_path / "suite") + result = _pytest(str(directory), report_dir=None) + assert result.returncode == 0, result.stdout + assert "report written" not in result.stdout + assert not list(tmp_path.rglob("*.json")) + assert not list(tmp_path.rglob("*.ndjson")) + + +def test_a_report_that_cannot_be_written_fails_the_run(tmp_path: Path) -> None: + """Loudly, because a pipeline that silently got no report serves a stale one. + + The destination is placed under a regular file, which no platform will let + ``mkdir`` turn into a directory. The run itself passes, so a non-zero exit + can only have come from the failure to write. + """ + blocker = tmp_path / "not-a-directory" + blocker.write_text("", encoding="utf-8") + directory = _write_suite(tmp_path / "suite") + result = _pytest(str(directory), report_dir=blocker / "reports") + assert "could not write the conformance report" in result.stdout + assert result.returncode != 0 + + +# -- assembling the documents ------------------------------------------------ + + +def test_a_failure_is_not_revised_away_by_a_later_phase() -> None: + """A scenario whose steps passed and whose teardown blew up is a failure.""" + run = scenario_run( + _identity(), + [ + _phase("passed", when="setup"), + _phase("passed", when="call"), + _phase("failed", when="teardown", message="teardown exploded"), + ], + [StepRun(status=Status.passed)], + ) + assert run.status is Status.failed + assert run.message == "teardown exploded" + + +def test_a_verdict_no_step_accounts_for_reaches_the_stream_anyway() -> None: + """A strict xfail that passes fails a scenario every step of which passed. + + A consumer reads a test case's outcome as the worst of its steps, so a + verdict left only in this package's own head would be lost on the way out. + It is attached to the after-hook, which is where a test case failing outside + its own steps belongs. + """ + run = scenario_run( + _identity(), + [ + _phase("passed", when="setup"), + _phase("failed", when="call", message="[XPASS(strict)] python-sdk#619"), + _phase("passed", when="teardown"), + ], + [StepRun(status=Status.passed)], + ) + assert run.status is Status.failed + assert run.teardown is not None + assert run.teardown.status is Status.passed, "the phase itself did pass" + + # The stream is where the discrepancy has to be resolved, and it is: + # `_step_runs` upgrades the after-hook when nothing else carries the verdict. + steps = _step_runs(_Pickle(id="p", step_ids=("s",), payload={}), run) + worst = max((step.status.value for step in steps), key=SEVERITY.index) + assert worst == Status.failed.value + assert "XPASS(strict)" in steps[-1].message + + +def test_a_gated_skip_is_skipped_for_every_step() -> None: + """The gate stops a scenario in setup, so no step of it ran.""" + run = scenario_run( + _identity("@stale"), + [_phase("skipped", when="setup", message="provider does not declare @stale")], + [], + ) + assert run.status is Status.skipped + assert run.setup is not None + assert run.setup.message == "provider does not declare @stale" + + +def test_the_declaration_reports_what_the_configuration_declares() -> None: + """The declared set, and nothing beside it restating what a skip says.""" + suite = SuiteReport(config=_config(capabilities={Capability.EVENTS})) + declaration = suite.build(_results())["declaration"] + assert declaration == {"declared": [Capability.EVENTS.tag]} + + +def test_a_reserved_capability_cannot_be_declared() -> None: + """A tag no scenario carries is a claim nothing can check, so it is refused. + + Refused at construction rather than dropped at emission time: the adopter + wrote it down and meant something by it, and a config silently different + from the one they wrote is worse than one that will not build. This is also + where their own code is still on the stack. + """ + for reserved in RESERVED_CAPABILITIES: + with pytest.raises(ValueError, match=f"reserved capabilities {reserved.tag}"): + _config(capabilities={Capability.EVENTS, reserved}) + + +def test_a_reserved_capability_cannot_reach_the_declaration() -> None: + """Including by the route that actually caused it: declaring everything. + + The schema forbids a capability no executed scenario carries from appearing + in ``declaration.declared``, because such a tag cannot produce a skip and so + plays no part in reading the results -- it only invites a reader to believe + something was verified when nothing examined it. A real report from another + implementation asserts ``@targeting`` and ``@caching`` for exactly this + reason: that adoption declares "every capability except X" and collected the + reserved tags on the way past. So the default is the declarable set rather + than the whole enum. + """ + declared = SuiteReport( + config=_config_leaving_capabilities_to_their_default() + ).build(_results())["declaration"]["declared"] + + assert declared == sorted(capability.tag for capability in DECLARABLE_CAPABILITIES) + assert RESERVED_CAPABILITIES, "the rule is vacuous if nothing is reserved" + for reserved in RESERVED_CAPABILITIES: + assert reserved.tag not in declared + + +def test_known_deviations_are_omitted_rather_than_emitted_empty() -> None: + """Stating none is a claim; omitting the field is silence.""" + assert "knownDeviations" not in SuiteReport(config=_config()).build(_results()) + + acknowledged = SuiteReport( + config=_config( + known_deviations=( + KnownDeviation( + issue=DEVIATION_ISSUE, + summary="a boolean satisfies an Integer request", + capability=Capability.NUMERIC_COERCION, + ), + ) + ) + ).build(_results())["knownDeviations"] + assert acknowledged == [ + { + "issue": DEVIATION_ISSUE, + "summary": "a boolean satisfies an Integer request", + "capability": Capability.NUMERIC_COERCION.tag, + } + ] + + +def test_the_provider_name_falls_back_to_the_suite_name() -> None: + """A suite whose every scenario was skipped never saw a provider. + + Reporting the suite name is more useful than the empty string the schema + would reject. + """ + envelope = SuiteReport(config=_config()).build(_results()) + assert envelope["provider"]["name"] == "stub" + + +def test_the_backend_block_and_its_backend_type_are_always_written() -> None: + """Both are required by the schema, so neither is conditional here. + + A provider with no backend still had its flag state manipulated somehow, + and how is what the rest of the document is worth: the same scenarios + passing over the control API and passing through + in-process manipulation of a provider that does have a backend are not the + same claim. The value comes straight off the control, because nothing + outside a control can tell which path it took. + """ + in_process = SuiteReport(config=_config()).build(_results()) + assert in_process["backend"]["backendType"] == "in-process" + + http = SuiteReport(config=_config(control=_HttpControl())).build(_results()) + assert http["backend"]["backendType"] == "http" + + silent = SuiteReport(config=_config(control=_SilentControl())).build(_results()) + assert silent["backend"]["backendType"] == "custom" + + +def test_an_empty_description_is_left_out_rather_than_emitted_blank() -> None: + """``description`` is free text for a person, and the schema leaves it optional.""" + + class _Nameless(_StubControl): + @property + def description(self) -> str: + return "" + + backend = SuiteReport(config=_config(control=_Nameless())).build(_results())[ + "backend" + ] + assert backend == {"backendType": "in-process"} + + +@pytest.mark.parametrize( + ("suite_name", "expected"), + [ + ("in-memory", "in-memory"), + ("flagd/rpc", "flagd-rpc"), + ("../escape", "escape"), + ("...", "report"), + ], +) +def test_a_suite_name_cannot_write_outside_its_directory( + suite_name: str, expected: str +) -> None: + """Suite names are chosen to read well in a failure message, not to be paths.""" + assert envelope_file_name(suite_name) == f"{expected}.json" + assert stream_file_name(suite_name) == f"{expected}.ndjson" + + +def test_only_tags_the_schema_accepts_are_carried() -> None: + assert normalise_tags({"events", "Not A Tag", "stale"}) == ("@events", "@stale") + + +def test_a_feature_uri_is_slash_separated_on_every_platform() -> None: + """The same string has to appear in the source, the document and the pickles. + + pytest-bdd builds it with ``os.path.join``, so on Windows it arrives + backslash-separated -- and a report emitted there would otherwise not be + comparable with one emitted on Linux. + """ + assert feature_uri(f"{CANONICAL_DIRECTORY}/errors.feature") == ( + f"{CANONICAL_DIRECTORY}/errors.feature" + ) + assert feature_uri(os.path.join(CANONICAL_DIRECTORY, "errors.feature")) == ( + f"{CANONICAL_DIRECTORY}/errors.feature" + ) + + +# -- classifying one phase --------------------------------------------------- + + +def test_an_expected_failure_is_still_a_failure() -> None: + """An xfail marker records a known deviation; it does not excuse one.""" + status, message = classify_phase( + _phase("skipped", xfail_reason="the SDK coerces a bool to an int") + ) + assert status is Status.failed + assert "the SDK coerces a bool to an int" in message + + +def test_a_phase_that_merely_worked_says_nothing_in_particular() -> None: + assert classify_phase(_phase("passed", when="setup")) == (Status.passed, "") + assert classify_phase(_phase("passed", when="call")) == (Status.passed, "") + + +def test_a_skip_keeps_its_reason() -> None: + assert classify_phase( + _phase("skipped", when="setup", message="provider does not declare @stale") + ) == (Status.skipped, "provider does not declare @stale") + assert classify_phase(_phase("skipped", when="setup")) == ( + Status.skipped, + "skipped", + ) diff --git a/uv.lock b/uv.lock index 12e4e0899..8422702f5 100644 --- a/uv.lock +++ b/uv.lock @@ -869,6 +869,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/50/a9caea39ad19c431c1a3f8a31114df65b260cdfe67786b6c7e7c040c4c44/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6", size = 3783731, upload-time = "2026-06-12T20:02:43.319Z" }, ] +[[package]] +name = "cucumber-messages" +version = "34.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/b8/16f4f31045776b7b7fc962e72ebc2a8bf4a12df2f18d8c9be4e258cbe3bb/cucumber_messages-34.2.0.tar.gz", hash = "sha256:712102e0a0f7fb7a3d068a2754b31ce9b605fb04ab65f9f37282f9c27f7254d4", size = 11703, upload-time = "2026-07-19T13:07:38.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/81/00577acc873ebce750da0429c6726e111c75e62caa71843166e522e19b80/cucumber_messages-34.2.0-py3-none-any.whl", hash = "sha256:2b20b7a7151b2ccc296b8f101b413b2b6b8a2f90c3af748d5a357b59243ee2fc", size = 13006, upload-time = "2026-07-19T13:07:37.692Z" }, +] + [[package]] name = "docker" version = "7.1.0" @@ -2138,6 +2147,8 @@ name = "openfeature-tck" version = "0.1.0" source = { editable = "tools/openfeature-tck" } dependencies = [ + { name = "cucumber-messages" }, + { name = "gherkin-official" }, { name = "openfeature-sdk" }, { name = "pytest" }, { name = "pytest-bdd" }, @@ -2158,6 +2169,8 @@ dev = [ [package.metadata] requires-dist = [ + { name = "cucumber-messages", specifier = ">=34.0.0,<35.0.0" }, + { name = "gherkin-official", specifier = ">=29.0.0" }, { name = "openfeature-sdk", specifier = ">=0.10.0" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-bdd", specifier = ">=8.1.0,<9.0.0" },