From b60fb2d496c9ffd4641f942eae601b7d86d1acf8 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Thu, 1 Oct 2026 02:14:48 +0200 Subject: [PATCH 1/9] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Add=20ub-test-reports:?= =?UTF-8?q?=20the=20Sphinx-free=20core=20moved=20out=20of=20sphinx-test-re?= =?UTF-8?q?ports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sixth publishable member, packages/ub-test-reports (distribution `ub-test-reports`, import `ub_test_reports`, 1.0.0.dev0, flit). The ten modules that never needed Sphinx -- cli, fields, identity, jsonparser, junitparser, needs_export, projectconfig, pytest_plugin, remote, results -- and schemas/JUnit.xsd move with `git mv`; their intra-package imports and docstring cross-references now name `ub_test_reports`, and so do the names the plugin prints (its `-p` spelling, the `ub_test_reports.xml_shape` registration name, the warning prefix). The wire names `sphinxcontrib.test_reports:file|line` are unchanged. sphinx-test-reports becomes the Sphinx extension only: hard dependencies on sphinx, docutils, sphinx-needs and `ub-test-reports>=1.0.0.dev0,<2` (the floor the core's release pull request moves to 1.0.0); lxml and ub-project move to the core. `[sphinx]` stays as an empty extra and `[pytest]` as a pass-through to `ub-test-reports[pytest]`, both until 4.0; the `test-reports` script is the core's. The package root imports `setup` eagerly, `setup` calls `app.require_sphinx((7, 4))`, and `toolchain.py` and `compat-requirements.txt` are deleted. The three module aliases point at `ub_test_reports`. Root wiring: the member in `[project] dependencies` and the sources, ruff `src`, ty `include` and `typecheck`, tasks `test-ub-test-reports`, `build-ub-test-reports` and `import-check-ub-test-reports`; the `toolchain` marker is gone. The lock is master's plus the semantic entries only. The suite is NOT green at this commit: the tests still name the moved modules and move in the next one. --- .../compat-requirements.txt | 29 ------- packages/sphinx-test-reports/pyproject.toml | 60 ++++++++------- .../src/sphinx_test_reports/__init__.py | 70 +++-------------- .../directives/test_case.py | 2 +- .../directives/test_common.py | 6 +- .../directives/test_results.py | 2 +- .../src/sphinx_test_reports/test_reports.py | 7 +- .../src/sphinx_test_reports/toolchain.py | 77 ------------------- .../sphinxcontrib/test_reports/__init__.py | 26 ++++--- .../sphinxcontrib/test_reports/jsonparser.py | 6 +- .../sphinxcontrib/test_reports/junitparser.py | 6 +- .../test_reports/pytest_plugin.py | 6 +- packages/ub-test-reports/AGENTS.md | 52 +++++++++++++ packages/ub-test-reports/CLAUDE.md | 1 + packages/ub-test-reports/LICENSE | 21 +++++ packages/ub-test-reports/README.rst | 32 ++++++++ packages/ub-test-reports/docs/changelog.rst | 12 +++ packages/ub-test-reports/pyproject.toml | 60 +++++++++++++++ .../src/ub_test_reports/__init__.py | 17 ++++ .../src/ub_test_reports}/cli.py | 8 +- .../src/ub_test_reports}/fields.py | 4 +- .../src/ub_test_reports}/identity.py | 0 .../src/ub_test_reports}/jsonparser.py | 2 +- .../src/ub_test_reports}/junitparser.py | 4 +- .../src/ub_test_reports}/needs_export.py | 12 +-- .../src/ub_test_reports}/projectconfig.py | 2 +- .../src/ub_test_reports/py.typed | 0 .../src/ub_test_reports}/pytest_plugin.py | 10 +-- .../src/ub_test_reports}/remote.py | 0 .../src/ub_test_reports}/results.py | 4 +- .../src/ub_test_reports}/schemas/JUnit.xsd | 0 pyproject.toml | 50 ++++++++---- uv.lock | 55 +++++++++---- 33 files changed, 363 insertions(+), 280 deletions(-) delete mode 100644 packages/sphinx-test-reports/compat-requirements.txt delete mode 100644 packages/sphinx-test-reports/src/sphinx_test_reports/toolchain.py create mode 100644 packages/ub-test-reports/AGENTS.md create mode 100644 packages/ub-test-reports/CLAUDE.md create mode 100644 packages/ub-test-reports/LICENSE create mode 100644 packages/ub-test-reports/README.rst create mode 100644 packages/ub-test-reports/docs/changelog.rst create mode 100644 packages/ub-test-reports/pyproject.toml create mode 100644 packages/ub-test-reports/src/ub_test_reports/__init__.py rename packages/{sphinx-test-reports/src/sphinx_test_reports => ub-test-reports/src/ub_test_reports}/cli.py (99%) rename packages/{sphinx-test-reports/src/sphinx_test_reports => ub-test-reports/src/ub_test_reports}/fields.py (98%) rename packages/{sphinx-test-reports/src/sphinx_test_reports => ub-test-reports/src/ub_test_reports}/identity.py (100%) rename packages/{sphinx-test-reports/src/sphinx_test_reports => ub-test-reports/src/ub_test_reports}/jsonparser.py (98%) rename packages/{sphinx-test-reports/src/sphinx_test_reports => ub-test-reports/src/ub_test_reports}/junitparser.py (98%) rename packages/{sphinx-test-reports/src/sphinx_test_reports => ub-test-reports/src/ub_test_reports}/needs_export.py (97%) rename packages/{sphinx-test-reports/src/sphinx_test_reports => ub-test-reports/src/ub_test_reports}/projectconfig.py (99%) create mode 100644 packages/ub-test-reports/src/ub_test_reports/py.typed rename packages/{sphinx-test-reports/src/sphinx_test_reports => ub-test-reports/src/ub_test_reports}/pytest_plugin.py (98%) rename packages/{sphinx-test-reports/src/sphinx_test_reports => ub-test-reports/src/ub_test_reports}/remote.py (100%) rename packages/{sphinx-test-reports/src/sphinx_test_reports => ub-test-reports/src/ub_test_reports}/results.py (95%) rename packages/{sphinx-test-reports/src/sphinx_test_reports => ub-test-reports/src/ub_test_reports}/schemas/JUnit.xsd (100%) diff --git a/packages/sphinx-test-reports/compat-requirements.txt b/packages/sphinx-test-reports/compat-requirements.txt deleted file mode 100644 index cfd435417..000000000 --- a/packages/sphinx-test-reports/compat-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# The RELEASED dependencies this package needs beyond what its wheel declares, for the -# compat cell in .github/workflows/release.yaml -- the one gate that runs a member's suite -# against everything as PUBLISHED rather than as checked out. It is installed BEFORE the -# wheel and then passed again as a `-c` constraint, so it cannot be downgraded by the -# wheel's own `Requires-Dist`. -# -# This file covers something neither sibling's does: sphinx-test-reports is the first member -# here whose RUNTIME dependencies are optional. Its 2.0.0 release moved Sphinx and -# sphinx-needs behind a `sphinx` extra -- so that the `test-reports` converter and the pytest -# plugin install without a documentation toolchain -- and the compat cell installs a bare -# wheel path with no extras. Measured on the built wheel: `Requires-Dist` is `lxml` and -# `ub-project`, full stop. -# -# Without these two lines the failure is LOUD rather than silent, which is the good case: -# `release.yaml` runs the `import_check` walk BEFORE pytest, and that walk fails -# `FAIL 6 of 25 modules failed to import` naming `sphinx_needs`, exit 1, under the step's -# default `bash -e`. Measured by running the cell by hand. With them, the walk imports 25 of -# 25 and the suite runs. -# -# Note that `sphinx` alone is already present in that environment -- the root `test` group -# carries it for the other suites -- and `sphinx-needs` is NOT, because it is a `[project]` -# dependency of the workspace ROOT and `uv export --only-group test` deliberately excludes -# it. It is named here anyway: this file states what THIS package needs, and a future change -# to the `test` group must not be able to take it away silently. -# -# The specifiers are the member's own `sphinx` extra, verbatim, so the cell tests the range -# the package actually claims. -sphinx>=7.4 -sphinx-needs>=8.5.0,<9 diff --git a/packages/sphinx-test-reports/pyproject.toml b/packages/sphinx-test-reports/pyproject.toml index d406d84be..d87babd51 100644 --- a/packages/sphinx-test-reports/pyproject.toml +++ b/packages/sphinx-test-reports/pyproject.toml @@ -26,30 +26,35 @@ keywords = ["sphinx", "documentation", "test-reports"] # and publishes; the gap shows up only as an install failure for a user on an interpreter # inside it. This package declared a bare `>=3.11` before the import. requires-python = ">=3.11,<4" -dependencies = ["lxml", "ub-project>=1.1.0,<2"] +# HARD dependencies: this package is the Sphinx extension and nothing else. The converter, +# the pytest plugin, the parsers and the `[test_reports]` model are `ub-test-reports`, which +# runs without the documentation toolchain and is the dependency that brings `lxml` and +# `ub-project` along. +# +# TIGHT tracking on both siblings, not a compatibility range: sphinx-needs and +# ub-test-reports are members of this workspace, and `check_workspace.py` check (4) +# requires `>=,<` on every intra-workspace +# edge. uv never validates a specifier it resolves through `workspace = true` (uv#9811), so +# the floor is the only statement of what was actually tested, and `propagate_floors.py` +# moves it at each sibling release -- ub-test-reports' `1.0.0.dev0` becomes `1.0.0` in the +# core's own release pull request. The published 2.0.0 said `sphinx-needs>=6.0.1`; +# narrowing it is what the import costs, and the changelog says so. +dependencies = [ + "sphinx>=7.4", + "docutils>=0.21", + "sphinx-needs>=8.5.0,<9", + "ub-test-reports>=1.0.0.dev0,<2", +] [project.optional-dependencies] -# The Sphinx extension. The `test-reports` command and the pytest plugin run -# without the documentation toolchain, so it is an extra rather than a -# dependency; the lazy -# `setup` in `__init__.py` enforces these floors when the extension is loaded, -# since an extra is opt-in and pip never sees them otherwise. -# -# TIGHT tracking on the sibling, not a compatibility range: sphinx-needs is a member of -# this workspace, and `check_workspace.py` check (4) requires -# `>=,<` on every intra-workspace edge -- -# including one declared in an extra. uv never validates a specifier it resolves through -# `workspace = true` (uv#9811), so the floor is the only statement of what was actually -# tested, and `propagate_floors.py` moves it at each sphinx-needs release. The published -# 2.0.0 said `>=6.0.1`; narrowing it is what the import costs, and the changelog says so. -sphinx = ["sphinx>=7.4", "docutils>=0.21", "sphinx-needs>=8.5.0,<9"] -# The pytest plugin (sphinx_test_reports.pytest_plugin); pytest is not -# a dependency of the extension or the converter. 7.0 is where everything the -# plugin uses exists (pytest.Config/Item/Mark, Config.stash, -# issue_config_time_warning). pytest before 7.3.2 does not run on Python 3.12 -# at all -- that floor is pytest's own, not this one's; the `toolchain-free` CI job runs -# the plugin's tests with no documentation toolchain installed at all. -pytest = ["pytest>=7.0"] +# Accepted and ignored until 4.0: 2.0.0 documented `pip install "sphinx-test-reports[sphinx]"` +# when the toolchain was an extra, and an install line that names a missing extra warns +# under both pip and uv. The toolchain is a dependency now +sphinx = [] +# A pass-through until 4.0: 2.0.0 documented `pip install "sphinx-test-reports[pytest]"` for +# the plugin, which is `ub-test-reports[pytest]` now. Kept so that line still installs +# pytest; check (4) wants the extra's specifier spelled with the same tight floor +pytest = ["ub-test-reports[pytest]>=1.0.0.dev0,<2"] # There is no `test` extra any more: the workspace's test tooling is the ROOT's shared # `test` dependency group, which every cell and the release compat cell install, and # `check_workspace.py` check (7) refuses a `[dependency-groups]` table here. `nox` and @@ -76,9 +81,6 @@ Repository = "https://github.com/useblocks/sphinx-needs" Issues = "https://github.com/useblocks/sphinx-needs/issues" Changelog = "https://github.com/useblocks/sphinx-needs/blob/master/packages/sphinx-test-reports/docs/changelog.rst" -[project.scripts] -test-reports = "sphinx_test_reports.cli:main" - # hatchling rather than flit, for this member alone. The wheel ships TWO top-level # packages: `sphinx_test_reports`, and `sphinxcontrib/test_reports/`, the aliases that keep # the pre-3.0 import name working until 4.0. flit ships exactly one module and silently @@ -86,7 +88,8 @@ test-reports = "sphinx_test_reports.cli:main" # extension would break on the first release. The `toolchain-free` CI job builds the sdist # and the wheel the way the release does (the wheel FROM the sdist) and fails when either # is missing a tracked file under `src/`, because an editable install reads `src/` and would -# keep every test green over a broken release. +# keep every test green over a broken release. (The job is named for ub-test-reports, whose +# suite it runs without Sphinx; this fence only reads archives, so it needs no toolchain.) [build-system] requires = ["hatchling >=1.27,<2"] build-backend = "hatchling.build" @@ -108,6 +111,5 @@ only-include = ["src/sphinx_test_reports", "src/sphinxcontrib"] # `check_workspace.Member.module`, `import_check.py` and `release.yaml` all derive it. # Everything else -- ruff, ty, pytest and the dependency groups -- is the ROOT's, and # `check_workspace.py` check (7) refuses those tables here: a member `[tool.ruff]` does not -# extend the root's configuration, it replaces it. The `toolchain` pytest marker this -# package declared moved to the root's `markers` list with the rest of the pytest -# configuration; `[tool.mypy]` went because the workspace type-checks with ty. +# extend the root's configuration, it replaces it. `[tool.mypy]` went because the workspace +# type-checks with ty. diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/__init__.py b/packages/sphinx-test-reports/src/sphinx_test_reports/__init__.py index b6fcb44f8..c3004ff79 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/__init__.py +++ b/packages/sphinx-test-reports/src/sphinx_test_reports/__init__.py @@ -1,18 +1,10 @@ -"""Sphinx-Test-Reports. +"""Sphinx-Test-Reports, the Sphinx extension. -``setup`` is resolved lazily (PEP 562) so that importing a submodule of this -package does not import Sphinx: the ``test-reports`` command and -:mod:`sphinx_test_reports.projectconfig` are used where the -documentation toolchain is not installed -- it is the ``sphinx`` extra of the -package, not a dependency -- and every import of a submodule runs this file -first. Sphinx still finds ``setup`` through normal attribute access when it -loads this package as an extension. - -Resolving ``setup`` is also where the toolchain is checked. An extra is opt-in, -so a project that installs the bare package into an environment already holding -an older Sphinx or sphinx-needs never shows pip the extra's version floors; -:mod:`sphinx_test_reports.toolchain` enforces them here instead, with -the install line in the message. +The converter, the pytest plugin, the parsers and the ``[test_reports]`` model are +``ub-test-reports`` (:mod:`ub_test_reports`), which this package depends on; what is here +is the extension that turns the same reports into needs inside a Sphinx build. Its +dependencies -- Sphinx, docutils, sphinx-needs and the core -- are hard requirements, so +``setup`` is imported eagerly. """ __all__ = ["__version__", "setup"] @@ -21,50 +13,6 @@ #: ``poe bump``; the extension's metadata reads it from here. __version__ = "2.0.0" - -def __getattr__(name: str) -> object: - if name != "setup": - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - from sphinx_test_reports.toolchain import INSTALL_HINT, unmet_requirements - - unmet = unmet_requirements() - if unmet: - # Before the import: an outdated sphinx-needs may well import and fail - # only later, inside a directive, with a traceback that does not say - # why. Sphinx fetches `setup` with getattr(), which only tolerates - # AttributeError, so the error reaches the user as it is raised here. - message = ( - f"Could not load extension {__name__}: {'; '.join(unmet)}. " - f"Install the Sphinx extension's dependencies with: {INSTALL_HINT}" - ) - try: - from sphinx.errors import ExtensionError - except ImportError: - raise ImportError(message) from None - raise ExtensionError(message) - - try: - from sphinx_test_reports.test_reports import setup - except ImportError as error: - # Sphinx wraps an ImportError from importing the *package* in a clean - # "Could not import extension" message, but fetches `setup` with - # getattr(), which only tolerates AttributeError. Resolving lazily - # would let a missing sphinx-needs escape as a raw traceback, so the - # message Sphinx would have produced is raised here instead -- when - # Sphinx is there to receive it -- naming the extra that installs the - # toolchain, the likely cause. The wrapped exception goes in the second - # argument only: ExtensionError.__str__ renders it as - # "(exception: ...)", so spelling it out in the message too would print - # it twice. - try: - from sphinx.errors import ExtensionError - except ImportError: - raise error from None - raise ExtensionError( - f"Could not import extension {__name__}; the Sphinx extension's " - f"dependencies are an extra, install them with: {INSTALL_HINT}", - error, - ) from error - - return setup +# The order is load-bearing: `test_reports` imports `__version__` from this package while +# this import runs, so the literal above has to be assigned before it. +from sphinx_test_reports.test_reports import setup diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_case.py b/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_case.py index 7bd4eff35..fd86db67d 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_case.py +++ b/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_case.py @@ -8,7 +8,7 @@ from sphinx_test_reports.config import DEFAULT_OPTIONS from sphinx_test_reports.directives.test_common import TestCommonDirective from sphinx_test_reports.exceptions import TestReportInvalidOptionError -from sphinx_test_reports.identity import split_case_name +from ub_test_reports.identity import split_case_name class TestCase(nodes.General, nodes.Element): diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_common.py b/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_common.py index a1dbc0e5e..f97130e06 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_common.py +++ b/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_common.py @@ -20,9 +20,9 @@ SphinxError, TestReportFileNotSetError, ) -from sphinx_test_reports.identity import deterministic_case_id -from sphinx_test_reports.jsonparser import JsonParser -from sphinx_test_reports.junitparser import JUnitParser +from ub_test_reports.identity import deterministic_case_id +from ub_test_reports.jsonparser import JsonParser +from ub_test_reports.junitparser import JUnitParser # fmt: on diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_results.py b/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_results.py index 8280a58ef..6ed5ee975 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_results.py +++ b/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_results.py @@ -3,7 +3,7 @@ from docutils import nodes from docutils.parsers.rst import Directive -from sphinx_test_reports.junitparser import JUnitParser +from ub_test_reports.junitparser import JUnitParser class TestResults(nodes.General, nodes.Element): diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/test_reports.py b/packages/sphinx-test-reports/src/sphinx_test_reports/test_reports.py index f9ea5bdf8..dc1f1410c 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/test_reports.py +++ b/packages/sphinx-test-reports/src/sphinx_test_reports/test_reports.py @@ -33,14 +33,14 @@ ) from sphinx_test_reports.environment import install_styles_static_files from sphinx_test_reports.exceptions import InvalidConfigurationError -from sphinx_test_reports.fields import ( +from sphinx_test_reports.functions import tr_link +from ub_test_reports.fields import ( FIELDS, RENAMEABLE_FIELDS, RESERVED_NAMES, declaration, ) -from sphinx_test_reports.functions import tr_link -from sphinx_test_reports.projectconfig import ( +from ub_test_reports.projectconfig import ( BRIDGE_KEYS, DEFAULT_FIELD_NAMES, DEFAULT_TOML_FILENAME, @@ -101,6 +101,7 @@ def setup(app: Sphinx) -> dict[str, object]: * test_env * test_report """ + app.require_sphinx((7, 4)) # Name of the need field carrying the path of the XML *report*. app.add_config_value("tr_file_option", DEFAULT_FIELD_NAMES["file_option"], "html") diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/toolchain.py b/packages/sphinx-test-reports/src/sphinx_test_reports/toolchain.py deleted file mode 100644 index e11f59baa..000000000 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/toolchain.py +++ /dev/null @@ -1,77 +0,0 @@ -"""The documentation toolchain the Sphinx extension needs, checked at load time. - -Sphinx and sphinx-needs are the ``sphinx`` extra of this package rather than -dependencies: the ``test-reports`` command runs where they are not installed. -An extra is opt-in, so nothing shows pip the extra's version floors when a -project installs the bare package into an environment that already holds an -older toolchain -- the old versions stay, and the extension would fail later, -inside a directive, with a traceback that does not say why. - -:func:`unmet_requirements` compares the installed toolchain against the floors -the extra declares, read from this package's own metadata so that they live in -``pyproject.toml`` alone. The lazy ``setup`` in the package ``__init__`` calls -it before the extension is imported and raises Sphinx's ``ExtensionError`` -with the install line. -""" - -from __future__ import annotations - -from importlib.metadata import PackageNotFoundError, requires, version -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from packaging.requirements import Requirement - -#: The distribution whose metadata declares the toolchain. -DISTRIBUTION = "sphinx-test-reports" -#: The extra holding the Sphinx extension's dependencies. -EXTRA = "sphinx" -#: The install line named by every message about a missing or outdated toolchain. -INSTALL_HINT = f'pip install "{DISTRIBUTION}[{EXTRA}]"' - - -def toolchain_requirements() -> list[Requirement]: - """The requirements the extra declares, read from the installed metadata. - - Empty without metadata (the package imported from a source tree on - ``sys.path``) and without ``packaging``, which parses them: it is not a - dependency of this package but arrives with Sphinx (and with pytest), so it - is there wherever the extension is loaded. - """ - try: - from packaging.requirements import Requirement - except ImportError: - return [] - try: - declared = requires(DISTRIBUTION) or [] - except PackageNotFoundError: - return [] - requirements = [Requirement(spec) for spec in declared] - return [ - requirement - for requirement in requirements - if requirement.marker is not None - and requirement.marker.evaluate({"extra": EXTRA}) - ] - - -def unmet_requirements() -> list[str]: - """One sentence per requirement of the extra the environment falls short of. - - A requirement whose distribution is not installed at all is not reported: - the import that fails on it states the problem more precisely, and a - toolchain importable from a source tree without metadata must not be - refused on the strength of the metadata alone. - """ - unmet: list[str] = [] - for requirement in toolchain_requirements(): - try: - installed = version(requirement.name) - except PackageNotFoundError: - continue - if not requirement.specifier.contains(installed, prereleases=True): - unmet.append( - f"{requirement.name} {installed} is installed, " - f"but {requirement.name}{requirement.specifier} is required" - ) - return unmet diff --git a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/__init__.py b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/__init__.py index 813a816a9..34f264df4 100644 --- a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/__init__.py +++ b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/__init__.py @@ -1,15 +1,17 @@ """The pre-3.0 name of sphinx-test-reports, kept working until 4.0. -The package is :mod:`sphinx_test_reports` now. Four old names still resolve, each -with a warning that names the new one: +The extension is :mod:`sphinx_test_reports` now, and the parsers and the pytest plugin are +:mod:`ub_test_reports`, the ``ub-test-reports`` distribution it depends on. Four old names +still resolve, each with a warning that names the new one: * this package, as a Sphinx extension (``extensions = ["sphinxcontrib.test_reports"]``), which warns through Sphinx's logger, type ``test_reports.deprecated`` -- the channel a documentation build shows, fails under ``-W`` and silences with ``suppress_warnings``; * ``pytest_plugin``, ``junitparser`` and ``jsonparser``, one file each next to this - one, which put the REAL module into :data:`sys.modules` under the old name and raise - one :class:`FutureWarning` per process (not a :class:`DeprecationWarning`, which - Python's default filters hide outside ``__main__``). It is attributed to the + one, which put the REAL module (``ub_test_reports.``) into :data:`sys.modules` + under the old name and raise one :class:`FutureWarning` per process (not a + :class:`DeprecationWarning`, which Python's default filters hide outside + ``__main__``). It is attributed to the ``import`` statement that names the module; ``importlib.import_module``, and pytest when it loads a ``-p`` or ``pytest_plugins`` name, are frames of their own and take the attribution instead. @@ -18,9 +20,9 @@ error: there is deliberately no finder here that would alias the rest. ``setup`` is resolved lazily (PEP 562), so importing one of the module aliases does -not import Sphinx: ``junitparser`` and ``pytest_plugin`` are used where the -documentation toolchain is not installed, and every import of a submodule runs this -file first. +not import Sphinx -- every import of a submodule runs this file first, and a test run +that loads ``-p sphinxcontrib.test_reports.pytest_plugin`` has no use for a Sphinx +import. """ from __future__ import annotations @@ -52,10 +54,10 @@ def _setup(app: Sphinx) -> dict[str, Any]: type="test_reports", subtype="deprecated", ) - # After the warning, so a toolchain error from the real `setup` has the deprecation - # line above it. Through Sphinx rather than by calling the real `setup`: a conf.py - # that lists both names then registers the extension once, and the real package's - # own lazy `setup` -- with its toolchain check -- is what runs. + # After the warning, so an error from the real `setup` has the deprecation line above + # it. Through Sphinx rather than by calling the real `setup`: a conf.py that lists both + # names then registers the extension once, and the real package's own `setup` is what + # runs. app.setup_extension(_NEW_NAME) extension = app.extensions[_NEW_NAME] # `Extension` pops these three out of the metadata it keeps; hand back all of it. diff --git a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/jsonparser.py b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/jsonparser.py index fde372f3f..c2ed4ebd4 100644 --- a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/jsonparser.py +++ b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/jsonparser.py @@ -1,4 +1,4 @@ -"""Deprecated: the old name of :mod:`sphinx_test_reports.jsonparser`, removed in 4.0. +"""Deprecated: the old name of :mod:`ub_test_reports.jsonparser`, removed in 4.0. Importing it gives the real module -- the same object, so classes and ``mock.patch`` targets written against the old path keep working, and warning filters naming it keep @@ -12,10 +12,10 @@ import sys import warnings -from sphinx_test_reports import jsonparser as _module +from ub_test_reports import jsonparser as _module warnings.warn( - "sphinxcontrib.test_reports.jsonparser has moved to sphinx_test_reports.jsonparser; " + "sphinxcontrib.test_reports.jsonparser has moved to ub_test_reports.jsonparser; " "import it from there. The old name stops working in sphinx-test-reports 4.0.", FutureWarning, # 2 is the frame that ran the import: `warnings` skips importlib's frozen bootstrap diff --git a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/junitparser.py b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/junitparser.py index 5ed5e4502..a9776e5fe 100644 --- a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/junitparser.py +++ b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/junitparser.py @@ -1,4 +1,4 @@ -"""Deprecated: the old name of :mod:`sphinx_test_reports.junitparser`, removed in 4.0. +"""Deprecated: the old name of :mod:`ub_test_reports.junitparser`, removed in 4.0. Importing it gives the real module -- the same object, so classes and ``mock.patch`` targets written against the old path keep working, and warning filters naming it keep @@ -12,10 +12,10 @@ import sys import warnings -from sphinx_test_reports import junitparser as _module +from ub_test_reports import junitparser as _module warnings.warn( - "sphinxcontrib.test_reports.junitparser has moved to sphinx_test_reports.junitparser; " + "sphinxcontrib.test_reports.junitparser has moved to ub_test_reports.junitparser; " "import it from there. The old name stops working in sphinx-test-reports 4.0.", FutureWarning, # 2 is the frame that ran the import: `warnings` skips importlib's frozen bootstrap diff --git a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/pytest_plugin.py b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/pytest_plugin.py index 2fe691d05..e1fdf1f68 100644 --- a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/pytest_plugin.py +++ b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/pytest_plugin.py @@ -1,4 +1,4 @@ -"""Deprecated: the old name of :mod:`sphinx_test_reports.pytest_plugin`, removed in 4.0. +"""Deprecated: the old name of :mod:`ub_test_reports.pytest_plugin`, removed in 4.0. Importing it gives the real module -- the same object, so classes and ``mock.patch`` targets written against the old path keep working, and warning filters naming it keep @@ -12,10 +12,10 @@ import sys import warnings -from sphinx_test_reports import pytest_plugin as _module +from ub_test_reports import pytest_plugin as _module warnings.warn( - "sphinxcontrib.test_reports.pytest_plugin has moved to sphinx_test_reports.pytest_plugin; " + "sphinxcontrib.test_reports.pytest_plugin has moved to ub_test_reports.pytest_plugin; " "import it from there. The old name stops working in sphinx-test-reports 4.0.", FutureWarning, # 2 is the frame that ran the import: `warnings` skips importlib's frozen bootstrap diff --git a/packages/ub-test-reports/AGENTS.md b/packages/ub-test-reports/AGENTS.md new file mode 100644 index 000000000..c77c71141 --- /dev/null +++ b/packages/ub-test-reports/AGENTS.md @@ -0,0 +1,52 @@ +# AGENTS.md — packages/ub-test-reports + +The delta for this package. Everything repository-level — the workspace layout, the +commands, the lock, lint/format/type-check configuration, the release recipe, the pull +request requirements — is in the ROOT [`AGENTS.md`](../../AGENTS.md), and this file does +not repeat it. + +## What it is + +The Sphinx-free core of sphinx-test-reports: the report parsers, the result vocabulary, +the deterministic case IDs, the `[test_reports]` model of `ubproject.toml`, the +`test-reports` converter that writes a `needs.json`, and the pytest plugin. It is a +**tool, not a Sphinx extension** — nothing to add to `conf.py`, no `Framework :: Sphinx` +classifier. sphinx-test-reports, the extension, depends on it and turns the same reports +into needs inside a build; the dependency never runs the other way. Its documentation is +the "Without Sphinx" section of sphinx-test-reports' site; it has no site of its own. + +## Commands + +```bash +uv run poe test-ub-test-reports # the suite (no sphinx axis: it has no Sphinx) +uv run poe import-check-ub-test-reports # import every module from the built wheel +uv run poe build-ub-test-reports # sdist + wheel into dist/ub-test-reports +``` + +## Rules + +- **Nothing here may import Sphinx, sphinx-needs or docutils** — not at module level and + not in a function body, because the converter runs as a build action and the plugin + inside a test run, and neither has a documentation toolchain. A default `.venv` cannot + catch a violation: every environment the workspace root produces has Sphinx in it. CI's + `toolchain-free` job is the fence — it installs the built wheel where the toolchain is + absent and runs the whole suite there. +- **`ub-project` is its `ubproject.toml` reader.** Finding, loading and anchoring the file + come from there; what stays here is the `[test_reports]` policy and `TomlConfigError`. +- **The `[test_reports]` model is a parity surface**: ubCode reads the same table and is + held to the same behaviour. A behaviour change in it — keys, types, normalisation, + defaults — says so in the changelog, so ubCode can follow. +- **The pytest plugin is opt-in**: `-p ub_test_reports.pytest_plugin`, and no `pytest11` + entry point — an auto-loaded plugin would change every pytest run in any environment that + merely has this package installed, the extension's users included. The `pytest` extra's + floor is fenced by CI's `plugin-floor` job, which runs the suite on the oldest pytest of + each Python it names. +- **The wire names `sphinxcontrib.test_reports:file` and `sphinxcontrib.test_reports:line` + must not change.** They are the documented `record_property` names the plugin reserves, + not an import path, and reports written by older plugins carry them. +- **The test fixtures are this package's own copies** (`tests/fixtures/`), so that its + suite reads nothing from another member's tree and runs from the installed wheel; some of + them also exist under sphinx-test-reports' `tests/doc_test/utils/`, where its test + projects and docs read them. Neither copy ships in an sdist. +- **Tests build paths with `Path`** and read and write text with an explicit `encoding`, so + that the suite holds on Windows too. diff --git a/packages/ub-test-reports/CLAUDE.md b/packages/ub-test-reports/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/packages/ub-test-reports/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/packages/ub-test-reports/LICENSE b/packages/ub-test-reports/LICENSE new file mode 100644 index 000000000..616fe15fc --- /dev/null +++ b/packages/ub-test-reports/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 useblocks + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/ub-test-reports/README.rst b/packages/ub-test-reports/README.rst new file mode 100644 index 000000000..a8dd79ab3 --- /dev/null +++ b/packages/ub-test-reports/README.rst @@ -0,0 +1,32 @@ +ub-test-reports +=============== + +A tool for the sphinx-needs family -- not a Sphinx extension, and nothing to add to +``conf.py``. + +It turns test results into needs without running Sphinx, and holds everything about test +reports that does not need a documentation build: + +- **the parsers** -- JUnit XML (pytest, googletest, ctest, nose and friends) and + tox-envreport style JSON; +- **the result vocabulary** -- one spelling for every outcome, whichever report it came + from; +- **deterministic case IDs** -- the same case gets the same need ID in every run; +- **the** ``[test_reports]`` **model** of ``ubproject.toml``, read through ``ub-project``; +- **the converter** -- the ``test-reports build needs`` command, which writes a + ``needs.json`` that sphinx-needs can import; +- **the pytest plugin** -- ``-p ub_test_reports.pytest_plugin``, which shapes the JUnit XML + pytest writes (source locations, per-case properties for traceability). + +Install the converter with:: + + pip install ub-test-reports + +and the pytest plugin with:: + + pip install "ub-test-reports[pytest]" + +To show the same reports inside a Sphinx documentation build, install the extension, +``sphinx-test-reports``, which depends on this package. + +Documentation: the "Without Sphinx" section of https://sphinx-test-reports.readthedocs.io/en/latest/ diff --git a/packages/ub-test-reports/docs/changelog.rst b/packages/ub-test-reports/docs/changelog.rst new file mode 100644 index 000000000..be7050f07 --- /dev/null +++ b/packages/ub-test-reports/docs/changelog.rst @@ -0,0 +1,12 @@ +.. _changelog: + +Changelog +========= + +Unreleased +---------- + +- 1.0.0 is the first release: the converter, the pytest plugin, the parsers, the result + vocabulary, the deterministic IDs and the ``[test_reports]`` model, moved out of + sphinx-test-reports 2.0.0 unchanged; the plugin is ``-p ub_test_reports.pytest_plugin``; + its wire names are unchanged. diff --git a/packages/ub-test-reports/pyproject.toml b/packages/ub-test-reports/pyproject.toml new file mode 100644 index 000000000..10f7a8fb4 --- /dev/null +++ b/packages/ub-test-reports/pyproject.toml @@ -0,0 +1,60 @@ +[project] +name = "ub-test-reports" +version = "1.0.0.dev0" +description = "Test results as needs without Sphinx: report parsers, the needs.json converter and the pytest plugin" +authors = [{ name = "team useblocks", email = "info@useblocks.com" }] +license = { file = "LICENSE" } +readme = "README.rst" +requires-python = ">=3.11,<4" +# NO Sphinx, docutils or sphinx-needs, and that is the package's whole contract: the +# converter runs as a build action and the plugin inside a test run, neither of which has a +# documentation toolchain. ub-project is the `ubproject.toml` reader, TIGHT-tracked like +# every intra-workspace edge (`check_workspace.py` check (4)); `propagate_floors.py` moves +# the floor at each ub-project release. The `toolchain-free` CI job installs the built wheel +# where Sphinx is absent and runs the whole suite there +dependencies = ["lxml", "ub-project>=1.1.0,<2"] +keywords = ["test-reports", "junit", "needs.json", "sphinx-needs", "pytest"] +# NOT `Framework :: Sphinx`: nothing here depends on Sphinx +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Framework :: Pytest", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Software Development :: Testing", + "Typing :: Typed", +] + +[project.optional-dependencies] +# The pytest plugin (`-p ub_test_reports.pytest_plugin`); pytest is not a dependency of the +# converter. 7.0 is where everything the plugin uses exists (pytest.Config/Item/Mark, +# Config.stash, issue_config_time_warning). pytest before 7.3.2 does not run on Python +# 3.12 at all -- that floor is pytest's own, not this one's. The `plugin-floor` CI job runs +# the whole suite on pytest 7.0.1 (Python 3.11) and 7.3.2 (Python 3.12). +pytest = ["pytest>=7.0"] + +[project.scripts] +test-reports = "ub_test_reports.cli:main" + +[project.urls] +Documentation = "https://sphinx-test-reports.readthedocs.io/en/latest/" +Repository = "https://github.com/useblocks/sphinx-needs" +Issues = "https://github.com/useblocks/sphinx-needs/issues" +Changelog = "https://github.com/useblocks/sphinx-needs/blob/master/packages/ub-test-reports/docs/changelog.rst" + +[build-system] +requires = ["flit_core >=3.4,<5"] +build-backend = "flit_core.buildapi" + +# Deliberately NO `[tool.flit.sdist]`: flit's default sdist is the module (with its +# `py.typed` marker and `schemas/JUnit.xsd`), README, LICENSE and this file, and nothing +# else -- so neither `tests/` (and its fixture copies) nor `docs/` ships. Everything else -- +# ruff, ty, pytest and the dependency groups -- is the ROOT's, and +# `tools/src/sn_tools/check_workspace.py` refuses those tables here. diff --git a/packages/ub-test-reports/src/ub_test_reports/__init__.py b/packages/ub-test-reports/src/ub_test_reports/__init__.py new file mode 100644 index 000000000..252e7bdaa --- /dev/null +++ b/packages/ub-test-reports/src/ub_test_reports/__init__.py @@ -0,0 +1,17 @@ +"""ub-test-reports: test results as needs, without Sphinx. + +The report parsers, the result vocabulary, the deterministic case IDs, the +``[test_reports]`` model of ``ubproject.toml``, the needs.json converter behind the +``test-reports`` command, and the pytest plugin (``-p ub_test_reports.pytest_plugin``). +The modules are the API; this file only carries the version. + +**Nothing in this package may import Sphinx, sphinx-needs or docutils** -- not at module +level and not in a function body. sphinx-test-reports, the Sphinx extension, depends on +this package, never the other way round. +""" + +__all__ = ["__version__"] + +#: Checked against ``[project] version`` by ``check_workspace.py`` and stamped by +#: ``poe bump``. +__version__ = "1.0.0.dev0" diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/cli.py b/packages/ub-test-reports/src/ub_test_reports/cli.py similarity index 99% rename from packages/sphinx-test-reports/src/sphinx_test_reports/cli.py rename to packages/ub-test-reports/src/ub_test_reports/cli.py index 0130b4cef..37c6ec8d8 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/cli.py +++ b/packages/ub-test-reports/src/ub_test_reports/cli.py @@ -19,15 +19,15 @@ from collections.abc import Mapping, Sequence from pathlib import Path -from sphinx_test_reports.junitparser import JUnitParser -from sphinx_test_reports.needs_export import ( +from ub_test_reports.junitparser import JUnitParser +from ub_test_reports.needs_export import ( DEFAULT_VERSION, Report, build_needs_file, iter_cases, optional, ) -from sphinx_test_reports.projectconfig import ( +from ub_test_reports.projectconfig import ( CONVERSION_KEYS, DEFAULT_NEED_TYPE, DEFAULT_TOML_FILENAME, @@ -40,7 +40,7 @@ load_project_config, needs_settings, ) -from sphinx_test_reports.remote import ( +from ub_test_reports.remote import ( DEFAULT_URL_PATTERN, check_url_pattern, normalise_remote_url, diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/fields.py b/packages/ub-test-reports/src/ub_test_reports/fields.py similarity index 98% rename from packages/sphinx-test-reports/src/sphinx_test_reports/fields.py rename to packages/ub-test-reports/src/ub_test_reports/fields.py index a823ccaa6..08bbb0d67 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/fields.py +++ b/packages/ub-test-reports/src/ub_test_reports/fields.py @@ -18,7 +18,7 @@ from collections.abc import Iterable, Mapping -from sphinx_test_reports.results import CANONICAL_RESULTS +from ub_test_reports.results import CANONICAL_RESULTS #: ``name -> (JSON type, description)`` for every field declared under a fixed #: name. The build registers all of them, on test-file, test-suite and @@ -56,7 +56,7 @@ #: ``role -> (JSON type, description)`` for the fields whose *name* the #: configuration chooses; the roles are the keys of -#: :data:`~sphinx_test_reports.projectconfig.DEFAULT_FIELD_NAMES`. +#: :data:`~ub_test_reports.projectconfig.DEFAULT_FIELD_NAMES`. #: Keying them by role rather than by name is what lets the description follow #: a renamed field. RENAMEABLE_FIELDS: dict[str, tuple[str, str]] = { diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/identity.py b/packages/ub-test-reports/src/ub_test_reports/identity.py similarity index 100% rename from packages/sphinx-test-reports/src/sphinx_test_reports/identity.py rename to packages/ub-test-reports/src/ub_test_reports/identity.py diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/jsonparser.py b/packages/ub-test-reports/src/ub_test_reports/jsonparser.py similarity index 98% rename from packages/sphinx-test-reports/src/sphinx_test_reports/jsonparser.py rename to packages/ub-test-reports/src/ub_test_reports/jsonparser.py index 09262c781..7e1dbbc05 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/jsonparser.py +++ b/packages/ub-test-reports/src/ub_test_reports/jsonparser.py @@ -12,7 +12,7 @@ from functools import reduce from typing import Any -from sphinx_test_reports.results import normalize_result +from ub_test_reports.results import normalize_result def dict_get(root, items, default=None): diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/junitparser.py b/packages/ub-test-reports/src/ub_test_reports/junitparser.py similarity index 98% rename from packages/sphinx-test-reports/src/sphinx_test_reports/junitparser.py rename to packages/ub-test-reports/src/ub_test_reports/junitparser.py index c42633df2..fa1ac668e 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/junitparser.py +++ b/packages/ub-test-reports/src/ub_test_reports/junitparser.py @@ -6,7 +6,7 @@ from lxml import etree, objectify # ty: ignore[unresolved-import] -from sphinx_test_reports.results import normalize_result +from ub_test_reports.results import normalize_result #: Attributes the JUnit/googletest dialects define themselves. Every *other* #: attribute is a ``RecordProperty`` value in attribute form: googletest wrote @@ -50,7 +50,7 @@ #: ```` children carrying a result, in the precedence order used to #: classify a case that has more than one kind of them. These are XML element #: names, not ``result`` values -- ```` is read as the result -#: ``failed`` (see :mod:`sphinx_test_reports.results`). +#: ``failed`` (see :mod:`ub_test_reports.results`). RESULT_PART_KINDS = ("skipped", "failure", "error") diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/needs_export.py b/packages/ub-test-reports/src/ub_test_reports/needs_export.py similarity index 97% rename from packages/sphinx-test-reports/src/sphinx_test_reports/needs_export.py rename to packages/ub-test-reports/src/ub_test_reports/needs_export.py index 64bad4ae2..ee1a17b2b 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/needs_export.py +++ b/packages/ub-test-reports/src/ub_test_reports/needs_export.py @@ -14,7 +14,7 @@ also writes into the files it produces itself, so the type of a field is readable from the artifact instead of only from a Sphinx build with this extension loaded. The declarations come from - :mod:`sphinx_test_reports.fields`, the same table the extension + :mod:`ub_test_reports.fields`, the same table the extension registers its fields from. * **Nothing depends on the wall clock or on dict ordering**, so the file is a cacheable build artifact and a diffable piece of evidence. @@ -24,14 +24,14 @@ import textwrap from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence -from sphinx_test_reports.fields import case_needs_schema -from sphinx_test_reports.identity import ( +from ub_test_reports.fields import case_needs_schema +from ub_test_reports.identity import ( UNKNOWN, deterministic_case_id, split_case_name, ) -from sphinx_test_reports.projectconfig import DEFAULT_FIELD_NAMES -from sphinx_test_reports.remote import DEFAULT_URL_PATTERN, source_url +from ub_test_reports.projectconfig import DEFAULT_FIELD_NAMES +from ub_test_reports.remote import DEFAULT_URL_PATTERN, source_url #: A need field value as it appears in needs.json. ``None`` is the value of an #: exported property the case does not carry, as the build leaves it. @@ -178,7 +178,7 @@ def build_need( :data:`DEFAULT_FIELD_NAMES`) -- and so are the values: the title is the case name and ``result`` keeps the parser's spelling (``failed``, which is also a documented field value and the ``tr_failed`` CSS class, see - :mod:`sphinx_test_reports.results`), so that a need imported from + :mod:`ub_test_reports.results`), so that a need imported from the produced ``needs.json`` and one created locally from the same report are indistinguishable to a schema, a filter or a ``needtable``. diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/projectconfig.py b/packages/ub-test-reports/src/ub_test_reports/projectconfig.py similarity index 99% rename from packages/sphinx-test-reports/src/sphinx_test_reports/projectconfig.py rename to packages/ub-test-reports/src/ub_test_reports/projectconfig.py index b455fd92b..f73ab2a71 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/projectconfig.py +++ b/packages/ub-test-reports/src/ub_test_reports/projectconfig.py @@ -47,12 +47,12 @@ from pathlib import Path from typing import NoReturn -from sphinx_test_reports.fields import RESERVED_NAMES from ub_project import DEFAULT_FILENAME, ProjectConfigError, anchor, load_toml # Re-exported: the Sphinx bridge, the converter and the tests import the walk # from here. It raises nothing of its own, so it needs no wrapper. from ub_project import find_project_config as find_project_config +from ub_test_reports.fields import RESERVED_NAMES #: Default file the configuration is read from. Looked up by walking up from #: the ``confdir`` (Sphinx) or the working directory (a converter); see diff --git a/packages/ub-test-reports/src/ub_test_reports/py.typed b/packages/ub-test-reports/src/ub_test_reports/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/pytest_plugin.py b/packages/ub-test-reports/src/ub_test_reports/pytest_plugin.py similarity index 98% rename from packages/sphinx-test-reports/src/sphinx_test_reports/pytest_plugin.py rename to packages/ub-test-reports/src/ub_test_reports/pytest_plugin.py index 2302793e1..ea432e33f 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/pytest_plugin.py +++ b/packages/ub-test-reports/src/ub_test_reports/pytest_plugin.py @@ -1,6 +1,6 @@ """pytest plugin: shape the JUnit XML the way this extension reads it. -Enable it with ``-p sphinx_test_reports.pytest_plugin`` (or in +Enable it with ``-p ub_test_reports.pytest_plugin`` (or in ``addopts``) and write the report with ``--junitxml`` under ``junit_family = xunit1``. Two things then happen to every ````: @@ -92,7 +92,7 @@ } #: Registration name of the per-session hook object, :class:`_XmlShape`. -_HOOKS = "sphinx_test_reports.xml_shape" +_HOOKS = "ub_test_reports.xml_shape" Recorder = Callable[[str, str], None] @@ -357,7 +357,7 @@ class TestReportsConfigWarning(pytest.PytestWarning): Issued once at start-up. Where the project turns warnings into errors (``filterwarnings = error``, ``-W error``) it becomes a clean usage error rather than a traceback; - ``ignore::sphinx_test_reports.pytest_plugin.TestReportsConfigWarning`` + ``ignore::ub_test_reports.pytest_plugin.TestReportsConfigWarning`` silences it. """ @@ -376,7 +376,7 @@ def _report_family(config: pytest.Config) -> str | None: def _notify(config: pytest.Config, message: str) -> None: """Issue *message* as :class:`TestReportsConfigWarning` at configure time.""" - warning = TestReportsConfigWarning(f"sphinx_test_reports.pytest_plugin: {message}") + warning = TestReportsConfigWarning(f"ub_test_reports.pytest_plugin: {message}") try: config.issue_config_time_warning(warning, stacklevel=3) except TestReportsConfigWarning as error: @@ -401,7 +401,7 @@ def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line( "markers", f"{MARKER}(properties): properties written to the JUnit XML of the " - "test; attached by sphinx_test_reports.pytest_plugin.add_test_properties", + "test; attached by ub_test_reports.pytest_plugin.add_test_properties", ) lines: Sequence[str] = config.getini(OPTION) try: diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/remote.py b/packages/ub-test-reports/src/ub_test_reports/remote.py similarity index 100% rename from packages/sphinx-test-reports/src/sphinx_test_reports/remote.py rename to packages/ub-test-reports/src/ub_test_reports/remote.py diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/results.py b/packages/ub-test-reports/src/ub_test_reports/results.py similarity index 95% rename from packages/sphinx-test-reports/src/sphinx_test_reports/results.py rename to packages/ub-test-reports/src/ub_test_reports/results.py index dd99cc2bd..6b64a18d9 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/results.py +++ b/packages/ub-test-reports/src/ub_test_reports/results.py @@ -10,7 +10,7 @@ for googletest's ``status="notrun"``). The JSON parser passed its report's own value through untouched. So the same outcome could arrive under two spellings, and a single test case was ``failure`` while the count of them on its suite was -``failed`` (:data:`~sphinx_test_reports.fields.FIELDS`). +``failed`` (:data:`~ub_test_reports.fields.FIELDS`). ``error`` stays a noun-state deliberately: it agrees with the ``errors`` count beside it and with pytest, which spells that outcome ``error`` too. The pair @@ -29,7 +29,7 @@ #: consumers that never load this extension. #: #: A tuple, in the order a reader wants them rather than alphabetically: -#: :func:`~sphinx_test_reports.fields.declaration` renders it into the +#: :func:`~ub_test_reports.fields.declaration` renders it into the #: declared description of the field, and the converter's output has to be #: byte-stable, which a set's iteration order is not. CANONICAL_RESULTS: tuple[str, ...] = ( diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/schemas/JUnit.xsd b/packages/ub-test-reports/src/ub_test_reports/schemas/JUnit.xsd similarity index 100% rename from packages/sphinx-test-reports/src/sphinx_test_reports/schemas/JUnit.xsd rename to packages/ub-test-reports/src/ub_test_reports/schemas/JUnit.xsd diff --git a/pyproject.toml b/pyproject.toml index 89fcd2068..2f77cddf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "sphinx-mounts", "sphinx-codelinks", "sphinx-test-reports", + "ub-test-reports", "ub-project", "sphinx-needs-workspace-tools", ] @@ -68,6 +69,7 @@ sphinx-needs = { workspace = true } sphinx-mounts = { workspace = true } sphinx-codelinks = { workspace = true } sphinx-test-reports = { workspace = true } +ub-test-reports = { workspace = true } ub-project = { workspace = true } sphinx-needs-workspace-tools = { workspace = true } sphinx-needs-testkit = { workspace = true } @@ -257,11 +259,6 @@ markers = [ "jstest: marks tests as JavaScript test (deselect with '-m \"not jstest\"')", "benchmark: marks tests as expensive benchmark test (deselect with '-m \"not benchmark\"')", "bazel: tests that require a Bazel installation on PATH (deselect with '-m \"not bazel\"')", - # sphinx-test-reports', relocated from that member's own `[tool.pytest.ini_options]`, - # which check (7) refuses here: 8 sites / 25 tests carry it, and without the entry every - # one of them raises `PytestUnknownMarkWarning`. The `toolchain-free` CI job is what - # deselects them, in an environment with no Sphinx at all - "toolchain: needs the documentation toolchain (Sphinx, sphinx-needs) installed (deselect with '-m \"not toolchain\"')", ] filterwarnings = [ "ignore:.*removed in Python 3.14.*:DeprecationWarning", @@ -284,6 +281,8 @@ src = [ "packages/sphinx-codelinks/src", "packages/sphinx-test-reports", "packages/sphinx-test-reports/src", + "packages/ub-test-reports", + "packages/ub-test-reports/src", "packages/sphinx-needs-testkit", "packages/sphinx-needs-testkit/src", "packages/ub-project", @@ -354,6 +353,7 @@ include = [ "packages/sphinx-mounts/src", "packages/sphinx-codelinks/src", "packages/sphinx-test-reports/src", + "packages/ub-test-reports/src", "packages/ub-project/src", "tools/src", ".github/scripts", @@ -382,7 +382,7 @@ allowed-unresolved-imports = [ # sphinx-codelinks' optional libclang engine: `clang.cindex` ships no py.typed and sits # behind the `libclang` extra at runtime, so the typing floor deliberately lacks it "clang.**", - # sphinx-test-reports' pytest plugin. The `typing` environment deliberately has no + # ub-test-reports' pytest plugin. The `typing` environment deliberately has no # pytest -- it pins the oldest supported sphinx and docutils and nothing else it does # not need -- so the plugin's own imports cannot resolve there. The plugin IS type-checked # otherwise; what is unresolved is the runner it plugs into, which is exactly the shape @@ -712,6 +712,18 @@ help = "Run the ub-project suite; trailing args go to pytest" cmd = "pytest" cwd = "packages/ub-project" +# ub-test-reports, the Sphinx-free core of sphinx-test-reports. Like ub-project's, the +# short name keeps the whole distribution name (the naming rule drops only a `sphinx-` +# prefix), and there are no `-sphinxN` cells: the package has no Sphinx, and the environment +# that matters is the one WITHOUT it -- ci.yaml's `toolchain-free` job, which runs this +# suite there from the built wheel. No path is named: nothing under its `src/` is called +# `test_*.py`, unlike the extension's, so `poe test-ub-test-reports tests/test_cli_config.py` +# narrows the run rather than adding to it +[tool.poe.tasks.test-ub-test-reports] +help = "Run the ub-test-reports suite; trailing args go to pytest" +cmd = "pytest" +cwd = "packages/ub-test-reports" + # BARE, not `typecheck-needs`: the naming rule suffixes a task that acts on ONE package, # and this one checks the whole repository -- both packages plus the tooling -- out of a # single `[tool.ty]` configuration and a single environment. Splitting it per package would @@ -723,7 +735,7 @@ help = "Type-check every package with ty against the oldest supported sphinx (th # but it is NOT a fence on `src.include` above: ty intersects the two, so a stale `include` # makes this check zero files and exit 0. The canary step in ci.yaml's Lint job is what # guards that. -cmd = "ty check packages/sphinx-needs/src/sphinx_needs packages/sphinx-mounts/src/sphinx_mounts packages/sphinx-codelinks/src/sphinx_codelinks packages/sphinx-test-reports/src packages/ub-project/src/ub_project tools/src .github/scripts scripts --python .venvs/typing" +cmd = "ty check packages/sphinx-needs/src/sphinx_needs packages/sphinx-mounts/src/sphinx_mounts packages/sphinx-codelinks/src/sphinx_codelinks packages/sphinx-test-reports/src packages/ub-test-reports/src/ub_test_reports packages/ub-project/src/ub_project tools/src .github/scripts scripts --python .venvs/typing" executor = { type = "uv", frozen = true, no-group = "dev", group = ["typing"] } env = { UV_PROJECT_ENVIRONMENT = ".venvs/typing" } @@ -847,14 +859,22 @@ cmd = "uv build --package sphinx-test-reports --no-sources -o dist/sphinx-test-r [tool.poe.tasks.import-check-reports] help = "Import every sphinx-test-reports module from its built wheel, with dependencies as PUBLISHED" -# `--extra sphinx`, because this is the first member in the workspace whose RUNTIME -# dependencies are optional: the bare wheel's `Requires-Dist` is `lxml` and `ub-project`, and -# `sphinx_test_reports.test_reports` imports sphinx-needs. Without the extra the walk -# fails on every module that imports sphinx or docutils (the directives, the bridge, -# the environment and the exceptions). `--extra pytest` too, for `pytest_plugin`, which imports -# pytest and pluggy. The release workflow's compat cell gets the toolchain from this -# member's `compat-requirements.txt`, and pytest from the root `test` group it installs -cmd = "python tools/src/sn_tools/import_check.py sphinx-test-reports --extra sphinx --extra pytest" +# No extras: the wheel's `Requires-Dist` is the toolchain (sphinx, docutils, sphinx-needs) +# and the core, `ub-test-reports`, all hard. The walk covers `sphinx_test_reports` only -- +# the converter and the plugin are the core's, checked by `import-check-ub-test-reports`. +# RED until ub-test-reports is on PyPI, by design: the resolution is the published one, and +# that is the release order (the core first, then this package) +cmd = "python tools/src/sn_tools/import_check.py sphinx-test-reports" + +[tool.poe.tasks.build-ub-test-reports] +help = "Build the ub-test-reports sdist + wheel into dist/ub-test-reports" +cmd = "uv build --package ub-test-reports --no-sources -o dist/ub-test-reports" + +[tool.poe.tasks.import-check-ub-test-reports] +help = "Import every ub_test_reports module from its built wheel, with dependencies as PUBLISHED" +# `--extra pytest` for `pytest_plugin`, which imports pytest and pluggy; nothing else in the +# wheel needs more than its `Requires-Dist` (lxml, ub-project) +cmd = "python tools/src/sn_tools/import_check.py ub-test-reports --extra pytest" [tool.poe.tasks.build-ub-project] help = "Build the ub-project sdist + wheel into dist/ub-project" diff --git a/uv.lock b/uv.lock index f40a46454..17fcb2f15 100644 --- a/uv.lock +++ b/uv.lock @@ -23,6 +23,7 @@ members = [ "sphinx-needs-workspace-tools", "sphinx-test-reports", "ub-project", + "ub-test-reports", ] [[package]] @@ -3000,6 +3001,7 @@ dependencies = [ { name = "sphinx-needs-workspace-tools" }, { name = "sphinx-test-reports" }, { name = "ub-project" }, + { name = "ub-test-reports" }, ] [package.optional-dependencies] @@ -3148,6 +3150,7 @@ requires-dist = [ { name = "sphinx-test-reports", editable = "packages/sphinx-test-reports" }, { name = "sphinx-test-reports", extras = ["docs"], marker = "extra == 'docs-reports'", editable = "packages/sphinx-test-reports" }, { name = "ub-project", editable = "packages/ub-project" }, + { name = "ub-test-reports", editable = "packages/ub-test-reports" }, ] provides-extras = ["plotting", "docs", "docs-mounts", "docs-codelinks", "docs-reports", "codelinks-libclang", "theme-im", "theme-furo", "theme-pds", "theme-rtd"] @@ -3287,8 +3290,13 @@ name = "sphinx-test-reports" version = "2.0.0" source = { editable = "packages/sphinx-test-reports" } dependencies = [ - { name = "lxml" }, - { name = "ub-project" }, + { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or extra == 'group-22-sphinx-needs-workspace-sphinx-7' or extra == 'group-22-sphinx-needs-workspace-sphinx-8' or extra == 'group-22-sphinx-needs-workspace-typing'" }, + { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (python_full_version >= '3.12' and extra != 'group-22-sphinx-needs-workspace-sphinx-7' and extra != 'group-22-sphinx-needs-workspace-sphinx-8' and extra != 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-9' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-8') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-typing')" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or extra == 'group-22-sphinx-needs-workspace-sphinx-7' or extra == 'group-22-sphinx-needs-workspace-typing' or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-sphinx-9')" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and extra != 'group-22-sphinx-needs-workspace-sphinx-7' and extra != 'group-22-sphinx-needs-workspace-sphinx-9' and extra != 'group-22-sphinx-needs-workspace-typing') or extra == 'group-22-sphinx-needs-workspace-sphinx-8' or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-9' and extra == 'group-22-sphinx-needs-workspace-typing')" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (python_full_version >= '3.12' and extra != 'group-22-sphinx-needs-workspace-sphinx-7' and extra != 'group-22-sphinx-needs-workspace-sphinx-8' and extra != 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-9' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-8') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-typing')" }, + { name = "sphinx-needs" }, + { name = "ub-test-reports" }, ] [package.optional-dependencies] @@ -3304,32 +3312,23 @@ docs = [ { name = "sphinxcontrib-plantuml" }, ] pytest = [ - { name = "pytest" }, -] -sphinx = [ - { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or extra == 'group-22-sphinx-needs-workspace-sphinx-7' or extra == 'group-22-sphinx-needs-workspace-sphinx-8' or extra == 'group-22-sphinx-needs-workspace-typing'" }, - { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (python_full_version >= '3.12' and extra != 'group-22-sphinx-needs-workspace-sphinx-7' and extra != 'group-22-sphinx-needs-workspace-sphinx-8' and extra != 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-9' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-8') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-typing')" }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or extra == 'group-22-sphinx-needs-workspace-sphinx-7' or extra == 'group-22-sphinx-needs-workspace-typing' or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-sphinx-9')" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and extra != 'group-22-sphinx-needs-workspace-sphinx-7' and extra != 'group-22-sphinx-needs-workspace-sphinx-9' and extra != 'group-22-sphinx-needs-workspace-typing') or extra == 'group-22-sphinx-needs-workspace-sphinx-8' or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-9' and extra == 'group-22-sphinx-needs-workspace-typing')" }, - { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (python_full_version >= '3.12' and extra != 'group-22-sphinx-needs-workspace-sphinx-7' and extra != 'group-22-sphinx-needs-workspace-sphinx-8' and extra != 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-9' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-8') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-typing')" }, - { name = "sphinx-needs" }, + { name = "ub-test-reports", extra = ["pytest"] }, ] [package.metadata] requires-dist = [ - { name = "docutils", marker = "extra == 'sphinx'", specifier = ">=0.21" }, - { name = "lxml" }, + { name = "docutils", specifier = ">=0.21" }, { name = "packaging", marker = "extra == 'docs'" }, { name = "pillow", marker = "extra == 'docs'" }, - { name = "pytest", marker = "extra == 'pytest'", specifier = ">=7.0" }, + { name = "sphinx", specifier = ">=7.4" }, { name = "sphinx", marker = "extra == 'docs'" }, - { name = "sphinx", marker = "extra == 'sphinx'", specifier = ">=7.4" }, { name = "sphinx-design", marker = "extra == 'docs'" }, { name = "sphinx-immaterial", marker = "extra == 'docs'" }, + { name = "sphinx-needs", editable = "packages/sphinx-needs" }, { name = "sphinx-needs", marker = "extra == 'docs'", editable = "packages/sphinx-needs" }, - { name = "sphinx-needs", marker = "extra == 'sphinx'", editable = "packages/sphinx-needs" }, { name = "sphinxcontrib-plantuml", marker = "extra == 'docs'" }, - { name = "ub-project", editable = "packages/ub-project" }, + { name = "ub-test-reports", editable = "packages/ub-test-reports" }, + { name = "ub-test-reports", extras = ["pytest"], marker = "extra == 'pytest'", editable = "packages/ub-test-reports" }, ] provides-extras = ["sphinx", "pytest", "docs"] @@ -3854,6 +3853,28 @@ name = "ub-project" version = "1.1.0" source = { editable = "packages/ub-project" } +[[package]] +name = "ub-test-reports" +version = "1.0.0.dev0" +source = { editable = "packages/ub-test-reports" } +dependencies = [ + { name = "lxml" }, + { name = "ub-project" }, +] + +[package.optional-dependencies] +pytest = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "lxml" }, + { name = "pytest", marker = "extra == 'pytest'", specifier = ">=7.0" }, + { name = "ub-project", editable = "packages/ub-project" }, +] +provides-extras = ["pytest"] + [[package]] name = "urllib3" version = "2.8.0" From 455555e08365da8c993d72e5c72224f7a86fc4d5 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Thu, 1 Oct 2026 02:22:09 +0200 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=A7=AA=20Divide=20the=20test=20suite?= =?UTF-8?q?=20between=20the=20core=20and=20the=20extension?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ub-test-reports gets its share of the suite, the extension keeps the tests that need a Sphinx build. Six modules move whole with `git mv` (cli_config, identity, junit_parser, junit_parser_gtest, needs_export, pytest_plugin). test_cli_convert, test_project_config and test_result_vocabulary are split by the old `toolchain` marker: the unmarked tests go to the core's file of the same name, the marked ones stay in the extension's, and the marker is gone from every site. Deleted with the lazy toolchain check: tests/test_toolchain.py, TestSphinxFree.test_a_missing_sphinx_needs_is_an_extension_error and test_the_deprecation_comes_before_a_toolchain_error. test_projectconfig_imports_without_sphinx moves to the core. test_aliases.py: the three module aliases are checked against `ub_test_reports`, and the walk of old names that must fail plainly covers both packages, located with find_spec rather than an import of the (now eager) extension root. The core reads its own copies of the fixtures it needs, under tests/fixtures/; the extension's tests/doc_test/utils/ is unchanged but for the dead xml_data_2.xml, which nothing read. New in the core: test_json_parser.py (the JSON parser called directly), and two tests pinning the plugin's wire names `sphinxcontrib.test_reports:file|line`. --- .../sphinx-test-reports/tests/conftest.py | 14 +- .../tests/doc_test/utils/xml_data_2.xml | 10 - .../sphinx-test-reports/tests/test_aliases.py | 111 ++- .../tests/test_cli_convert.py | 570 +------------ .../tests/test_json_parser.py | 4 +- .../tests/test_project_config.py | 751 +----------------- .../tests/test_properties.py | 28 +- .../tests/test_result_vocabulary.py | 137 +--- .../tests/test_toolchain.py | 158 ---- packages/ub-test-reports/tests/__init__.py | 0 packages/ub-test-reports/tests/conftest.py | 9 + .../ub-test-reports/tests/fixtures/ctest.xml | 37 + .../tests/fixtures/gtest_data.xml | 31 + .../tests/fixtures/json_complex_data.json | 53 ++ .../tests/fixtures/json_custom_data.json | 72 ++ .../tests/fixtures/json_data.json | 49 ++ .../tests/fixtures/nose_data.xml | 7 + .../tests/fixtures/pytest_data.xml | 52 ++ .../tests/fixtures/pytest_data_5_1.xml | 93 +++ .../tests/fixtures/pytest_data_6_2.xml | 29 + .../tests/fixtures/pytest_nested_example.xml | 40 + .../tests/fixtures/runner_error_data.xml | 21 + .../tests/fixtures/xml_data.xml | 7 + .../tests/fixtures/xml_data_error.xml | 12 + .../tests/test_cli_config.py | 6 +- .../ub-test-reports/tests/test_cli_convert.py | 599 ++++++++++++++ .../tests/test_identity.py | 2 +- .../ub-test-reports/tests/test_json_parser.py | 143 ++++ .../tests/test_junit_parser.py | 42 +- .../tests/test_junit_parser_gtest.py | 6 +- .../tests/test_needs_export.py | 4 +- .../tests/test_project_config.py | 739 +++++++++++++++++ .../tests/test_pytest_plugin.py | 53 +- .../tests/test_result_vocabulary.py | 151 ++++ 34 files changed, 2297 insertions(+), 1743 deletions(-) delete mode 100644 packages/sphinx-test-reports/tests/doc_test/utils/xml_data_2.xml delete mode 100644 packages/sphinx-test-reports/tests/test_toolchain.py create mode 100644 packages/ub-test-reports/tests/__init__.py create mode 100644 packages/ub-test-reports/tests/conftest.py create mode 100644 packages/ub-test-reports/tests/fixtures/ctest.xml create mode 100644 packages/ub-test-reports/tests/fixtures/gtest_data.xml create mode 100644 packages/ub-test-reports/tests/fixtures/json_complex_data.json create mode 100644 packages/ub-test-reports/tests/fixtures/json_custom_data.json create mode 100644 packages/ub-test-reports/tests/fixtures/json_data.json create mode 100644 packages/ub-test-reports/tests/fixtures/nose_data.xml create mode 100644 packages/ub-test-reports/tests/fixtures/pytest_data.xml create mode 100644 packages/ub-test-reports/tests/fixtures/pytest_data_5_1.xml create mode 100644 packages/ub-test-reports/tests/fixtures/pytest_data_6_2.xml create mode 100644 packages/ub-test-reports/tests/fixtures/pytest_nested_example.xml create mode 100644 packages/ub-test-reports/tests/fixtures/runner_error_data.xml create mode 100644 packages/ub-test-reports/tests/fixtures/xml_data.xml create mode 100644 packages/ub-test-reports/tests/fixtures/xml_data_error.xml rename packages/{sphinx-test-reports => ub-test-reports}/tests/test_cli_config.py (99%) create mode 100644 packages/ub-test-reports/tests/test_cli_convert.py rename packages/{sphinx-test-reports => ub-test-reports}/tests/test_identity.py (99%) create mode 100644 packages/ub-test-reports/tests/test_json_parser.py rename packages/{sphinx-test-reports => ub-test-reports}/tests/test_junit_parser.py (83%) rename packages/{sphinx-test-reports => ub-test-reports}/tests/test_junit_parser_gtest.py (96%) rename packages/{sphinx-test-reports => ub-test-reports}/tests/test_needs_export.py (99%) create mode 100644 packages/ub-test-reports/tests/test_project_config.py rename packages/{sphinx-test-reports => ub-test-reports}/tests/test_pytest_plugin.py (93%) create mode 100644 packages/ub-test-reports/tests/test_result_vocabulary.py diff --git a/packages/sphinx-test-reports/tests/conftest.py b/packages/sphinx-test-reports/tests/conftest.py index abb52087d..2bac6d86b 100644 --- a/packages/sphinx-test-reports/tests/conftest.py +++ b/packages/sphinx-test-reports/tests/conftest.py @@ -1,21 +1,15 @@ """Pytest conftest module containing common test configuration and fixtures.""" -import importlib.util import shutil from pathlib import Path from tempfile import mkdtemp import pytest -# The documentation toolchain is an extra of the package, and the converter's -# and the pytest plugin's tests run where it is not installed (the -# `toolchain_free` and `plugin_floor` nox sessions), so Sphinx's fixtures are -# loaded only where Sphinx is. Tests that need a build carry the `toolchain` -# mark and are deselected there. `pytester`, which the plugin's tests drive, -# ships with pytest itself. -pytest_plugins = ( - ["sphinx.testing.fixtures"] if importlib.util.find_spec("sphinx") else [] -) + ["pytester"] +# Sphinx is a dependency of this package, so its fixtures are always there. `pytester`, +# which the alias tests drive the old plugin name with, ships with pytest itself. (The +# converter's and the plugin's own suite is ub-test-reports', and runs without Sphinx.) +pytest_plugins = ["sphinx.testing.fixtures", "pytester"] def copy_srcdir_to_tmpdir(srcdir, tmp): diff --git a/packages/sphinx-test-reports/tests/doc_test/utils/xml_data_2.xml b/packages/sphinx-test-reports/tests/doc_test/utils/xml_data_2.xml deleted file mode 100644 index 30edbbc92..000000000 --- a/packages/sphinx-test-reports/tests/doc_test/utils/xml_data_2.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - key "name" could not be accessed in junit object - - - Passed on the last test run - - diff --git a/packages/sphinx-test-reports/tests/test_aliases.py b/packages/sphinx-test-reports/tests/test_aliases.py index a038f0e23..d0b96d1df 100644 --- a/packages/sphinx-test-reports/tests/test_aliases.py +++ b/packages/sphinx-test-reports/tests/test_aliases.py @@ -3,12 +3,12 @@ ``sphinxcontrib.test_reports`` keeps four names until 4.0: the package itself as a Sphinx extension, which warns through Sphinx's logger as ``[test_reports.deprecated]``, and the ``pytest_plugin``, ``junitparser`` and ``jsonparser`` modules, which ARE the real modules -and raise one ``FutureWarning`` at the importing line. Every other old module name fails -as an ordinary missing module. +-- in ``ub_test_reports``, the core this extension depends on -- and raise one +``FutureWarning`` at the importing line. Every other old module name, of either package, +fails as an ordinary missing module. -The module tests run where the documentation toolchain is not installed (the -``toolchain-free`` CI job runs this file against the BUILT wheel); the builds carry the -``toolchain`` mark. +Neither real package is imported at module level here: the extension's root imports Sphinx +eagerly, and the walk below needs only where the two packages are. """ from __future__ import annotations @@ -19,15 +19,17 @@ import sys import textwrap import warnings +from importlib.util import find_spec from pathlib import Path from unittest import mock import pytest -import sphinx_test_reports - OLD = "sphinxcontrib.test_reports" +#: the extension, which the old EXTENSION name loads NEW = "sphinx_test_reports" +#: the core, where the three old MODULE names point +CORE = "ub_test_reports" #: The aliased modules, each with one public name to check identity and patching by. ALIASES = { @@ -36,20 +38,33 @@ "jsonparser": "JsonParser", } -#: Every other module of the real package, walked rather than listed, so a module added -#: later is covered without anyone remembering this file. Read off the files rather than + +def _root(package: str) -> Path: + """The directory of *package*, found without importing it.""" + spec = find_spec(package) + assert spec is not None, package + assert spec.origin is not None, package + return Path(spec.origin).parent + + +#: Every other module of BOTH real packages -- the extension and the core -- walked rather +#: than listed, so a module added later to either is covered without anyone remembering +#: this file: under the old name each one must fail plainly. Read off the files rather than #: through `pkgutil`, which imports each subpackage -- and `directives` imports Sphinx. -_ROOT = Path(sphinx_test_reports.__file__).parent +_ROOTS = [_root(NEW), _root(CORE)] UNALIASED = sorted( - name - for name in ( - ".".join(path.relative_to(_ROOT).with_suffix("").parts).removesuffix( - ".__init__" + { + name + for root in _ROOTS + for name in ( + ".".join(path.relative_to(root).with_suffix("").parts).removesuffix( + ".__init__" + ) + for path in root.rglob("*.py") + if path.name != "__init__.py" or path.parent != root ) - for path in _ROOT.rglob("*.py") - if path.name != "__init__.py" or path.parent != _ROOT - ) - if name not in ALIASES + if name not in ALIASES + } ) @@ -84,16 +99,17 @@ def _restore_old_names(): def test_every_real_module_is_either_aliased_or_walked() -> None: - # the fence's fence: an empty walk would make the unaliased tests pass vacuously - assert "identity" in UNALIASED - assert "directives.test_case" in UNALIASED + # the fence's fence: an empty walk would make the unaliased tests pass vacuously, and + # a walk of one root would leave the other package's old names unchecked + assert "identity" in UNALIASED # the core's + assert "directives.test_case" in UNALIASED # the extension's assert set(ALIASES).isdisjoint(UNALIASED) @pytest.mark.parametrize("module", sorted(ALIASES)) def test_the_alias_is_the_real_module(module: str) -> None: imported, _ = _import_old(module) - real = importlib.import_module(f"{NEW}.{module}") + real = importlib.import_module(f"{CORE}.{module}") assert imported is real assert sys.modules[f"{OLD}.{module}"] is real assert getattr(sys.modules[OLD], module) is real @@ -108,7 +124,7 @@ def test_the_alias_warns_once_at_the_importing_line(module: str) -> None: (warning,) = caught assert (warning.filename, warning.lineno) == ("", 1) message = str(warning.message) - assert f"{OLD}.{module} has moved to {NEW}.{module}" in message + assert f"{OLD}.{module} has moved to {CORE}.{module}" in message assert "4.0" in message @@ -124,17 +140,17 @@ def test_a_second_import_does_not_warn_again(module: str) -> None: @pytest.mark.parametrize("module", sorted(ALIASES)) def test_the_real_module_keeps_its_own_spec(module: str) -> None: _import_old(module) - real = importlib.import_module(f"{NEW}.{module}") - assert real.__name__ == f"{NEW}.{module}" + real = importlib.import_module(f"{CORE}.{module}") + assert real.__name__ == f"{CORE}.{module}" assert real.__spec__ is not None - assert real.__spec__.name == f"{NEW}.{module}" + assert real.__spec__.name == f"{CORE}.{module}" @pytest.mark.parametrize("module", sorted(ALIASES)) def test_patching_through_the_old_path_reaches_the_real_module(module: str) -> None: _import_old(module) name = ALIASES[module] - real = importlib.import_module(f"{NEW}.{module}") + real = importlib.import_module(f"{CORE}.{module}") with warnings.catch_warnings(): warnings.simplefilter("ignore", FutureWarning) with mock.patch(f"{OLD}.{module}.{name}") as patched: @@ -152,6 +168,7 @@ def test_an_unaliased_old_name_fails_plainly(module: str) -> None: assert excinfo.value.name is not None assert excinfo.value.name.startswith(OLD) assert NEW not in str(excinfo.value) + assert CORE not in str(excinfo.value) def _run_user_code(tmp_path: Path, source: str) -> subprocess.CompletedProcess[str]: @@ -211,7 +228,7 @@ class TestPytestPlugin: OLD_PLUGIN = f"{OLD}.pytest_plugin" SOURCE = """ -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @add_test_properties(test_type="unit") def test_decorated(): @@ -234,7 +251,7 @@ def test_the_old_name_loads_the_plugin_and_warns(self, pytester) -> None: result, report = self._run(pytester, self.OLD_PLUGIN) result.assert_outcomes(passed=1) result.stderr.fnmatch_lines( - [f"*FutureWarning: {self.OLD_PLUGIN} has moved to {NEW}.pytest_plugin*"] + [f"*FutureWarning: {self.OLD_PLUGIN} has moved to {CORE}.pytest_plugin*"] ) xml = report.read_text(encoding="utf-8") # the plugin's hooks ran: the property is written, and written once @@ -259,13 +276,13 @@ def test_the_old_filter_line_matches_the_real_class(self, pytester) -> None: assert result.ret == pytest.ExitCode.USAGE_ERROR def test_both_names_fail_loudly_naming_both(self, pytester) -> None: - result, report = self._run(pytester, f"{NEW}.pytest_plugin", self.OLD_PLUGIN) + result, report = self._run(pytester, f"{CORE}.pytest_plugin", self.OLD_PLUGIN) assert result.ret != 0 assert not report.exists() result.stderr.fnmatch_lines( [ "*Plugin already registered under a different name: " - f"{self.OLD_PLUGIN}= None: # fenced here and documented, not worked around. result, report = self._run( pytester, - f"{NEW}.pytest_plugin", + f"{CORE}.pytest_plugin", family="xunit2", ini=f"filterwarnings =\n error\n ignore::{self.OLD_PLUGIN}.TestReportsConfigWarning\n", ) @@ -286,7 +303,7 @@ def test_the_old_filter_line_must_move_with_the_p_line(self, pytester) -> None: result.stderr.fnmatch_lines( [ f"*ignore::{self.OLD_PLUGIN}.TestReportsConfigWarning*", - f"*FutureWarning: {self.OLD_PLUGIN} has moved to {NEW}.pytest_plugin*", + f"*FutureWarning: {self.OLD_PLUGIN} has moved to {CORE}.pytest_plugin*", ] ) @@ -323,7 +340,6 @@ def _needs(app) -> dict: return dict(SphinxNeedsData(app.env).get_needs_view()) -@pytest.mark.toolchain class TestExtensionAlias: """``extensions = ["sphinxcontrib.test_reports"]`` builds, and says so.""" @@ -340,7 +356,9 @@ def test_the_old_name_builds_with_the_deprecation_warning(self, test_app) -> Non # the real extension is registered, and its metadata is what the alias reports real = app.extensions[NEW] alias = app.extensions[OLD] - assert alias.version == real.version == sphinx_test_reports.__version__ + from sphinx_test_reports import __version__ + + assert alias.version == real.version == __version__ assert alias.parallel_read_safe == real.parallel_read_safe @pytest.mark.parametrize( @@ -408,29 +426,6 @@ def test_a_strict_build_fails_on_it_and_suppress_warnings_clears_it( assert passed.returncode == 0, passed.stderr -@pytest.mark.toolchain -def test_the_deprecation_comes_before_a_toolchain_error( - make_app, tmp_path, monkeypatch -) -> None: - """A project that names the old extension with an outdated toolchain sees the - deprecation first, then the real extension's toolchain error, which names the new one. - """ - import io - - from sphinx.errors import ExtensionError - - from sphinx_test_reports import toolchain - - monkeypatch.setattr( - toolchain, "unmet_requirements", lambda: ["sphinx-needs 5.1.0 < 8.5.0"] - ) - warning = io.StringIO() - with pytest.raises(ExtensionError, match=f"Could not load extension {NEW}"): - _make_old_project(make_app, tmp_path, {}, warning=warning) - # the error aborted the application, so the warning stream holds only what came first - _assert_the_deprecation_once(warning.getvalue()) - - def _make_old_project(make_app, tmp_path: Path, confoverrides: dict, **kwargs): import shutil diff --git a/packages/sphinx-test-reports/tests/test_cli_convert.py b/packages/sphinx-test-reports/tests/test_cli_convert.py index ba471dba1..1832d24c2 100644 --- a/packages/sphinx-test-reports/tests/test_cli_convert.py +++ b/packages/sphinx-test-reports/tests/test_cli_convert.py @@ -9,23 +9,24 @@ documentation toolchain into the test-result conversion; * the output must be byte-stable, because it is a cached build artifact and qualification evidence. + +This is the half that needs Sphinx: the converter's output checked against sphinx-needs' +schema and imported into a build. The converter's own tests are ub-test-reports', +``packages/ub-test-reports/tests/test_cli_convert.py``. """ import json -import subprocess -import sys from pathlib import Path import pytest UTILS = Path(__file__).parent / "doc_test" / "utils" GTEST_XML = UTILS / "gtest_data.xml" -PYTEST_XML = UTILS / "pytest_data.xml" def _convert(tmp_path, *args, xml=GTEST_XML): """Run the converter and return (exit_code, parsed output or None).""" - from sphinx_test_reports.cli import main + from ub_test_reports.cli import main output = tmp_path / "needs.json" code = main(["build", "needs", str(xml), "--output", str(output), *args]) @@ -46,29 +47,7 @@ def _field_type(declaration): return next(iter(set(kind) - {"null"})) -class TestNoSphinxImport: - """The converter has to be usable without the documentation toolchain.""" - - def test_importing_the_cli_does_not_import_sphinx(self): - script = ( - "import sys;" - "import sphinx_test_reports.cli;" - "leaked = sorted(m for m in sys.modules" - " if m == 'sphinx' or m.startswith(('sphinx.', 'sphinx_needs')));" - "print(','.join(leaked))" - ) - result = subprocess.run( - [sys.executable, "-c", script], - capture_output=True, - text=True, - check=True, - ) - - assert result.stdout.strip() == "" - - class TestEnvelope: - @pytest.mark.toolchain def test_output_passes_the_sphinx_needs_schema(self, tmp_path): """The output must validate against sphinx-needs' own needs.json schema.""" from sphinx_needs import needsfile @@ -78,546 +57,7 @@ def test_output_passes_the_sphinx_needs_schema(self, tmp_path): assert code == 0 assert needsfile.check_needs_data(data).schema == [] - def test_envelope_carries_project_and_current_version(self, tmp_path): - _, data = _convert(tmp_path, "--project", "Score Docs-as-Code") - - assert data["project"] == "Score Docs-as-Code" - assert data["current_version"] in data["versions"] - - def test_needs_amount_matches_the_number_of_cases(self, tmp_path): - _, data = _convert(tmp_path) - - version = data["versions"][data["current_version"]] - assert version["needs_amount"] == len(version["needs"]) == 5 - - def test_every_written_field_is_declared(self, tmp_path): - """The file says what its fields are, without a Sphinx build to ask.""" - _, data = _convert(tmp_path) - - version = data["versions"][data["current_version"]] - declared = set(version["needs_schema"]["properties"]) - written = {key for need in version["needs"].values() for key in need} - - assert written - declared == set() - - def test_no_timestamp_is_written(self, tmp_path): - """A wall clock would defeat action caching and evidence diffs.""" - _, data = _convert(tmp_path) - - assert "created" not in data - assert "created" not in data["versions"][data["current_version"]] - - def test_output_is_byte_stable_across_runs(self, tmp_path): - from sphinx_test_reports.cli import main - - first = tmp_path / "first.json" - second = tmp_path / "second.json" - for output in (first, second): - assert ( - main(["build", "needs", str(GTEST_XML), "--output", str(output)]) == 0 - ) - - assert first.read_bytes() == second.read_bytes() - - -class TestNeedContent: - def test_ids_match_the_deterministic_scheme(self, tmp_path): - from sphinx_test_reports.identity import deterministic_case_id - - _, data = _convert(tmp_path) - - expected = deterministic_case_id( - classname="MathTest", name="Addition", file="src/math_test.cc" - ) - assert expected in _needs(data) - - def test_source_location_is_emitted_verbatim(self, tmp_path): - _, data = _convert(tmp_path) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["case_file"] == "src/math_test.cc" - assert need["case_line"] == "12" - # `file` is the report path, as in a locally created test-case need. - assert need["file"] == str(GTEST_XML) - - def test_type_and_title_match_the_directive_s(self, tmp_path): - # The directive titles a case need with the case name; a needtable - # title column must not tell an imported case from a built one. - _, data = _convert(tmp_path) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["type"] == "testcase" - assert need["title"] == "Addition" - - def test_result_vocabulary_includes_disabled(self, tmp_path): - _, data = _convert(tmp_path) - - need = _needs(data)["testcase__MathTest__DISABLED_Division_jnyzp"] - assert need["result"] == "disabled" - - def test_content_keeps_every_failure_part(self, tmp_path): - """R2: the debug output has to survive the conversion.""" - _, data = _convert(tmp_path) - - content = _needs(data)["testcase__MathTest__Subtraction_srmht"]["content"] - assert "Expected equality of these values" in content - assert "Actual: false" in content - assert "overflow guard hit" in content - - def test_result_text_is_the_first_failure_message(self, tmp_path): - _, data = _convert(tmp_path) - - need = _needs(data)["testcase__MathTest__Subtraction_srmht"] - assert need["result_text"].startswith("src/math_test.cc:22") - assert "\n" not in need["result_text"] - - def test_named_properties_become_fields(self, tmp_path, capsys): - # Only properties the build would accept (extra_options, or here the - # flag standing in for it) become fields; the rest is reported once. - code, data = _convert(tmp_path, "--no-config", "--extra-option", "TestType") - assert code == 0 - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["TestType"] == "requirements-based" - assert "PartiallyVerifies" not in need - message = capsys.readouterr().err - assert "properties not exported" in message - assert "PartiallyVerifies" in message - assert "extra_options" in message - - def test_unexported_properties_are_reported_once_for_all_names( - self, tmp_path, capsys - ): - # One line naming every left-out property -- not one line per name, - # and not one per case that carries it. - code, _ = _convert(tmp_path, "--no-config") - assert code == 0 - lines = [ - line - for line in capsys.readouterr().err.splitlines() - if "properties not exported" in line - ] - assert len(lines) == 1 - assert "PartiallyVerifies" in lines[0] and "TestType" in lines[0] - - def test_an_exported_property_is_present_on_every_case(self, tmp_path): - # Null where the case has no such property, as the build leaves a - # registered field a directive did not set -- so one schema can - # require the field of imported and locally created needs alike. - _, data = _convert(tmp_path, "--no-config", "--extra-option", "TestType") - needs = _needs(data) - assert needs["testcase__MathTest__Addition_hcuyy"]["TestType"] == ( - "requirements-based" - ) - assert needs["testcase__MathTest__Subtraction_srmht"]["TestType"] is None - assert all("TestType" in need for need in needs.values()) - - def test_unnamed_properties_are_left_out_quietly_when_none_exist( - self, tmp_path, capsys - ): - code, _ = _convert(tmp_path, "--no-config", xml=PYTEST_XML) - assert code == 0 - assert "properties not exported" not in capsys.readouterr().err - - def test_tags_are_configurable(self, tmp_path): - _, data = _convert(tmp_path, "--tags", "TEST") - - assert _needs(data)["testcase__MathTest__Addition_hcuyy"]["tags"] == ["TEST"] - - -class TestLinkProperties: - def test_a_property_can_be_promoted_to_a_link_field(self, tmp_path): - _, data = _convert( - tmp_path, "--link-property", "PartiallyVerifies=partially_verifies" - ) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["partially_verifies"] == ["REQ_1", "REQ_2"] - assert "PartiallyVerifies" not in need - - def test_link_fields_are_always_present_even_when_empty(self, tmp_path): - """A converter must emit its fields unconditionally, so schemas can require them.""" - _, data = _convert( - tmp_path, "--link-property", "PartiallyVerifies=partially_verifies" - ) - - need = _needs(data)["testcase__MathTest__DISABLED_Division_jnyzp"] - assert need["partially_verifies"] == [] - - def test_malformed_link_property_is_rejected(self, tmp_path): - from sphinx_test_reports.cli import main - - code = main( - [ - "build", - "needs", - str(GTEST_XML), - "--output", - str(tmp_path / "out.json"), - "--link-property", - "NoEqualsSign", - ] - ) - - assert code != 0 - - -class TestRemoteUrls: - def test_external_url_and_remote_url_are_synthesized(self, tmp_path): - _, data = _convert( - tmp_path, - "--remote-url", - "https://github.com/org/repo", - "--commit", - "abc123", - ) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - expected = "https://github.com/org/repo/blob/abc123/src/math_test.cc#L12" - assert need["external_url"] == expected - assert need["remote_url"] == expected - - def test_scp_style_remote_is_normalised(self, tmp_path): - _, data = _convert( - tmp_path, - "--remote-url", - "git@github.com:org/repo.git", - "--commit", - "abc123", - ) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["remote_url"].startswith("https://github.com/org/repo/blob/abc123/") - - def test_url_pattern_is_configurable(self, tmp_path): - _, data = _convert( - tmp_path, - "--remote-url", - "https://gitlab.com/org/repo", - "--commit", - "abc123", - "--url-pattern", - "{base}/-/blob/{commit}/{file}#L{line}", - ) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["remote_url"] == ( - "https://gitlab.com/org/repo/-/blob/abc123/src/math_test.cc#L12" - ) - - def test_credentials_in_the_remote_url_are_not_written(self, tmp_path): - # GitLab's CI_REPOSITORY_URL embeds the job token; the base lands in - # every need of a cached artifact and, imported, in published HTML. - from sphinx_test_reports.cli import main - - output = tmp_path / "needs.json" - code = main( - [ - "build", - "needs", - str(GTEST_XML), - "--output", - str(output), - "--no-config", - "--remote-url", - "https://gitlab-ci-token:glcbt-secret@gitlab.example.com/org/repo.git", - "--commit", - "abc123", - ] - ) - assert code == 0 - text = output.read_text(encoding="utf-8") - assert "glcbt-secret" not in text and "gitlab-ci-token" not in text - need = _needs(json.loads(text))["testcase__MathTest__Addition_hcuyy"] - assert need["remote_url"] == ( - "https://gitlab.example.com/org/repo/blob/abc123/src/math_test.cc#L12" - ) - - def test_without_repo_metadata_the_url_fields_are_empty(self, tmp_path): - """A hermetic sandbox has no git remote; that must not drop the need.""" - _, data = _convert(tmp_path) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["remote_url"] == "" - assert need["external_url"] == "" - - -class TestUrlPatternErrors: - """A bad template is a configuration error at the start, not a traceback.""" - - def test_an_unknown_placeholder_is_an_error(self, tmp_path, capsys): - code, data = _convert( - tmp_path, - "--no-config", - "--remote-url", - "https://github.com/o/r", - "--commit", - "abc", - "--url-pattern", - "{base}/blob/{ref}/{file}#L{line}", - ) - assert code == 2 - assert data is None - message = capsys.readouterr().err - assert "--url-pattern" in message - assert "{ref}" in message - - def test_an_unbalanced_brace_is_an_error(self, tmp_path, capsys): - code, _ = _convert( - tmp_path, "--no-config", "--url-pattern", "{base/blob/{commit}/{file}" - ) - assert code == 2 - assert "malformed" in capsys.readouterr().err - - def test_an_attribute_lookup_is_an_error_not_a_traceback(self, tmp_path, capsys): - # str.format resolves {base.__class__}; only KeyError was caught. - code, data = _convert( - tmp_path, - "--no-config", - "--remote-url", - "https://github.com/o/r", - "--commit", - "abc", - "--url-pattern", - "{base.__class__}/{file}", - ) - assert code == 2 - assert data is None - message = capsys.readouterr().err - assert "unknown placeholder {base.__class__}" in message - assert "Traceback" not in message - - def test_the_pattern_is_checked_before_any_report_is_read(self, tmp_path, capsys): - # A missing report and a bad pattern: the pattern error wins, because - # the template is checked before the first file is opened. - code, data = _convert( - tmp_path, - "--no-config", - "--url-pattern", - "{base}/blob/{ref}/{file}", - xml=tmp_path / "does-not-exist.xml", - ) - assert code == 2 - assert data is None - message = capsys.readouterr().err - assert "{ref}" in message - assert "no such file" not in message - - -class TestMultipleInputs: - def test_several_reports_are_merged_into_one_file(self, tmp_path): - from sphinx_test_reports.cli import main - - output = tmp_path / "needs.json" - code = main( - ["build", "needs", str(GTEST_XML), str(PYTEST_XML), "--output", str(output)] - ) - data = json.loads(output.read_text(encoding="utf-8")) - - assert code == 0 - assert len(_needs(data)) > 5 - - def test_the_same_report_given_twice_is_refused(self, tmp_path, capsys): - # Silently collapsing the repeats would produce a valid file that has - # lost half its evidence -- the worst outcome for a cached artifact. - from sphinx_test_reports.cli import main - - output = tmp_path / "needs.json" - code = main( - [ - "build", - "needs", - str(GTEST_XML), - str(GTEST_XML), - "--no-config", - "-o", - str(output), - ] - ) - assert code == 2 - assert not output.exists() - message = capsys.readouterr().err - assert "more than once" in message - assert "testcase__" in message - - def test_a_missing_input_file_exits_nonzero(self, tmp_path): - from sphinx_test_reports.cli import main - - code = main( - [ - "build", - "needs", - str(tmp_path / "nope.xml"), - "--output", - str(tmp_path / "out.json"), - ] - ) - - assert code != 0 - - -class TestDiagnostics: - def test_absent_line_attributes_warn_about_junit_family(self, tmp_path, capsys): - """pytest's default junit_family drops file/line; say so, don't guess.""" - from sphinx_test_reports.cli import main - - main( - [ - "build", - "needs", - str(PYTEST_XML), - "--output", - str(tmp_path / "out.json"), - ] - ) - - assert "junit_family" in capsys.readouterr().err - - def test_nested_suites_get_the_hint_too(self, tmp_path, capsys): - # The parser files the cases of a nested report under testsuite_nested; - # a hint that only looked at the top level went quiet on exactly the - # Ant/Maven-shaped reports that most often lack source locations. - code, data = _convert( - tmp_path, "--no-config", xml=UTILS / "pytest_nested_example.xml" - ) - assert code == 0 - assert all(need["case_line"] == "" for need in _needs(data).values()) - assert "junit_family" in capsys.readouterr().err - - def test_reports_with_line_attributes_do_not_warn(self, tmp_path, capsys): - from sphinx_test_reports.cli import main - - main(["build", "needs", str(GTEST_XML), "--output", str(tmp_path / "out.json")]) - - assert "junit_family" not in capsys.readouterr().err - - def test_a_report_without_test_cases_warns(self, tmp_path, capsys): - # A pom.xml parses as one empty suite: a valid, empty needs.json with - # exit 0 is the one outcome a cached build action must never get - # silently. - pom = tmp_path / "pom.xml" - pom.write_text( - "4.0.0\n", encoding="utf-8" - ) - code, data = _convert(tmp_path, "--no-config", xml=pom) - assert code == 0 - assert data["versions"][data["current_version"]]["needs_amount"] == 0 - message = capsys.readouterr().err - assert "pom.xml" in message and "no test cases" in message - - def test_a_report_with_test_cases_does_not_get_the_empty_warning( - self, tmp_path, capsys - ): - code, _ = _convert(tmp_path, "--no-config") - assert code == 0 - assert "no test cases" not in capsys.readouterr().err - - -def test_the_cli_is_runnable_as_a_module(): - result = subprocess.run( - [ - sys.executable, - "-m", - "sphinx_test_reports.cli", - "build", - "needs", - "--help", - ], - capture_output=True, - text=True, - ) - - assert result.returncode == 0 - assert "--output" in result.stdout - - -def test_console_script_is_installed(): - """The name that goes into a BUILD file has to be a real entry point.""" - script = Path(sys.executable).parent / "test-reports" - if not script.exists(): - pytest.skip("package not installed into this environment") - - result = subprocess.run( - [str(script), "build", "needs", "--help"], capture_output=True, text=True - ) - - assert result.returncode == 0 - assert "--output" in result.stdout - - -@pytest.mark.parametrize("flag", ["--remote-url", "--commit"]) -def test_url_synthesis_needs_both_parts(tmp_path, flag): - """Half the metadata cannot produce a URL; fail loudly instead of guessing.""" - from sphinx_test_reports.cli import main - - code = main( - [ - "build", - "needs", - str(GTEST_XML), - "--output", - str(tmp_path / "out.json"), - flag, - "value", - ] - ) - - assert code != 0 - - -class TestResultVocabulary: - """The export uses the parser's vocabulary, which is the build's. - - ``failed`` is a documented need field value and a CSS class - (``tr_failed``), and the shipped report template filters on it. A project - that mixes imported and locally created test-case needs filters both with - one expression only if the two writers spell the result alike. - """ - - def test_failure_is_exported_as_the_build_spells_it(self, tmp_path): - _, data = _convert(tmp_path) - - assert ( - _needs(data)["testcase__MathTest__Subtraction_srmht"]["result"] == "failed" - ) - - @pytest.mark.parametrize( - ("need_id", "expected"), - [ - ("testcase__MathTest__Addition_hcuyy", "passed"), - ("testcase__MathTest__DISABLED_Division_jnyzp", "disabled"), - ("testcase__ParamTest_0__Legacy_owuvz", "skipped"), - ], - ) - def test_other_results_are_unchanged(self, tmp_path, need_id, expected): - _, data = _convert(tmp_path) - - assert _needs(data)[need_id]["result"] == expected - - -class TestContentIsNotDuplicated: - """googletest repeats the failure text in the message attribute. - - Emitting both verbatim shows the same stack trace twice in the rendered - need; the message block is only worth its space when it says something the - body does not. - """ - - def test_a_message_contained_in_the_body_is_not_repeated(self, tmp_path): - _, data = _convert(tmp_path) - - content = _needs(data)["testcase__MathTest__Subtraction_srmht"]["content"] - assert content.count("Expected equality of these values") == 1 - assert "message" not in content - - def test_a_message_absent_from_the_body_is_kept(self, tmp_path): - _, data = _convert(tmp_path) - - content = _needs(data)["testcase__ParamTest_0__Legacy_owuvz"]["content"] - assert "Skipped via GTEST_SKIP" in content - assert "not applicable on this platform" in content - -@pytest.mark.toolchain class TestImportIntoABuild: """The documented consumption path: ``needimport`` of the produced file. diff --git a/packages/sphinx-test-reports/tests/test_json_parser.py b/packages/sphinx-test-reports/tests/test_json_parser.py index 56e05c746..9c90b8a3b 100644 --- a/packages/sphinx-test-reports/tests/test_json_parser.py +++ b/packages/sphinx-test-reports/tests/test_json_parser.py @@ -8,14 +8,14 @@ def test_init_json_parser(): - from sphinx_test_reports.jsonparser import JsonParser + from ub_test_reports.jsonparser import JsonParser parser = JsonParser(json_path) assert parser is not None def test_parse_json_data(): - from sphinx_test_reports.jsonparser import JsonParser + from ub_test_reports.jsonparser import JsonParser json_mapping = { "json_config": { diff --git a/packages/sphinx-test-reports/tests/test_project_config.py b/packages/sphinx-test-reports/tests/test_project_config.py index 690694042..0b393a296 100644 --- a/packages/sphinx-test-reports/tests/test_project_config.py +++ b/packages/sphinx-test-reports/tests/test_project_config.py @@ -4,31 +4,22 @@ its keys onto the ``tr_*`` config values. Both must agree on which file describes a project, or the build is configured by something other than what the project declares. + +This is the half that needs Sphinx: the bridge onto the ``tr_*`` values, its precedence +and Sphinx's confval type check. The loader's half is ub-test-reports', +``packages/ub-test-reports/tests/test_project_config.py``. """ -import os -import subprocess -import sys from io import StringIO from pathlib import Path from shutil import copytree import pytest -import ub_project -from sphinx_test_reports.projectconfig import ( - BRIDGE_KEYS, - BUILD_TABLE, - DEFAULT_FIELD_NAMES, +from ub_test_reports.projectconfig import ( DEFAULT_TOML_FILENAME, SECTION, - TomlConfigError, - field_names, - find_project_config, - load_project_config, - needs_settings, ) -from ub_project import ProjectConfigError def _write(tmp_path, toml_source, name=DEFAULT_TOML_FILENAME): @@ -37,448 +28,6 @@ def _write(tmp_path, toml_source, name=DEFAULT_TOML_FILENAME): return config -class TestLoader: - """The Sphinx-free loader: parsing, normalising, anchoring, rejecting.""" - - def test_missing_file_is_none(self, tmp_path): - assert load_project_config(tmp_path / DEFAULT_TOML_FILENAME) is None - - def test_missing_section_is_empty(self, tmp_path): - _write(tmp_path, '[project]\nname = "x"\n') - assert load_project_config(tmp_path / DEFAULT_TOML_FILENAME) == {} - - def test_full_section_round_trips(self, tmp_path): - _write( - tmp_path, - """ - [test_reports] - file_option = "report_file" - source_file_option = "file" - import_encoding = "latin1" - deterministic_case_ids = true - suite_id_length = 4 - extra_options = ["more_info"] - property_link_types = { request = "req" } - """, - ) - config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - assert config["file_option"] == "report_file" - assert config["source_file_option"] == "file" - assert config["import_encoding"] == "latin1" - assert config["deterministic_case_ids"] is True - assert config["suite_id_length"] == 4 - assert config["extra_options"] == ["more_info"] - assert config["property_link_types"] == {"request": "req"} - - def test_build_needs_table_is_validated_but_never_bridged(self, tmp_path): - # [test_reports.build.needs] belongs to the command line. The build - # validates it -- one file, one verdict -- but must not map it onto a - # tr_* value. - _write( - tmp_path, - """ - [test_reports] - file_option = "report_file" - - [test_reports.build.needs] - project = "demo" - tags = ["ci"] - """, - ) - reported = [] - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) - assert reported == [] - assert needs_settings(section) == {"project": "demo", "tags": ["ci"]} - assert BUILD_TABLE not in BRIDGE_KEYS - - @pytest.mark.parametrize( - ("key", "value"), - [ - ("project", "42"), - ("tags", '"ci, unit"'), # a bare string is not an array - ("tags", "[1]"), - ("link_properties", '["a"]'), - ("link_properties", '{ Verifies = ["verifies"] }'), # values too - ], - ) - def test_build_needs_wrong_types_are_rejected(self, tmp_path, key, value): - _write(tmp_path, f"[test_reports.build.needs]\n{key} = {value}\n") - with pytest.raises(TomlConfigError, match=f"build.needs.{key}"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_unknown_artifact_under_build_is_reported_but_not_fatal(self, tmp_path): - # `build` holds one table per artifact the command line produces. A - # newer command may produce one this version does not know, and a file - # naming it must not take the build down. - _write( - tmp_path, - """ - [test_reports.build.needs] - project = "p" - - [test_reports.build.graph] - format = "svg" - """, - ) - reported = [] - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) - assert needs_settings(section) == {"project": "p"} - assert section[BUILD_TABLE] == {"needs": {"project": "p"}} - assert len(reported) == 1 - assert "graph" in reported[0] - assert "[test_reports.build]" in reported[0] - - def test_a_non_table_needs_artifact_is_rejected(self, tmp_path): - _write(tmp_path, '[test_reports.build]\nneeds = "yes"\n') - with pytest.raises(TomlConfigError, match=r"build.needs"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_build_needs_unknown_key_is_reported_but_not_fatal(self, tmp_path): - _write(tmp_path, "[test_reports.build.needs]\nprojct = 'typo'\nproject = 'p'\n") - reported = [] - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) - assert needs_settings(section) == {"project": "p"} - assert len(reported) == 1 - assert "projct" in reported[0] - assert "[test_reports.build.needs]" in reported[0] - - def test_need_type_and_case_type_must_agree(self, tmp_path): - # The converter takes the need type (and the deterministic-ID prefix) - # from build.needs.need_type, the build from case's type. Disagreeing - # produces a needs.json the build neither registers nor cross-links. - _write( - tmp_path, - """ - [test_reports.build.needs] - need_type = "testcase" - - [test_reports.case] - directive = "test-case" - type = "check" - name = "Check" - prefix = "CH_" - color = "#999999" - style = "rectangle" - """, - ) - with pytest.raises(TomlConfigError, match="need_type"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_a_missing_side_is_compared_at_its_default(self, tmp_path): - # A customised case next to a convert table without need_type is a - # disagreement too: the converter would write the default type. - _write( - tmp_path, - """ - [test_reports.build.needs] - project = "p" - - [test_reports.case] - directive = "test-case" - type = "check" - name = "Check" - prefix = "CH_" - color = "#999999" - style = "rectangle" - """, - ) - with pytest.raises(TomlConfigError, match=r"'testcase'.*'check'"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - # ... and the mirror: need_type set, case left at its default. - _write(tmp_path, '[test_reports.build.needs]\nneed_type = "check"\n') - with pytest.raises(TomlConfigError, match=r"'check'.*'testcase'"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_without_a_convert_table_the_case_type_is_free(self, tmp_path): - # A project that only builds may name its case type as it likes. - _write( - tmp_path, - """ - [test_reports.case] - directive = "test-case" - type = "check" - name = "Check" - prefix = "CH_" - color = "#999999" - style = "rectangle" - """, - ) - assert ( - load_project_config(tmp_path / DEFAULT_TOML_FILENAME)["case"][1] == "check" - ) - - def test_empty_link_property_names_are_rejected_by_the_loader(self, tmp_path): - # One verdict for both consumers: the build refuses what the converter - # would refuse. - _write( - tmp_path, - '[test_reports.build.needs]\nlink_properties = { Verifies = "" }\n', - ) - with pytest.raises(TomlConfigError, match="link_properties"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_field_names_default_to_the_build_s(self, tmp_path): - _write(tmp_path, "[test_reports]\nsource_file_option = 'src'\n") - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - names = field_names(section) - assert names["source_file_option"] == "src" - assert names["file_option"] == DEFAULT_FIELD_NAMES["file_option"] == "file" - assert names["source_line_option"] == "case_line" - - def test_colliding_field_names_are_rejected(self, tmp_path): - # file (report path) and file (source path) cannot share a field. - _write(tmp_path, "[test_reports]\nsource_file_option = 'file'\n") - with pytest.raises(TomlConfigError, match="both name the need field 'file'"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - @pytest.mark.parametrize( - ("key", "name"), - [ - ("file_option", "case"), - ("source_file_option", "result"), - ("source_line_option", "id"), - ], - ) - def test_a_rename_onto_a_fixed_field_is_rejected(self, tmp_path, key, name): - # Every test-case need has these already: the directives would pass - # the keyword twice, the converter overwrite one value with the other. - _write(tmp_path, f"[test_reports]\n{key} = '{name}'\n") - with pytest.raises(TomlConfigError, match=f"{key} = '{name}'"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_need_type_and_case_type_agreeing_is_fine(self, tmp_path): - _write( - tmp_path, - """ - [test_reports.build.needs] - need_type = "check" - - [test_reports.case] - directive = "test-case" - type = "check" - name = "Check" - prefix = "CH_" - color = "#999999" - style = "rectangle" - """, - ) - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - assert section["case"][1] == needs_settings(section)["need_type"] == "check" - - def test_unknown_key_is_reported_but_not_fatal(self, tmp_path): - # ubproject.toml is shared with tools on independent release cadences, - # so a key this reader does not model must not take the build down -- - # but a typo has to be visible, and the key must not be passed on. - _write(tmp_path, "[test_reports]\ndeterministic_id = true\n") - reported = [] - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) - assert section == {} - assert len(reported) == 1 - assert "deterministic_id" in reported[0] - assert "deterministic_case_ids" in reported[0] # supported keys listed - - def test_unknown_key_needs_no_reporter(self, tmp_path): - _write(tmp_path, "[test_reports]\nnope = 1\nfile_option = 'f'\n") - assert load_project_config(tmp_path / DEFAULT_TOML_FILENAME) == { - "file_option": "f" - } - - @pytest.mark.parametrize( - ("key", "value"), - [ - ("property_link_types", '{ request = ["req"] }'), - ("property_link_types", "{ request = 3 }"), - ], - ) - def test_table_values_are_type_checked(self, tmp_path, key, value): - # Without this the value reaches the directives, which fail with a bare - # TypeError on an unhashable field name instead of a config error. - _write(tmp_path, f"[test_reports]\n{key} = {value}\n") - with pytest.raises(TomlConfigError, match=key): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_json_mapping_nesting_stays_free_form(self, tmp_path): - # It mirrors an arbitrary parser mapping, so only the outer table is - # checked -- validating deeper would reject valid configurations. - _write( - tmp_path, - "[test_reports.json_mapping.json_config.testsuite]\nname = 1\n", - ) - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - assert section["json_mapping"] == {"json_config": {"testsuite": {"name": 1}}} - - @pytest.mark.skipif( - hasattr(os, "geteuid") and os.geteuid() == 0, - reason="root reads unreadable files", - ) - @pytest.mark.skipif( - sys.platform == "win32", - reason="os.chmod on Windows only sets the read-only attribute; the file stays readable", - ) - def test_unreadable_file_is_a_config_error(self, tmp_path): - # is_file() succeeding does not mean the open will; an unwrapped - # OSError would surface as a traceback instead of a config error. - config = _write(tmp_path, "[test_reports]\nfile_option = 'f'\n") - config.chmod(0o000) - try: - with pytest.raises(TomlConfigError, match="cannot be read"): - load_project_config(config) - finally: - config.chmod(0o644) - - @pytest.mark.parametrize( - ("key", "value"), - [ - ("file_option", "42"), - ("suite_id_length", '"four"'), # string for int - ("suite_id_length", "true"), # bool must not pass for int - ("deterministic_case_ids", '"yes"'), # string for bool - ("extra_options", '"more_info"'), # a bare string is not an array - ("property_link_types", '["a"]'), - ], - ) - def test_wrong_types_are_rejected(self, tmp_path, key, value): - _write(tmp_path, f"[test_reports]\n{key} = {value}\n") - with pytest.raises(TomlConfigError, match=key): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_invalid_toml_is_rejected(self, tmp_path): - _write(tmp_path, "[test-reports\n") - with pytest.raises(TomlConfigError, match="invalid TOML"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_section_must_be_a_table(self, tmp_path): - _write(tmp_path, "test_reports = 5\n") - with pytest.raises(TomlConfigError, match="must be a table"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_need_type_positional_list_still_works(self, tmp_path): - # The conf.py spelling, so existing projects can copy their lists over - # verbatim. - _write( - tmp_path, - '[test_reports]\ncase = ["test-case", "testcase", "Test-Case", "TC_", "#999999", "rectangle"]\n', - ) - config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - assert config["case"] == [ - "test-case", - "testcase", - "Test-Case", - "TC_", - "#999999", - "rectangle", - ] - - def test_need_type_named_table(self, tmp_path): - # Six bare strings cannot be told apart; the table spelling names them. - _write( - tmp_path, - """ - [test_reports.case] - directive = "test-case" - type = "testcase" - name = "Test-Case" - prefix = "TC_" - color = "#999999" - style = "rectangle" - """, - ) - config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - assert config["case"] == [ - "test-case", - "testcase", - "Test-Case", - "TC_", - "#999999", - "rectangle", - ] - - def test_need_type_table_rejects_partial_and_unknown(self, tmp_path): - _write(tmp_path, '[test_reports.case]\ndirective = "test-case"\n') - with pytest.raises(TomlConfigError, match="missing"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - _write( - tmp_path, - """ - [test_reports.case] - directive = "test-case" - type = "testcase" - name = "Test-Case" - prefix = "TC_" - color = "#999999" - style = "rectangle" - typo = true - """, - ) - with pytest.raises(TomlConfigError, match="unknown typo"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - @pytest.mark.parametrize( - ("toml_source", "problem"), - [ - # A non-string value inside the named table. Without the check the - # value is silently stringified and reaches sphinx-needs. - ( - """ - [test_reports.case] - directive = "test-case" - type = "testcase" - name = "Test-Case" - prefix = "TC_" - color = 999999 - style = "rectangle" - """, - "non-string color", - ), - # A non-string element of the positional list. - ( - '[test_reports]\ncase = ["test-case", "testcase", "Test-Case", "TC_", 999999, "rectangle"]\n', - "exactly 6 strings", - ), - # Too few elements. - ('[test_reports]\ncase = ["test-case", "testcase"]\n', "exactly 6 strings"), - ], - ) - def test_need_type_values_must_be_strings(self, tmp_path, toml_source, problem): - _write(tmp_path, toml_source) - with pytest.raises(TomlConfigError, match=problem): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_relative_paths_anchor_to_the_toml_directory(self, tmp_path): - # The file is self-describing: moving it as a unit keeps its relative - # paths meaningful, and both consumers resolve them identically. - subdir = tmp_path / "config" - subdir.mkdir() - _write( - subdir, - '[test_reports]\nrootdir = "docs"\nreport_template = "templates/report.txt"\n', - name=subdir / DEFAULT_TOML_FILENAME, - ) - config = load_project_config(subdir / DEFAULT_TOML_FILENAME) - assert config["rootdir"] == str(subdir / "docs") - assert config["report_template"] == str(subdir / "templates" / "report.txt") - - @pytest.mark.parametrize( - "suffix", - [ - pytest.param("", id="plain"), - # the two forms ``Path`` normalises away: a round trip through it - # would return ``/a/b`` for ``/a/b/`` and ``/a/b`` for ``/a//b``, - # so these are the cases that tell "left as the string it was" - # from "anchored, and absolute already" - pytest.param(os.sep, id="trailing-separator"), - pytest.param(f"{os.sep}{os.sep}x", id="doubled-separator"), - ], - ) - def test_absolute_paths_stay_untouched(self, tmp_path, suffix): - # a TOML literal string: in a basic string a Windows path's backslashes - # are escape sequences ("\U" starts a unicode escape) and the file is invalid - value = f"{tmp_path}{suffix}" - _write(tmp_path, f"[test_reports]\nrootdir = '{value}'\n") - config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - assert config["rootdir"] == value - - def _not_utf8(tmp_path): """A file saved in Latin-1: ``é`` is the lone byte 0xE9, which UTF-8 refuses.""" config = tmp_path / DEFAULT_TOML_FILENAME @@ -486,65 +35,6 @@ def _not_utf8(tmp_path): return config -class TestSharedReaderBoundary: - """ub-project reads the file; its exception never leaves this package. - - Both consumers catch :class:`TomlConfigError` and nothing else, so a - ``ProjectConfigError`` escaping the loader would reach the user as a - traceback. Each case asserts the exact type and that the message is the - shared reader's, word for word. - """ - - def _assert_re_raised(self, config): - with pytest.raises(TomlConfigError) as caught: - load_project_config(config) - assert type(caught.value) is TomlConfigError - cause = caught.value.__cause__ - assert isinstance(cause, ProjectConfigError) - assert str(caught.value) == str(cause) - return str(caught.value) - - def test_the_two_exceptions_are_unrelated(self): - # a subclass either way round would put ub-project's exception on this - # package's public surface - assert not issubclass(TomlConfigError, ProjectConfigError) - assert not issubclass(ProjectConfigError, TomlConfigError) - - def test_the_walk_is_the_shared_reader_s_own(self): - # re-exported, not copied: a local fork of the walk would pass every - # discovery test and drift from the reader the other members use - assert find_project_config is ub_project.find_project_config - - def test_invalid_toml(self, tmp_path): - message = self._assert_re_raised(_write(tmp_path, "[test-reports\n")) - assert message.startswith(f"{tmp_path / DEFAULT_TOML_FILENAME}: invalid TOML: ") - - @pytest.mark.skipif( - hasattr(os, "geteuid") and os.geteuid() == 0, - reason="root reads unreadable files", - ) - @pytest.mark.skipif( - sys.platform == "win32", - reason="os.chmod on Windows only sets the read-only attribute; the file stays readable", - ) - def test_unreadable_file(self, tmp_path): - config = _write(tmp_path, "[test_reports]\nfile_option = 'f'\n") - config.chmod(0o000) - try: - message = self._assert_re_raised(config) - finally: - config.chmod(0o644) - assert message.startswith(f"{config}: cannot be read: ") - - def test_a_file_that_is_not_utf8(self, tmp_path): - # New with ub-project: before it, the decode error escaped the loader - # as a bare UnicodeDecodeError. - config = _not_utf8(tmp_path) - message = self._assert_re_raised(config) - assert message.startswith(f"{config}: not valid UTF-8 TOML: ") - - -@pytest.mark.toolchain class TestSphinxBridge: """The build reads the same section and honours the same precedence.""" @@ -628,184 +118,6 @@ def test_bridge_rejects_a_file_that_is_not_utf8(self, tmp_path): ) -class TestDiscovery: - """The upward search that lets both consumers find the same file. - - The search is bounded by the repository root -- the directory holding - ``.git``: a ``pyproject.toml`` on the way up marks a Python distribution, - not the project, and must not end the search. Outside any repository there - is no such root, so the distribution root bounds it instead -- otherwise - the walk reaches the filesystem root and adopts a stranger's file. - """ - - def test_finds_the_file_in_the_starting_directory(self, tmp_path): - config = _write(tmp_path, "[test_reports]\n") - assert find_project_config(tmp_path) == config - - def test_walks_up_to_the_repository_root(self, tmp_path): - (tmp_path / ".git").mkdir() - config = _write(tmp_path, "[test_reports]\n") - deep = tmp_path / "docs" / "source" - deep.mkdir(parents=True) - assert find_project_config(deep) == config - - def test_a_pyproject_toml_beside_conf_py_does_not_end_the_search(self, tmp_path): - # docs/ carrying its own pyproject.toml (its own dependency set) still - # belongs to the project whose shared file sits at the repository root. - (tmp_path / ".git").mkdir() - config = _write(tmp_path, "[test_reports]\n") - docs = tmp_path / "docs" - docs.mkdir() - (docs / "pyproject.toml").write_text("", encoding="utf-8") - assert find_project_config(docs) == config - - def test_walks_past_a_workspace_member_pyproject_toml(self, tmp_path): - # A uv-workspace member: packages//pyproject.toml with the docs - # below it, and one ubproject.toml at the repository root describing - # the whole monorepo. - (tmp_path / ".git").mkdir() - config = _write(tmp_path, "[test_reports]\n") - member = tmp_path / "packages" / "dist" - docs = member / "docs" - docs.mkdir(parents=True) - (member / "pyproject.toml").write_text("", encoding="utf-8") - assert find_project_config(docs) == config - - def test_stops_at_a_nested_repository_without_the_file(self, tmp_path): - # A checkout nested inside another repository (a vendored tree, a - # submodule) must not adopt the outer repository's configuration. - _write(tmp_path, "[test_reports]\n") - inner = tmp_path / "vendor" / "inner" - docs = inner / "docs" - docs.mkdir(parents=True) - (inner / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") - assert find_project_config(docs) is None - - def test_stops_at_the_distribution_root_without_a_repository(self, tmp_path): - # An unpacked sdist, a CI artefact directory, an exported docs tree: - # no .git anywhere, so nothing above would end the walk and a - # stranger's file further up would be adopted. The distribution root - # bounds the search instead, so it is not. - _write(tmp_path, "[test_reports]\n") # a stranger's, two levels up - dist = tmp_path / "downloads" / "sphinx-test-reports-1.4.0" - docs = dist / "docs" - docs.mkdir(parents=True) - (dist / "pyproject.toml").write_text("", encoding="utf-8") - assert find_project_config(docs) is None - - def test_the_file_at_the_distribution_root_is_still_found(self, tmp_path): - # The distribution root bounds the search without hiding a file that - # sits on it: an sdist shipping its own ubproject.toml is configured - # by it. - dist = tmp_path / "sphinx-test-reports-1.4.0" - docs = dist / "docs" - docs.mkdir(parents=True) - (dist / "pyproject.toml").write_text("", encoding="utf-8") - config = _write(dist, "[test_reports]\n") - assert find_project_config(docs) == config - - def test_a_repository_marker_outranks_a_distribution_root(self, tmp_path): - # The distribution root is only the fallback boundary. Inside a - # repository the walk still passes a pyproject.toml on the way up -- - # the workspace-member layout above depends on it. - (tmp_path / ".git").mkdir() - config = _write(tmp_path, "[test_reports]\n") - member = tmp_path / "packages" / "dist" - docs = member / "docs" - docs.mkdir(parents=True) - (member / "pyproject.toml").write_text("", encoding="utf-8") - assert find_project_config(docs) == config - - def test_a_fruitless_search_reports_the_distribution_root(self, tmp_path): - dist = tmp_path / "sphinx-test-reports-1.4.0" - docs = dist / "docs" - docs.mkdir(parents=True) - (dist / "pyproject.toml").write_text("", encoding="utf-8") - reported = [] - assert find_project_config(docs, report=reported.append) is None - assert len(reported) == 1 - assert f"distribution root {dist}" in reported[0] - assert "pyproject.toml" in reported[0] - - def test_the_file_wins_over_the_marker_in_one_directory(self, tmp_path): - # The root marker only ends a *fruitless* step; a repository root - # holding the file is the canonical layout and must be found. - config = _write(tmp_path, "[test_reports]\n") - (tmp_path / ".git").mkdir() - assert find_project_config(tmp_path) == config - - def test_missing_file_is_none(self, tmp_path): - (tmp_path / ".git").mkdir() - assert find_project_config(tmp_path) is None - - def test_a_fruitless_search_reports_where_it_ended(self, tmp_path): - # "Not found" must not be silent: the report names the directory whose - # marker ended the search, so a misplaced file can be diagnosed. - (tmp_path / ".git").mkdir() - docs = tmp_path / "docs" - docs.mkdir() - reported = [] - assert find_project_config(docs, report=reported.append) is None - assert len(reported) == 1 - assert str(docs) in reported[0] - assert f"repository root {tmp_path}" in reported[0] - assert ".git" in reported[0] - - def test_a_successful_search_reports_nothing(self, tmp_path): - (tmp_path / ".git").mkdir() - _write(tmp_path, "[test_reports]\n") - reported = [] - find_project_config(tmp_path / "docs", report=reported.append) - assert reported == [] - - def test_a_relative_start_is_searched_from_the_working_directory( - self, tmp_path, monkeypatch - ): - # A converter started with a relative path must still see the parents. - (tmp_path / ".git").mkdir() - config = _write(tmp_path, "[test_reports]\n") - docs = tmp_path / "docs" - docs.mkdir() - monkeypatch.chdir(docs) - assert find_project_config(Path(".")) == config - - def test_a_symlinked_start_walks_the_link_s_parents(self, tmp_path): - # The start is made absolute WITHOUT resolving: a symlinked docs/ - # belongs to the repository it is linked into, not to the one its - # target lives in. The only case here that tells the two apart -- - # tmp_path is already resolved, so every other start is too. - repo = tmp_path / "repo" - (repo / ".git").mkdir(parents=True) - config = _write(repo, "[test_reports]\n") - elsewhere = tmp_path / "elsewhere" - (elsewhere / ".git").mkdir(parents=True) - target = elsewhere / "docs" - target.mkdir() - link = repo / "docs" - try: - link.symlink_to(target, target_is_directory=True) - except OSError: # Windows without the symlink privilege - pytest.skip("creating a symlink needs a privilege this account lacks") - assert find_project_config(link) == config - - -class TestPathAnchoring: - """Relative paths anchor at the TOML file's directory, as given.""" - - def test_anchoring_does_not_resolve_the_given_directory(self, tmp_path): - # The loader leaves the form of the directory it was handed alone -- a - # symlinked path stays symlinked. Whether to resolve it is the - # consumer's call (Sphinx resolves its confdir before the bridge runs), - # not something the loader decides behind its back. - real = tmp_path / "real" - real.mkdir() - link = tmp_path / "link" - link.symlink_to(real, target_is_directory=True) - config = _write(link, "[test_reports]\nrootdir = 'reports'\n") - section = load_project_config(config) - assert section["rootdir"] == str(link / "reports") - - def _build(srcdir, **kwargs): """Set up a Sphinx application, returning it with its warning output. @@ -842,7 +154,6 @@ def _basic_doc(tmp_path, toml=None, conf_extra=""): return docs -@pytest.mark.toolchain class TestBridgePrecedence: """``-D`` > TOML > conf.py, and the diagnostics for a file that is missing.""" @@ -950,7 +261,6 @@ def _documented_toml_example(): return "\n".join(block) + "\n" -@pytest.mark.toolchain class TestConfvalTypes: """The bridged values must pass Sphinx's own confval type check. @@ -986,54 +296,3 @@ def test_the_documented_example_applies_without_warnings(self, tmp_path): assert app.config.tr_file_option == "report_file" assert app.config.tr_rootdir == str(tmp_path / "docs") assert app.config.tr_case[0] == "test-case" - - -class TestSphinxFree: - """A consumer without the documentation toolchain can read the section.""" - - def test_projectconfig_imports_without_sphinx(self): - # Importing the module runs the package __init__, so the package must - # not import Sphinx eagerly either -- or a build action that turns - # reports into a needs.json dies with ModuleNotFoundError wherever - # Sphinx is not installed. Checked in a subprocess: this process has - # Sphinx imported already. - code = ( - "import sys\n" - "sys.modules['sphinx'] = None\n" # any `import sphinx...` now fails - "import sphinx_test_reports.projectconfig\n" - ) - result = subprocess.run( - [sys.executable, "-c", code], capture_output=True, text=True - ) - assert result.returncode == 0, result.stderr - - @pytest.mark.toolchain - def test_a_missing_sphinx_needs_is_an_extension_error(self): - # The lazy `setup` owns the message Sphinx would have produced for a - # broken extension import, because Sphinx fetches `setup` with - # getattr() and would otherwise show a raw traceback. Sphinx renders - # the wrapped exception itself, so the message must not carry it a - # second time -- but it names the extra that installs the toolchain, - # the likely cause since the toolchain stopped being a dependency. - # Checked in a subprocess: sphinx_needs is importable here. - code = ( - "import sys\n" - "sys.modules['sphinx_needs'] = None\n" # `from sphinx_needs...` fails - "import sphinx_test_reports as pkg\n" - "from sphinx.errors import ExtensionError\n" - "try:\n" - " pkg.setup\n" - "except ExtensionError as error:\n" - " print(error)\n" - "else:\n" - " raise AssertionError('no ExtensionError')\n" - ) - result = subprocess.run( - [sys.executable, "-c", code], capture_output=True, text=True - ) - assert result.returncode == 0, result.stderr - message = result.stdout.strip() - assert message.startswith("Could not import extension sphinx_test_reports") - assert message.count("(exception:") == 1 - assert "sphinx_needs" in message - assert 'pip install "sphinx-test-reports[sphinx]"' in message diff --git a/packages/sphinx-test-reports/tests/test_properties.py b/packages/sphinx-test-reports/tests/test_properties.py index 345e4d6a9..f43f741c3 100644 --- a/packages/sphinx-test-reports/tests/test_properties.py +++ b/packages/sphinx-test-reports/tests/test_properties.py @@ -25,7 +25,7 @@ class TestParserExtractsTestcaseProperties: """JUnitParser must extract from elements.""" def test_testcase_with_properties_returns_dict(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -38,7 +38,7 @@ def test_testcase_with_properties_returns_dict(self): assert tc["properties"]["priority"] == "high" def test_testcase_with_single_property(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -49,7 +49,7 @@ def test_testcase_with_single_property(self): assert tc["properties"] == {"verifies": "REQ_AUTH_003"} def test_testcase_without_properties_returns_empty_dict(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -60,7 +60,7 @@ def test_testcase_without_properties_returns_empty_dict(self): assert tc["properties"] == {} def test_testcase_skipped_without_properties_returns_empty_dict(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -72,7 +72,7 @@ def test_testcase_skipped_without_properties_returns_empty_dict(self): assert tc["properties"] == {} def test_multiple_suites_testcase_properties(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -84,7 +84,7 @@ def test_multiple_suites_testcase_properties(self): assert tc["properties"]["category"] == "integration" def test_testcase_without_any_properties_element(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -99,7 +99,7 @@ class TestParserExtractsTestsuiteProperties: """JUnitParser must extract from elements.""" def test_testsuite_with_properties(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -112,7 +112,7 @@ def test_testsuite_with_properties(self): assert suite["properties"]["build_id"] == "build-7742" def test_testsuite_without_properties(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -127,7 +127,7 @@ class TestParserBackwardCompatibility: """Adding properties extraction must not break existing XML without properties.""" def test_existing_xml_still_parses(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_no_properties_path) results = parser.parse() @@ -141,7 +141,7 @@ def test_existing_xml_still_parses(self): assert tc["properties"] == {} def test_existing_testcase_fields_unchanged(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_no_properties_path) results = parser.parse() @@ -152,7 +152,7 @@ def test_existing_testcase_fields_unchanged(self): assert tc["result"] == "passed" def test_existing_failure_testcase_unchanged(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_no_properties_path) results = parser.parse() @@ -271,7 +271,7 @@ class TestParserHandlesEmptyProperties: """JUnitParser must not crash on empty elements.""" def test_empty_testsuite_properties_element(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_empty_properties_path) results = parser.parse() @@ -281,7 +281,7 @@ def test_empty_testsuite_properties_element(self): assert suite["properties"] == {} def test_empty_testcase_properties_element(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_empty_properties_path) results = parser.parse() @@ -291,7 +291,7 @@ def test_empty_testcase_properties_element(self): assert tc["properties"] == {} def test_testcase_without_properties_alongside_empty(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_empty_properties_path) results = parser.parse() diff --git a/packages/sphinx-test-reports/tests/test_result_vocabulary.py b/packages/sphinx-test-reports/tests/test_result_vocabulary.py index 518b552af..a89b93088 100644 --- a/packages/sphinx-test-reports/tests/test_result_vocabulary.py +++ b/packages/sphinx-test-reports/tests/test_result_vocabulary.py @@ -13,145 +13,14 @@ participles, and the mapping is the only place that decides. The part-level ``kind`` is deliberately *not* normalised -- it names the XML element the evidence came from, which is what the rendered evidence heading reports. -""" -import os +This is the half that needs Sphinx: the shipped report template rendering the vocabulary. +The parsers' half is ub-test-reports', ``packages/ub-test-reports/tests/test_result_vocabulary.py``. +""" import pytest -UTILS = os.path.join(os.path.dirname(__file__), "doc_test", "utils") -XML_PATH = os.path.join(UTILS, "xml_data.xml") -JSON_PATH = os.path.join(UTILS, "json_data.json") - -#: The mapping the JSON parser needs, as ``tr_json_mapping`` declares it. -JSON_MAPPING = { - "testsuite": { - "name": (["name"], "unknown"), - "tests": (["tests"], "unknown"), - "errors": (["errors"], "unknown"), - "failures": (["failures"], "unknown"), - "skips": (["skips"], "unknown"), - "passed": (["passed"], "unknown"), - "time": (["time"], "unknown"), - "testcases": (["testcase"], "unknown"), - }, - "testcase": { - "name": (["name"], "unknown"), - "classname": (["classname"], "unknown"), - "file": (["file"], "unknown"), - "line": (["line"], "unknown"), - "time": (["time"], "unknown"), - "result": (["result"], "unknown"), - "type": (["type"], "unknown"), - "text": (["text"], "unknown"), - "message": (["message"], "unknown"), - "system-out": (["system-out"], "unknown"), - }, -} - - -class TestNormalisation: - """One function decides the vocabulary, so both parsers cannot disagree.""" - - def test_the_junit_failure_element_name_becomes_failed(self): - from sphinx_test_reports.results import normalize_result - - assert normalize_result("failure") == "failed" - - def test_normalising_the_canonical_spelling_changes_nothing(self): - """Normalisation runs on already-canonical values too, so it must be - idempotent -- a JSON report may already spell the result ``failed``.""" - from sphinx_test_reports.results import normalize_result - - assert normalize_result("failed") == "failed" - - @pytest.mark.parametrize("result", ["passed", "skipped", "error", "disabled"]) - def test_the_other_states_are_already_canonical(self, result): - from sphinx_test_reports.results import normalize_result - - assert normalize_result(result) == result - - def test_a_vocabulary_this_extension_does_not_know_is_left_alone(self): - """``tr_json_mapping`` points at an arbitrary report, so a project may - feed in states of its own. Rewriting those would break its filters.""" - from sphinx_test_reports.results import normalize_result - - assert normalize_result("flaky") == "flaky" - - def test_the_canonical_states_are_the_documented_ones(self): - """Ordered, because the declared field description is built from it and - the converter's output has to be byte-stable.""" - from sphinx_test_reports.results import CANONICAL_RESULTS - - assert CANONICAL_RESULTS == ( - "passed", - "failed", - "error", - "skipped", - "disabled", - ) - - -class TestDeclaredSchema: - """The converter writes the field declarations into the ``needs.json`` it - produces, so that a consumer which never loads this extension -- a schema - check, a metamodel validator -- learns the fields from the file. Naming the - states in the ``result`` description tells it the field's domain too. - """ - - def test_the_result_declaration_names_every_state(self): - from sphinx_test_reports.fields import declaration - from sphinx_test_reports.results import CANONICAL_RESULTS - - _, description = declaration("result") - - assert all(state in description for state in CANONICAL_RESULTS), description - - -class TestJUnitParser: - """The JUnit dialect is where the old spelling came from.""" - - def test_a_failure_child_yields_the_failed_result(self): - from sphinx_test_reports.junitparser import JUnitParser - - suite = JUnitParser(XML_PATH).parse()[0] - - assert suite["testcases"][2]["result"] == "failed" - - def test_a_result_part_keeps_the_name_of_its_xml_element(self): - """``kind`` reports which element the evidence came from -- the - converter capitalises it into the evidence heading -- so it stays the - XML name even though ``result`` no longer is.""" - from sphinx_test_reports.junitparser import JUnitParser - - suite = JUnitParser(XML_PATH).parse()[0] - - assert suite["testcases"][2]["parts"][0]["kind"] == "failure" - - -class TestJsonParser: - """The JSON parser's API is documented as being in sync with the JUnit - parser's, so the same report content has to produce the same result.""" - - def test_the_failure_spelling_in_a_json_report_is_normalised(self): - from sphinx_test_reports.jsonparser import JsonParser - - parser = JsonParser(JSON_PATH, json_mapping=JSON_MAPPING) - suite = parser.parse()[0] - - assert suite["testcases"][0]["result"] == "failed" - - def test_the_results_needing_no_normalisation_are_untouched(self): - from sphinx_test_reports.jsonparser import JsonParser - - parser = JsonParser(JSON_PATH, json_mapping=JSON_MAPPING) - suite = parser.parse()[0] - - assert suite["testcases"][1]["result"] == "passed" - assert suite["testcases"][2]["result"] == "skipped" - -@pytest.mark.toolchain @pytest.mark.parametrize( "test_app", [{"buildername": "html", "srcdir": "doc_test/default_tr_template"}], diff --git a/packages/sphinx-test-reports/tests/test_toolchain.py b/packages/sphinx-test-reports/tests/test_toolchain.py deleted file mode 100644 index 34629236a..000000000 --- a/packages/sphinx-test-reports/tests/test_toolchain.py +++ /dev/null @@ -1,158 +0,0 @@ -"""The documentation toolchain is an extra; the extension checks it at load time. - -``pip install sphinx-test-reports`` installs the converter's dependency only. -The Sphinx extension needs Sphinx and sphinx-needs at the versions the -``sphinx`` extra declares -- and an extra is opt-in, so a project that installs -the bare package next to an older toolchain never shows pip those floors. The -lazy ``setup`` compares the installed versions against the extra's declarations -when Sphinx loads the extension, and names the install line in its message. -""" - -import sys -from importlib.metadata import PackageNotFoundError -from io import StringIO - -import pytest - -import sphinx_test_reports as package -from sphinx_test_reports import toolchain - -DECLARED = [ - "lxml", - 'sphinx>=7.4; extra == "sphinx"', - 'sphinx-needs>=6.0.1; extra == "sphinx"', - 'pytest>=7.0; extra == "pytest"', - 'pytest>=7.0; extra == "test"', -] - -AT_THE_FLOORS = {"sphinx": "7.4.7", "sphinx-needs": "6.0.1"} - -VIOLATION = "sphinx-needs 5.1.0 is installed, but sphinx-needs>=6.0.1 is required" - - -def _environment(monkeypatch, installed, declared=DECLARED): - """Fake the metadata: what this package declares and what is installed.""" - - def requires(name): - assert name == toolchain.DISTRIBUTION - return declared - - def version(name): - try: - return installed[name] - except KeyError: - raise PackageNotFoundError(name) from None - - monkeypatch.setattr(toolchain, "requires", requires) - monkeypatch.setattr(toolchain, "version", version) - - -class TestUnmetRequirements: - def test_a_toolchain_at_the_floors_is_accepted(self, monkeypatch): - _environment(monkeypatch, AT_THE_FLOORS) - assert toolchain.unmet_requirements() == [] - - def test_a_version_below_the_floor_is_named_with_the_floor(self, monkeypatch): - _environment(monkeypatch, {**AT_THE_FLOORS, "sphinx-needs": "5.1.0"}) - assert toolchain.unmet_requirements() == [VIOLATION] - - def test_every_violation_is_reported_in_declaration_order(self, monkeypatch): - _environment(monkeypatch, {"sphinx": "7.3.7", "sphinx-needs": "5.1.0"}) - assert toolchain.unmet_requirements() == [ - "sphinx 7.3.7 is installed, but sphinx>=7.4 is required", - VIOLATION, - ] - - def test_a_missing_distribution_is_left_to_the_import(self, monkeypatch): - # The import that fails on a missing package states the problem more - # precisely -- and a toolchain importable from a source tree without - # metadata must not be refused on the strength of the metadata alone. - _environment(monkeypatch, {"sphinx": "7.4.7"}) - assert toolchain.unmet_requirements() == [] - - def test_other_extras_and_the_core_dependency_are_not_checked(self, monkeypatch): - _environment(monkeypatch, {**AT_THE_FLOORS, "lxml": "0.1", "pytest": "1.0"}) - assert toolchain.unmet_requirements() == [] - - def test_a_prerelease_above_the_floor_is_accepted(self, monkeypatch): - _environment(monkeypatch, {**AT_THE_FLOORS, "sphinx-needs": "9.0.0rc1"}) - assert toolchain.unmet_requirements() == [] - - def test_without_our_own_metadata_there_is_nothing_to_check(self, monkeypatch): - # Run from a source tree on sys.path, the package has no metadata. - def requires(name): - raise PackageNotFoundError(name) - - monkeypatch.setattr(toolchain, "requires", requires) - assert toolchain.unmet_requirements() == [] - - def test_without_packaging_the_check_is_skipped(self, monkeypatch): - # `packaging` is not a dependency of this package: it arrives with - # Sphinx (and with pytest). Where neither is installed, nothing loads - # the extension either -- so the check stands down instead of failing - # on its own import. - _environment(monkeypatch, {"sphinx": "7.3.7", "sphinx-needs": "5.1.0"}) - monkeypatch.setitem(sys.modules, "packaging.requirements", None) - assert toolchain.unmet_requirements() == [] - - def test_the_installed_metadata_declares_the_toolchain_under_the_extra(self): - # Against the real metadata: the extra the check reads is the one - # pyproject.toml declares. Renaming it there would silently disarm the - # check, and this test. - names = sorted( - requirement.name for requirement in toolchain.toolchain_requirements() - ) - assert names == ["docutils", "sphinx", "sphinx-needs"] - - def test_this_environment_meets_the_floors(self): - assert toolchain.unmet_requirements() == [] - - -@pytest.mark.toolchain -class TestLazySetup: - """Sphinx resolves ``setup`` through getattr(); the check runs first.""" - - def test_a_sufficient_toolchain_resolves_setup(self): - from sphinx_test_reports.test_reports import setup - - assert package.setup is setup - - def test_a_toolchain_below_the_floors_is_an_extension_error(self, monkeypatch): - from sphinx.errors import ExtensionError - - monkeypatch.setattr(toolchain, "unmet_requirements", lambda: [VIOLATION]) - with pytest.raises(ExtensionError) as info: - _ = package.setup - message = str(info.value) - assert message.startswith("Could not load extension sphinx_test_reports") - assert VIOLATION in message - assert toolchain.INSTALL_HINT in message - # Nothing was imported, so there is no exception to wrap; the message - # must not end in an empty "(exception: ...)". - assert "(exception:" not in message - - def test_sphinx_surfaces_the_error_when_loading_the_extension( - self, tmp_path, monkeypatch - ): - # Sphinx fetches `setup` with getattr(module, "setup", None), which - # only swallows AttributeError. The error must reach the user as the - # extension error it is, install line included -- not as "extension - # has no setup() function". - from sphinx.application import Sphinx - from sphinx.errors import ExtensionError - - monkeypatch.setattr(toolchain, "unmet_requirements", lambda: [VIOLATION]) - (tmp_path / "conf.py").write_text( - 'extensions = ["sphinx_test_reports"]\n', encoding="utf-8" - ) - (tmp_path / "index.rst").write_text("Index\n=====\n", encoding="utf-8") - with pytest.raises(ExtensionError, match=r"sphinx-test-reports\[sphinx\]"): - Sphinx( - srcdir=tmp_path, - confdir=tmp_path, - outdir=tmp_path / "_build", - doctreedir=tmp_path / "_doctrees", - buildername="html", - status=None, - warning=StringIO(), - ) diff --git a/packages/ub-test-reports/tests/__init__.py b/packages/ub-test-reports/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/ub-test-reports/tests/conftest.py b/packages/ub-test-reports/tests/conftest.py new file mode 100644 index 000000000..077eba3a3 --- /dev/null +++ b/packages/ub-test-reports/tests/conftest.py @@ -0,0 +1,9 @@ +"""ub-test-reports' suite needs no Sphinx: only pytest's own ``pytester``. + +It runs in the default environment and, in CI's ``toolchain-free`` and ``plugin-floor`` +jobs, against the BUILT wheel in an environment where Sphinx, sphinx-needs and docutils are +absent -- so nothing here may import them, and the fixtures it reads are this package's own +copies under ``tests/fixtures/``. +""" + +pytest_plugins = ["pytester"] diff --git a/packages/ub-test-reports/tests/fixtures/ctest.xml b/packages/ub-test-reports/tests/fixtures/ctest.xml new file mode 100644 index 000000000..bf757979d --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/ctest.xml @@ -0,0 +1,37 @@ + + + + usage: test <argument> + If <argument> is 0, print SUCCESS. Otherwise print FAIL. + + + + SUCCESS + + + + FAIL + + + + + FAIL + + + + + Skipped feature due to @skip tag +0 features passed, 0 failed, 1 skipped +0 scenarios passed, 0 failed, 14 skipped +0 steps passed, 0 failed, 42 skipped, 0 undefined + + + diff --git a/packages/ub-test-reports/tests/fixtures/gtest_data.xml b/packages/ub-test-reports/tests/fixtures/gtest_data.xml new file mode 100644 index 000000000..0c6ac8daf --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/gtest_data.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/ub-test-reports/tests/fixtures/json_complex_data.json b/packages/ub-test-reports/tests/fixtures/json_complex_data.json new file mode 100644 index 000000000..938ec42d1 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/json_complex_data.json @@ -0,0 +1,53 @@ +[ + { + "internals": { + "name": "test suite 1" + }, + "tests": 3, + "errors": 0, + "failures": 1, + "skips": 1, + "passed": 1, + "time": "20230515-15:04:32", + "testcase": { + "nested": [ + { + "name": "test case 1", + "classname": "class name 1", + "file": "my/test/file_1", + "line": 123, + "time": "20230515-15:04:32", + "result": "failure", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + }, + { + "name": "test case 2", + "classname": "class name 2", + "file": "my/test/file_2", + "line": 123, + "time": "20230515-15:04:32", + "result": "passed", + "type": "test type", + "text": "test text", + "message": "all went great :) (message)", + "system-out": "system-out text" + }, + { + "name": "test case 3", + "classname": "class name 3", + "file": "my/test/file_3", + "line": 123, + "time": "20230515-15:04:32", + "result": "skipped", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + } + ] + } + } +] diff --git a/packages/ub-test-reports/tests/fixtures/json_custom_data.json b/packages/ub-test-reports/tests/fixtures/json_custom_data.json new file mode 100644 index 000000000..098564c35 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/json_custom_data.json @@ -0,0 +1,72 @@ +[ + { + "name": "test suite 1", + "tests": 3, + "errors": 0, + "failures": 1, + "skips": 1, + "passed": 1, + "time": "20230515-15:04:32", + "testcase": [ + { + "name": "test case 1", + "classname": "class name 1", + "file": "my/test/file_1", + "line": 123, + "time": "20230515-15:04:32", + "result": "failure", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text", + "id": "TEST_CASE_1", + "status": "open", + "tags": "a,b,c", + "priority": "1" + }, + { + "name": "test case 2", + "classname": "class name 2", + "file": "my/test/file_2", + "line": 123, + "time": "20230515-15:04:32", + "result": "passed", + "type": "test type", + "text": "test text", + "message": "all went great :) (message)", + "system-out": "system-out text", + "id": "TEST_CASE_2", + "status": "closed", + "tags": "a,b", + "priority": "2" + }, + { + "name": "test case 3", + "classname": "class name 3", + "file": "my/test/file_3", + "line": 123, + "time": "20230515-15:04:32", + "result": "skipped", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text", + "id": "TEST_CASE_3", + "status": "", + "tags": "a" + }, + { + "name": "test case 4", + "classname": "class name 4", + "file": "my/test/file_3", + "line": 123, + "time": "20230515-15:04:32", + "result": "skipped", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + } + ] + } +] diff --git a/packages/ub-test-reports/tests/fixtures/json_data.json b/packages/ub-test-reports/tests/fixtures/json_data.json new file mode 100644 index 000000000..2ef48b783 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/json_data.json @@ -0,0 +1,49 @@ +[ + { + "name": "test suite 1", + "tests": 3, + "errors": 0, + "failures": 1, + "skips": 1, + "passed": 1, + "time": "20230515-15:04:32", + "testcase": [ + { + "name": "test case 1", + "classname": "class name 1", + "file": "my/test/file_1", + "line": 123, + "time": "20230515-15:04:32", + "result": "failure", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + }, + { + "name": "test case 2", + "classname": "class name 2", + "file": "my/test/file_2", + "line": 123, + "time": "20230515-15:04:32", + "result": "passed", + "type": "test type", + "text": "test text", + "message": "all went great :) (message)", + "system-out": "system-out text" + }, + { + "name": "test case 3", + "classname": "class name 3", + "file": "my/test/file_3", + "line": 123, + "time": "20230515-15:04:32", + "result": "skipped", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + } + ] + } +] diff --git a/packages/ub-test-reports/tests/fixtures/nose_data.xml b/packages/ub-test-reports/tests/fixtures/nose_data.xml new file mode 100644 index 000000000..1117f63b6 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/nose_data.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/packages/ub-test-reports/tests/fixtures/pytest_data.xml b/packages/ub-test-reports/tests/fixtures/pytest_data.xml new file mode 100644 index 000000000..e7a1495f1 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/pytest_data.xml @@ -0,0 +1,52 @@ + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a8a0e950> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a8a0ea10> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a8497450> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a8497c10> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84a2350> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84a2a50> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84a2cd0> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84ae250> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84ae950> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84bf090> + + + diff --git a/packages/ub-test-reports/tests/fixtures/pytest_data_5_1.xml b/packages/ub-test-reports/tests/fixtures/pytest_data_5_1.xml new file mode 100644 index 000000000..1a429b924 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/pytest_data_5_1.xml @@ -0,0 +1,93 @@ + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a8a0e950> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a8a0ea10> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a8497450> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a8497c10> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a84a2350> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a84a2a50> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a84a2cd0> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a84ae250> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a84ae950> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a84bf090> + + + + diff --git a/packages/ub-test-reports/tests/fixtures/pytest_data_6_2.xml b/packages/ub-test-reports/tests/fixtures/pytest_data_6_2.xml new file mode 100644 index 000000000..37bbc8f2f --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/pytest_data_6_2.xml @@ -0,0 +1,29 @@ + + + + + + def test_fail (): +> assert False +E assert False + +test_example.py:11: AssertionError + + + ~/test_example.py:13: An example reason + + + ~/test_example.py:17: An example reason + + + + + + def test_error(): +> raise Exception("Bang!") +E Exception: Bang! + +test_example.py:31: Exception + + + diff --git a/packages/ub-test-reports/tests/fixtures/pytest_nested_example.xml b/packages/ub-test-reports/tests/fixtures/pytest_nested_example.xml new file mode 100644 index 000000000..b240964ac --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/pytest_nested_example.xml @@ -0,0 +1,40 @@ + + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a8a0e950> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a8a0ea10> + + + + + + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84a2cd0> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84ae250> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84ae950> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84bf090> + + + + diff --git a/packages/ub-test-reports/tests/fixtures/runner_error_data.xml b/packages/ub-test-reports/tests/fixtures/runner_error_data.xml new file mode 100644 index 000000000..0bcf5a654 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/runner_error_data.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + diff --git a/packages/ub-test-reports/tests/fixtures/xml_data.xml b/packages/ub-test-reports/tests/fixtures/xml_data.xml new file mode 100644 index 000000000..135ca7003 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/xml_data.xml @@ -0,0 +1,7 @@ + + + + + details about failure + + diff --git a/packages/ub-test-reports/tests/fixtures/xml_data_error.xml b/packages/ub-test-reports/tests/fixtures/xml_data_error.xml new file mode 100644 index 000000000..1475938a2 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/xml_data_error.xml @@ -0,0 +1,12 @@ + + + + some failure details + + + stack trace here + + + + + diff --git a/packages/sphinx-test-reports/tests/test_cli_config.py b/packages/ub-test-reports/tests/test_cli_config.py similarity index 99% rename from packages/sphinx-test-reports/tests/test_cli_config.py rename to packages/ub-test-reports/tests/test_cli_config.py index 13f2ba725..ebfd864ff 100644 --- a/packages/sphinx-test-reports/tests/test_cli_config.py +++ b/packages/ub-test-reports/tests/test_cli_config.py @@ -11,13 +11,13 @@ import pytest -from sphinx_test_reports.cli import _DEFAULTS, main -from sphinx_test_reports.projectconfig import ( +from ub_test_reports.cli import _DEFAULTS, main +from ub_test_reports.projectconfig import ( CONVERSION_KEYS, DEFAULT_TOML_FILENAME, ) -UTILS = Path(__file__).parent / "doc_test" / "utils" +UTILS = Path(__file__).parent / "fixtures" PYTEST_XML = str(UTILS / "pytest_data.xml") GTEST_XML = str(UTILS / "gtest_data.xml") # carries elements diff --git a/packages/ub-test-reports/tests/test_cli_convert.py b/packages/ub-test-reports/tests/test_cli_convert.py new file mode 100644 index 000000000..8a942a950 --- /dev/null +++ b/packages/ub-test-reports/tests/test_cli_convert.py @@ -0,0 +1,599 @@ +"""Tests for the Sphinx-free ``test-reports build needs`` CLI (TR-A). + +This is the keystone of the build-system story: a test-XML to needs.json +conversion that runs as a build action *outside* Sphinx, so the docs build only +imports the result. Two properties are load-bearing and therefore tested +explicitly rather than assumed: + +* the CLI must not import Sphinx -- otherwise a Bazel action pulls the whole + documentation toolchain into the test-result conversion; +* the output must be byte-stable, because it is a cached build artifact and + qualification evidence. +""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +UTILS = Path(__file__).parent / "fixtures" +GTEST_XML = UTILS / "gtest_data.xml" +PYTEST_XML = UTILS / "pytest_data.xml" + + +def _convert(tmp_path, *args, xml=GTEST_XML): + """Run the converter and return (exit_code, parsed output or None).""" + from ub_test_reports.cli import main + + output = tmp_path / "needs.json" + code = main(["build", "needs", str(xml), "--output", str(output), *args]) + data = json.loads(output.read_text(encoding="utf-8")) if output.exists() else None + return code, data + + +def _needs(data): + version = data["current_version"] + return data["versions"][version]["needs"] + + +class TestNoSphinxImport: + """The converter has to be usable without the documentation toolchain.""" + + def test_importing_the_cli_does_not_import_sphinx(self): + script = ( + "import sys;" + "import ub_test_reports.cli;" + "leaked = sorted(m for m in sys.modules" + " if m == 'sphinx' or m.startswith(('sphinx.', 'sphinx_needs')));" + "print(','.join(leaked))" + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=True, + ) + + assert result.stdout.strip() == "" + + +class TestEnvelope: + def test_envelope_carries_project_and_current_version(self, tmp_path): + _, data = _convert(tmp_path, "--project", "Score Docs-as-Code") + + assert data["project"] == "Score Docs-as-Code" + assert data["current_version"] in data["versions"] + + def test_needs_amount_matches_the_number_of_cases(self, tmp_path): + _, data = _convert(tmp_path) + + version = data["versions"][data["current_version"]] + assert version["needs_amount"] == len(version["needs"]) == 5 + + def test_every_written_field_is_declared(self, tmp_path): + """The file says what its fields are, without a Sphinx build to ask.""" + _, data = _convert(tmp_path) + + version = data["versions"][data["current_version"]] + declared = set(version["needs_schema"]["properties"]) + written = {key for need in version["needs"].values() for key in need} + + assert written - declared == set() + + def test_no_timestamp_is_written(self, tmp_path): + """A wall clock would defeat action caching and evidence diffs.""" + _, data = _convert(tmp_path) + + assert "created" not in data + assert "created" not in data["versions"][data["current_version"]] + + def test_output_is_byte_stable_across_runs(self, tmp_path): + from ub_test_reports.cli import main + + first = tmp_path / "first.json" + second = tmp_path / "second.json" + for output in (first, second): + assert ( + main(["build", "needs", str(GTEST_XML), "--output", str(output)]) == 0 + ) + + assert first.read_bytes() == second.read_bytes() + + +class TestNeedContent: + def test_ids_match_the_deterministic_scheme(self, tmp_path): + from ub_test_reports.identity import deterministic_case_id + + _, data = _convert(tmp_path) + + expected = deterministic_case_id( + classname="MathTest", name="Addition", file="src/math_test.cc" + ) + assert expected in _needs(data) + + def test_source_location_is_emitted_verbatim(self, tmp_path): + _, data = _convert(tmp_path) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["case_file"] == "src/math_test.cc" + assert need["case_line"] == "12" + # `file` is the report path, as in a locally created test-case need. + assert need["file"] == str(GTEST_XML) + + def test_type_and_title_match_the_directive_s(self, tmp_path): + # The directive titles a case need with the case name; a needtable + # title column must not tell an imported case from a built one. + _, data = _convert(tmp_path) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["type"] == "testcase" + assert need["title"] == "Addition" + + def test_result_vocabulary_includes_disabled(self, tmp_path): + _, data = _convert(tmp_path) + + need = _needs(data)["testcase__MathTest__DISABLED_Division_jnyzp"] + assert need["result"] == "disabled" + + def test_content_keeps_every_failure_part(self, tmp_path): + """R2: the debug output has to survive the conversion.""" + _, data = _convert(tmp_path) + + content = _needs(data)["testcase__MathTest__Subtraction_srmht"]["content"] + assert "Expected equality of these values" in content + assert "Actual: false" in content + assert "overflow guard hit" in content + + def test_result_text_is_the_first_failure_message(self, tmp_path): + _, data = _convert(tmp_path) + + need = _needs(data)["testcase__MathTest__Subtraction_srmht"] + assert need["result_text"].startswith("src/math_test.cc:22") + assert "\n" not in need["result_text"] + + def test_named_properties_become_fields(self, tmp_path, capsys): + # Only properties the build would accept (extra_options, or here the + # flag standing in for it) become fields; the rest is reported once. + code, data = _convert(tmp_path, "--no-config", "--extra-option", "TestType") + assert code == 0 + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["TestType"] == "requirements-based" + assert "PartiallyVerifies" not in need + message = capsys.readouterr().err + assert "properties not exported" in message + assert "PartiallyVerifies" in message + assert "extra_options" in message + + def test_unexported_properties_are_reported_once_for_all_names( + self, tmp_path, capsys + ): + # One line naming every left-out property -- not one line per name, + # and not one per case that carries it. + code, _ = _convert(tmp_path, "--no-config") + assert code == 0 + lines = [ + line + for line in capsys.readouterr().err.splitlines() + if "properties not exported" in line + ] + assert len(lines) == 1 + assert "PartiallyVerifies" in lines[0] and "TestType" in lines[0] + + def test_an_exported_property_is_present_on_every_case(self, tmp_path): + # Null where the case has no such property, as the build leaves a + # registered field a directive did not set -- so one schema can + # require the field of imported and locally created needs alike. + _, data = _convert(tmp_path, "--no-config", "--extra-option", "TestType") + needs = _needs(data) + assert needs["testcase__MathTest__Addition_hcuyy"]["TestType"] == ( + "requirements-based" + ) + assert needs["testcase__MathTest__Subtraction_srmht"]["TestType"] is None + assert all("TestType" in need for need in needs.values()) + + def test_unnamed_properties_are_left_out_quietly_when_none_exist( + self, tmp_path, capsys + ): + code, _ = _convert(tmp_path, "--no-config", xml=PYTEST_XML) + assert code == 0 + assert "properties not exported" not in capsys.readouterr().err + + def test_tags_are_configurable(self, tmp_path): + _, data = _convert(tmp_path, "--tags", "TEST") + + assert _needs(data)["testcase__MathTest__Addition_hcuyy"]["tags"] == ["TEST"] + + +class TestLinkProperties: + def test_a_property_can_be_promoted_to_a_link_field(self, tmp_path): + _, data = _convert( + tmp_path, "--link-property", "PartiallyVerifies=partially_verifies" + ) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["partially_verifies"] == ["REQ_1", "REQ_2"] + assert "PartiallyVerifies" not in need + + def test_link_fields_are_always_present_even_when_empty(self, tmp_path): + """A converter must emit its fields unconditionally, so schemas can require them.""" + _, data = _convert( + tmp_path, "--link-property", "PartiallyVerifies=partially_verifies" + ) + + need = _needs(data)["testcase__MathTest__DISABLED_Division_jnyzp"] + assert need["partially_verifies"] == [] + + def test_malformed_link_property_is_rejected(self, tmp_path): + from ub_test_reports.cli import main + + code = main( + [ + "build", + "needs", + str(GTEST_XML), + "--output", + str(tmp_path / "out.json"), + "--link-property", + "NoEqualsSign", + ] + ) + + assert code != 0 + + +class TestRemoteUrls: + def test_external_url_and_remote_url_are_synthesized(self, tmp_path): + _, data = _convert( + tmp_path, + "--remote-url", + "https://github.com/org/repo", + "--commit", + "abc123", + ) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + expected = "https://github.com/org/repo/blob/abc123/src/math_test.cc#L12" + assert need["external_url"] == expected + assert need["remote_url"] == expected + + def test_scp_style_remote_is_normalised(self, tmp_path): + _, data = _convert( + tmp_path, + "--remote-url", + "git@github.com:org/repo.git", + "--commit", + "abc123", + ) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["remote_url"].startswith("https://github.com/org/repo/blob/abc123/") + + def test_url_pattern_is_configurable(self, tmp_path): + _, data = _convert( + tmp_path, + "--remote-url", + "https://gitlab.com/org/repo", + "--commit", + "abc123", + "--url-pattern", + "{base}/-/blob/{commit}/{file}#L{line}", + ) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["remote_url"] == ( + "https://gitlab.com/org/repo/-/blob/abc123/src/math_test.cc#L12" + ) + + def test_credentials_in_the_remote_url_are_not_written(self, tmp_path): + # GitLab's CI_REPOSITORY_URL embeds the job token; the base lands in + # every need of a cached artifact and, imported, in published HTML. + from ub_test_reports.cli import main + + output = tmp_path / "needs.json" + code = main( + [ + "build", + "needs", + str(GTEST_XML), + "--output", + str(output), + "--no-config", + "--remote-url", + "https://gitlab-ci-token:glcbt-secret@gitlab.example.com/org/repo.git", + "--commit", + "abc123", + ] + ) + assert code == 0 + text = output.read_text(encoding="utf-8") + assert "glcbt-secret" not in text and "gitlab-ci-token" not in text + need = _needs(json.loads(text))["testcase__MathTest__Addition_hcuyy"] + assert need["remote_url"] == ( + "https://gitlab.example.com/org/repo/blob/abc123/src/math_test.cc#L12" + ) + + def test_without_repo_metadata_the_url_fields_are_empty(self, tmp_path): + """A hermetic sandbox has no git remote; that must not drop the need.""" + _, data = _convert(tmp_path) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["remote_url"] == "" + assert need["external_url"] == "" + + +class TestUrlPatternErrors: + """A bad template is a configuration error at the start, not a traceback.""" + + def test_an_unknown_placeholder_is_an_error(self, tmp_path, capsys): + code, data = _convert( + tmp_path, + "--no-config", + "--remote-url", + "https://github.com/o/r", + "--commit", + "abc", + "--url-pattern", + "{base}/blob/{ref}/{file}#L{line}", + ) + assert code == 2 + assert data is None + message = capsys.readouterr().err + assert "--url-pattern" in message + assert "{ref}" in message + + def test_an_unbalanced_brace_is_an_error(self, tmp_path, capsys): + code, _ = _convert( + tmp_path, "--no-config", "--url-pattern", "{base/blob/{commit}/{file}" + ) + assert code == 2 + assert "malformed" in capsys.readouterr().err + + def test_an_attribute_lookup_is_an_error_not_a_traceback(self, tmp_path, capsys): + # str.format resolves {base.__class__}; only KeyError was caught. + code, data = _convert( + tmp_path, + "--no-config", + "--remote-url", + "https://github.com/o/r", + "--commit", + "abc", + "--url-pattern", + "{base.__class__}/{file}", + ) + assert code == 2 + assert data is None + message = capsys.readouterr().err + assert "unknown placeholder {base.__class__}" in message + assert "Traceback" not in message + + def test_the_pattern_is_checked_before_any_report_is_read(self, tmp_path, capsys): + # A missing report and a bad pattern: the pattern error wins, because + # the template is checked before the first file is opened. + code, data = _convert( + tmp_path, + "--no-config", + "--url-pattern", + "{base}/blob/{ref}/{file}", + xml=tmp_path / "does-not-exist.xml", + ) + assert code == 2 + assert data is None + message = capsys.readouterr().err + assert "{ref}" in message + assert "no such file" not in message + + +class TestMultipleInputs: + def test_several_reports_are_merged_into_one_file(self, tmp_path): + from ub_test_reports.cli import main + + output = tmp_path / "needs.json" + code = main( + ["build", "needs", str(GTEST_XML), str(PYTEST_XML), "--output", str(output)] + ) + data = json.loads(output.read_text(encoding="utf-8")) + + assert code == 0 + assert len(_needs(data)) > 5 + + def test_the_same_report_given_twice_is_refused(self, tmp_path, capsys): + # Silently collapsing the repeats would produce a valid file that has + # lost half its evidence -- the worst outcome for a cached artifact. + from ub_test_reports.cli import main + + output = tmp_path / "needs.json" + code = main( + [ + "build", + "needs", + str(GTEST_XML), + str(GTEST_XML), + "--no-config", + "-o", + str(output), + ] + ) + assert code == 2 + assert not output.exists() + message = capsys.readouterr().err + assert "more than once" in message + assert "testcase__" in message + + def test_a_missing_input_file_exits_nonzero(self, tmp_path): + from ub_test_reports.cli import main + + code = main( + [ + "build", + "needs", + str(tmp_path / "nope.xml"), + "--output", + str(tmp_path / "out.json"), + ] + ) + + assert code != 0 + + +class TestDiagnostics: + def test_absent_line_attributes_warn_about_junit_family(self, tmp_path, capsys): + """pytest's default junit_family drops file/line; say so, don't guess.""" + from ub_test_reports.cli import main + + main( + [ + "build", + "needs", + str(PYTEST_XML), + "--output", + str(tmp_path / "out.json"), + ] + ) + + assert "junit_family" in capsys.readouterr().err + + def test_nested_suites_get_the_hint_too(self, tmp_path, capsys): + # The parser files the cases of a nested report under testsuite_nested; + # a hint that only looked at the top level went quiet on exactly the + # Ant/Maven-shaped reports that most often lack source locations. + code, data = _convert( + tmp_path, "--no-config", xml=UTILS / "pytest_nested_example.xml" + ) + assert code == 0 + assert all(need["case_line"] == "" for need in _needs(data).values()) + assert "junit_family" in capsys.readouterr().err + + def test_reports_with_line_attributes_do_not_warn(self, tmp_path, capsys): + from ub_test_reports.cli import main + + main(["build", "needs", str(GTEST_XML), "--output", str(tmp_path / "out.json")]) + + assert "junit_family" not in capsys.readouterr().err + + def test_a_report_without_test_cases_warns(self, tmp_path, capsys): + # A pom.xml parses as one empty suite: a valid, empty needs.json with + # exit 0 is the one outcome a cached build action must never get + # silently. + pom = tmp_path / "pom.xml" + pom.write_text( + "4.0.0\n", encoding="utf-8" + ) + code, data = _convert(tmp_path, "--no-config", xml=pom) + assert code == 0 + assert data["versions"][data["current_version"]]["needs_amount"] == 0 + message = capsys.readouterr().err + assert "pom.xml" in message and "no test cases" in message + + def test_a_report_with_test_cases_does_not_get_the_empty_warning( + self, tmp_path, capsys + ): + code, _ = _convert(tmp_path, "--no-config") + assert code == 0 + assert "no test cases" not in capsys.readouterr().err + + +def test_the_cli_is_runnable_as_a_module(): + result = subprocess.run( + [ + sys.executable, + "-m", + "ub_test_reports.cli", + "build", + "needs", + "--help", + ], + capture_output=True, + text=True, + ) + + assert result.returncode == 0 + assert "--output" in result.stdout + + +def test_console_script_is_installed(): + """The name that goes into a BUILD file has to be a real entry point.""" + script = Path(sys.executable).parent / "test-reports" + if not script.exists(): + pytest.skip("package not installed into this environment") + + result = subprocess.run( + [str(script), "build", "needs", "--help"], capture_output=True, text=True + ) + + assert result.returncode == 0 + assert "--output" in result.stdout + + +@pytest.mark.parametrize("flag", ["--remote-url", "--commit"]) +def test_url_synthesis_needs_both_parts(tmp_path, flag): + """Half the metadata cannot produce a URL; fail loudly instead of guessing.""" + from ub_test_reports.cli import main + + code = main( + [ + "build", + "needs", + str(GTEST_XML), + "--output", + str(tmp_path / "out.json"), + flag, + "value", + ] + ) + + assert code != 0 + + +class TestResultVocabulary: + """The export uses the parser's vocabulary, which is the build's. + + ``failed`` is a documented need field value and a CSS class + (``tr_failed``), and the shipped report template filters on it. A project + that mixes imported and locally created test-case needs filters both with + one expression only if the two writers spell the result alike. + """ + + def test_failure_is_exported_as_the_build_spells_it(self, tmp_path): + _, data = _convert(tmp_path) + + assert ( + _needs(data)["testcase__MathTest__Subtraction_srmht"]["result"] == "failed" + ) + + @pytest.mark.parametrize( + ("need_id", "expected"), + [ + ("testcase__MathTest__Addition_hcuyy", "passed"), + ("testcase__MathTest__DISABLED_Division_jnyzp", "disabled"), + ("testcase__ParamTest_0__Legacy_owuvz", "skipped"), + ], + ) + def test_other_results_are_unchanged(self, tmp_path, need_id, expected): + _, data = _convert(tmp_path) + + assert _needs(data)[need_id]["result"] == expected + + +class TestContentIsNotDuplicated: + """googletest repeats the failure text in the message attribute. + + Emitting both verbatim shows the same stack trace twice in the rendered + need; the message block is only worth its space when it says something the + body does not. + """ + + def test_a_message_contained_in_the_body_is_not_repeated(self, tmp_path): + _, data = _convert(tmp_path) + + content = _needs(data)["testcase__MathTest__Subtraction_srmht"]["content"] + assert content.count("Expected equality of these values") == 1 + assert "message" not in content + + def test_a_message_absent_from_the_body_is_kept(self, tmp_path): + _, data = _convert(tmp_path) + + content = _needs(data)["testcase__ParamTest_0__Legacy_owuvz"]["content"] + assert "Skipped via GTEST_SKIP" in content + assert "not applicable on this platform" in content diff --git a/packages/sphinx-test-reports/tests/test_identity.py b/packages/ub-test-reports/tests/test_identity.py similarity index 99% rename from packages/sphinx-test-reports/tests/test_identity.py rename to packages/ub-test-reports/tests/test_identity.py index d75d05815..a1f73f659 100644 --- a/packages/sphinx-test-reports/tests/test_identity.py +++ b/packages/ub-test-reports/tests/test_identity.py @@ -17,7 +17,7 @@ import pytest -from sphinx_test_reports.identity import ( +from ub_test_reports.identity import ( PLACEHOLDER_FILE, case_display_name, deterministic_case_id, diff --git a/packages/ub-test-reports/tests/test_json_parser.py b/packages/ub-test-reports/tests/test_json_parser.py new file mode 100644 index 000000000..56b362e3f --- /dev/null +++ b/packages/ub-test-reports/tests/test_json_parser.py @@ -0,0 +1,143 @@ +"""The JSON parser, called directly: parse, the mapping, result normalisation, errors. + +There is no standard JSON test report, so the parser reads whatever ``tr_json_mapping`` +(or the converter's equivalent) declares: each output key maps to a path into the report +and a default. These tests drive it over the three JSON fixtures -- flat, nested paths, and +custom fields -- without Sphinx; the extension's own suite covers the same reports through a +build. +""" + +from pathlib import Path + +import pytest + +from ub_test_reports.jsonparser import JsonFileMissing, JsonParser, dict_get + +FIXTURES = Path(__file__).parent / "fixtures" + +#: A case's fields as the flat fixture spells them, each read from the key of its name. +CASE_KEYS = [ + "name", + "classname", + "file", + "line", + "time", + "result", + "type", + "text", + "message", + "system-out", +] + + +def _mapping(suite_name=("name",), testcases=("testcase",), extra=None): + """A ``tr_json_mapping`` entry: ``key -> (path into the report, default)``.""" + testcase = {key: ([key], "unknown") for key in CASE_KEYS} + testcase.update(extra or {}) + return { + "testsuite": { + "name": (list(suite_name), "unknown"), + "tests": (["tests"], "unknown"), + "errors": (["errors"], "unknown"), + "failures": (["failures"], "unknown"), + "skips": (["skips"], "unknown"), + "passed": (["passed"], "unknown"), + "time": (["time"], "unknown"), + "testcases": (list(testcases), "unknown"), + }, + "testcase": testcase, + } + + +def _parse(name, mapping): + parser = JsonParser(FIXTURES / name, json_mapping=mapping) + assert parser.validate() is True + return parser.parse() + + +class TestParse: + def test_a_flat_report_gives_one_suite_with_its_cases(self): + (suite,) = _parse("json_data.json", _mapping()) + assert suite["name"] == "test suite 1" + assert (suite["tests"], suite["failures"], suite["skips"]) == (3, 1, 1) + assert suite["testsuite_nested"] == [] + assert [case["name"] for case in suite["testcases"]] == [ + "test case 1", + "test case 2", + "test case 3", + ] + first = suite["testcases"][0] + assert first["classname"] == "class name 1" + assert first["line"] == 123 + assert first["message"] == "all went wrong :( (message)" + + def test_nested_paths_reach_into_the_report(self): + (suite,) = _parse( + "json_complex_data.json", + _mapping( + suite_name=("internals", "name"), testcases=("testcase", "nested") + ), + ) + assert suite["name"] == "test suite 1" + assert [case["classname"] for case in suite["testcases"]] == [ + "class name 1", + "class name 2", + "class name 3", + ] + + def test_custom_fields_take_their_default_where_a_case_lacks_them(self): + extra = { + "id": (["id"], None), + "status": (["status"], "unknown"), + "tags": (["tags"], "unknown"), + } + (suite,) = _parse("json_custom_data.json", _mapping(extra=extra)) + cases = suite["testcases"] + assert [case["id"] for case in cases] == [ + "TEST_CASE_1", + "TEST_CASE_2", + "TEST_CASE_3", + None, + ] + assert cases[0]["tags"] == "a,b,c" + # present but empty is the report's value, not the default + assert cases[2]["status"] == "" + assert cases[3]["status"] == "unknown" + + +class TestResultNormalisation: + """The JSON parser's ``result`` is the JUnit parser's vocabulary.""" + + def test_each_case_result_is_normalised(self): + (suite,) = _parse("json_data.json", _mapping()) + # the report spells the first `failure`, after the JUnit element name + assert [case["result"] for case in suite["testcases"]] == [ + "failed", + "passed", + "skipped", + ] + + def test_a_missing_result_keeps_its_default(self): + mapping = _mapping(extra={"result": (["no_such_key"], None)}) + (suite,) = _parse("json_data.json", mapping) + assert [case["result"] for case in suite["testcases"]] == [None, None, None] + + +class TestErrors: + def test_a_missing_file_is_named(self, tmp_path): + missing = tmp_path / "missing.json" + with pytest.raises(JsonFileMissing, match=r"missing\.json"): + JsonParser(missing, json_mapping=_mapping()) + + @pytest.mark.parametrize( + ("items", "expected"), + [ + (["nested", "a_list", 0, "finally"], "target_data"), + (["nested", "no_such_key"], "default"), # KeyError + (["nested", "a_list", 5], "default"), # IndexError + (["nested", "a_list", "finally"], "default"), # TypeError: list["finally"] + ], + ) + def test_dict_get_falls_back_to_the_default(self, items, expected): + data = {"nested": {"a_list": [{"finally": "target_data"}]}} + assert dict_get(data, items, "default") == expected diff --git a/packages/sphinx-test-reports/tests/test_junit_parser.py b/packages/ub-test-reports/tests/test_junit_parser.py similarity index 83% rename from packages/sphinx-test-reports/tests/test_junit_parser.py rename to packages/ub-test-reports/tests/test_junit_parser.py index 99f3371b2..428de4c21 100644 --- a/packages/sphinx-test-reports/tests/test_junit_parser.py +++ b/packages/ub-test-reports/tests/test_junit_parser.py @@ -1,28 +1,24 @@ import os -xml_path = os.path.join(os.path.dirname(__file__), "doc_test/utils", "xml_data.xml") -xml_pytest_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "pytest_data.xml" -) +xml_path = os.path.join(os.path.dirname(__file__), "fixtures", "xml_data.xml") +xml_pytest_path = os.path.join(os.path.dirname(__file__), "fixtures", "pytest_data.xml") xml_pytest51_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "pytest_data_5_1.xml" + os.path.dirname(__file__), "fixtures", "pytest_data_5_1.xml" ) xml_pytest62_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "pytest_data_6_2.xml" + os.path.dirname(__file__), "fixtures", "pytest_data_6_2.xml" ) -xml_nose_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "nose_data.xml" -) +xml_nose_path = os.path.join(os.path.dirname(__file__), "fixtures", "nose_data.xml") -xml_ctest_path = os.path.join(os.path.dirname(__file__), "doc_test/utils", "ctest.xml") +xml_ctest_path = os.path.join(os.path.dirname(__file__), "fixtures", "ctest.xml") xml_error_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "xml_data_error.xml" + os.path.dirname(__file__), "fixtures", "xml_data_error.xml" ) def test_init_parser(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_path) @@ -30,7 +26,7 @@ def test_init_parser(): def test_xml_object(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_path) obj = parser.junit_xml_object @@ -40,7 +36,7 @@ def test_xml_object(): def test_parse_easy_xml(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_path) assert hasattr(parser, "parse") @@ -54,7 +50,7 @@ def test_parse_easy_xml(): def test_parse_nosetest_xml(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_nose_path) assert hasattr(parser, "parse") @@ -73,7 +69,7 @@ def test_parse_nosetest_xml(): def test_parse_pytest_xml(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_pytest_path) assert hasattr(parser, "parse") @@ -94,7 +90,7 @@ def test_parse_pytest_xml(): def test_parse_pytest_51_xml(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_pytest51_path) assert hasattr(parser, "parse") @@ -105,7 +101,7 @@ def test_parse_pytest_51_xml(): def test_parse_pytest_61_gets_test_suite_attributes(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_pytest62_path) test_suites = parser.parse() @@ -123,7 +119,7 @@ def test_parse_pytest_61_gets_test_suite_attributes(): def test_parse_ctest_xml(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_ctest_path) test_suites = parser.parse() @@ -154,7 +150,7 @@ def test_parse_ctest_xml(): def test_parse_error_xml(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_error_path) test_suites = parser.parse() @@ -184,12 +180,12 @@ def test_parse_error_xml(): xml_runner_error_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "runner_error_data.xml" + os.path.dirname(__file__), "fixtures", "runner_error_data.xml" ) def _runner_error_case(name): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser suite = JUnitParser(xml_runner_error_path).parse()[0] return next(case for case in suite["testcases"] if case["name"] == name) @@ -237,7 +233,7 @@ def test_a_passing_testcase_next_to_errors_is_still_passed(): def test_error_counts_are_taken_from_the_testsuite(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser suite = JUnitParser(xml_runner_error_path).parse()[0] diff --git a/packages/sphinx-test-reports/tests/test_junit_parser_gtest.py b/packages/ub-test-reports/tests/test_junit_parser_gtest.py similarity index 96% rename from packages/sphinx-test-reports/tests/test_junit_parser_gtest.py rename to packages/ub-test-reports/tests/test_junit_parser_gtest.py index f6cd0f7fc..928ac076b 100644 --- a/packages/sphinx-test-reports/tests/test_junit_parser_gtest.py +++ b/packages/ub-test-reports/tests/test_junit_parser_gtest.py @@ -17,13 +17,11 @@ import os -xml_gtest_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "gtest_data.xml" -) +xml_gtest_path = os.path.join(os.path.dirname(__file__), "fixtures", "gtest_data.xml") def _suites(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser return JUnitParser(xml_gtest_path).parse() diff --git a/packages/sphinx-test-reports/tests/test_needs_export.py b/packages/ub-test-reports/tests/test_needs_export.py similarity index 99% rename from packages/sphinx-test-reports/tests/test_needs_export.py rename to packages/ub-test-reports/tests/test_needs_export.py index 5ec48e1cb..568f422db 100644 --- a/packages/sphinx-test-reports/tests/test_needs_export.py +++ b/packages/ub-test-reports/tests/test_needs_export.py @@ -7,12 +7,12 @@ import pytest -from sphinx_test_reports.needs_export import ( +from ub_test_reports.needs_export import ( build_content, build_need, build_needs_file, ) -from sphinx_test_reports.remote import ( +from ub_test_reports.remote import ( check_url_pattern, normalise_remote_url, source_url, diff --git a/packages/ub-test-reports/tests/test_project_config.py b/packages/ub-test-reports/tests/test_project_config.py new file mode 100644 index 000000000..b47e3c5e4 --- /dev/null +++ b/packages/ub-test-reports/tests/test_project_config.py @@ -0,0 +1,739 @@ +"""Tests for the declarative configuration (``ubproject.toml``, ``[test_reports]``). + +The loader validates and normalises the section, and the Sphinx build bridges +its keys onto the ``tr_*`` config values. Both must agree on which file +describes a project, or the build is configured by something other than what +the project declares. +""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +import ub_project +from ub_project import ProjectConfigError +from ub_test_reports.projectconfig import ( + BRIDGE_KEYS, + BUILD_TABLE, + DEFAULT_FIELD_NAMES, + DEFAULT_TOML_FILENAME, + TomlConfigError, + field_names, + find_project_config, + load_project_config, + needs_settings, +) + + +def _write(tmp_path, toml_source, name=DEFAULT_TOML_FILENAME): + config = tmp_path / name + config.write_text(toml_source, encoding="utf-8") + return config + + +class TestLoader: + """The Sphinx-free loader: parsing, normalising, anchoring, rejecting.""" + + def test_missing_file_is_none(self, tmp_path): + assert load_project_config(tmp_path / DEFAULT_TOML_FILENAME) is None + + def test_missing_section_is_empty(self, tmp_path): + _write(tmp_path, '[project]\nname = "x"\n') + assert load_project_config(tmp_path / DEFAULT_TOML_FILENAME) == {} + + def test_full_section_round_trips(self, tmp_path): + _write( + tmp_path, + """ + [test_reports] + file_option = "report_file" + source_file_option = "file" + import_encoding = "latin1" + deterministic_case_ids = true + suite_id_length = 4 + extra_options = ["more_info"] + property_link_types = { request = "req" } + """, + ) + config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + assert config["file_option"] == "report_file" + assert config["source_file_option"] == "file" + assert config["import_encoding"] == "latin1" + assert config["deterministic_case_ids"] is True + assert config["suite_id_length"] == 4 + assert config["extra_options"] == ["more_info"] + assert config["property_link_types"] == {"request": "req"} + + def test_build_needs_table_is_validated_but_never_bridged(self, tmp_path): + # [test_reports.build.needs] belongs to the command line. The build + # validates it -- one file, one verdict -- but must not map it onto a + # tr_* value. + _write( + tmp_path, + """ + [test_reports] + file_option = "report_file" + + [test_reports.build.needs] + project = "demo" + tags = ["ci"] + """, + ) + reported = [] + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) + assert reported == [] + assert needs_settings(section) == {"project": "demo", "tags": ["ci"]} + assert BUILD_TABLE not in BRIDGE_KEYS + + @pytest.mark.parametrize( + ("key", "value"), + [ + ("project", "42"), + ("tags", '"ci, unit"'), # a bare string is not an array + ("tags", "[1]"), + ("link_properties", '["a"]'), + ("link_properties", '{ Verifies = ["verifies"] }'), # values too + ], + ) + def test_build_needs_wrong_types_are_rejected(self, tmp_path, key, value): + _write(tmp_path, f"[test_reports.build.needs]\n{key} = {value}\n") + with pytest.raises(TomlConfigError, match=f"build.needs.{key}"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_unknown_artifact_under_build_is_reported_but_not_fatal(self, tmp_path): + # `build` holds one table per artifact the command line produces. A + # newer command may produce one this version does not know, and a file + # naming it must not take the build down. + _write( + tmp_path, + """ + [test_reports.build.needs] + project = "p" + + [test_reports.build.graph] + format = "svg" + """, + ) + reported = [] + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) + assert needs_settings(section) == {"project": "p"} + assert section[BUILD_TABLE] == {"needs": {"project": "p"}} + assert len(reported) == 1 + assert "graph" in reported[0] + assert "[test_reports.build]" in reported[0] + + def test_a_non_table_needs_artifact_is_rejected(self, tmp_path): + _write(tmp_path, '[test_reports.build]\nneeds = "yes"\n') + with pytest.raises(TomlConfigError, match=r"build.needs"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_build_needs_unknown_key_is_reported_but_not_fatal(self, tmp_path): + _write(tmp_path, "[test_reports.build.needs]\nprojct = 'typo'\nproject = 'p'\n") + reported = [] + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) + assert needs_settings(section) == {"project": "p"} + assert len(reported) == 1 + assert "projct" in reported[0] + assert "[test_reports.build.needs]" in reported[0] + + def test_need_type_and_case_type_must_agree(self, tmp_path): + # The converter takes the need type (and the deterministic-ID prefix) + # from build.needs.need_type, the build from case's type. Disagreeing + # produces a needs.json the build neither registers nor cross-links. + _write( + tmp_path, + """ + [test_reports.build.needs] + need_type = "testcase" + + [test_reports.case] + directive = "test-case" + type = "check" + name = "Check" + prefix = "CH_" + color = "#999999" + style = "rectangle" + """, + ) + with pytest.raises(TomlConfigError, match="need_type"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_a_missing_side_is_compared_at_its_default(self, tmp_path): + # A customised case next to a convert table without need_type is a + # disagreement too: the converter would write the default type. + _write( + tmp_path, + """ + [test_reports.build.needs] + project = "p" + + [test_reports.case] + directive = "test-case" + type = "check" + name = "Check" + prefix = "CH_" + color = "#999999" + style = "rectangle" + """, + ) + with pytest.raises(TomlConfigError, match=r"'testcase'.*'check'"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + # ... and the mirror: need_type set, case left at its default. + _write(tmp_path, '[test_reports.build.needs]\nneed_type = "check"\n') + with pytest.raises(TomlConfigError, match=r"'check'.*'testcase'"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_without_a_convert_table_the_case_type_is_free(self, tmp_path): + # A project that only builds may name its case type as it likes. + _write( + tmp_path, + """ + [test_reports.case] + directive = "test-case" + type = "check" + name = "Check" + prefix = "CH_" + color = "#999999" + style = "rectangle" + """, + ) + assert ( + load_project_config(tmp_path / DEFAULT_TOML_FILENAME)["case"][1] == "check" + ) + + def test_empty_link_property_names_are_rejected_by_the_loader(self, tmp_path): + # One verdict for both consumers: the build refuses what the converter + # would refuse. + _write( + tmp_path, + '[test_reports.build.needs]\nlink_properties = { Verifies = "" }\n', + ) + with pytest.raises(TomlConfigError, match="link_properties"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_field_names_default_to_the_build_s(self, tmp_path): + _write(tmp_path, "[test_reports]\nsource_file_option = 'src'\n") + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + names = field_names(section) + assert names["source_file_option"] == "src" + assert names["file_option"] == DEFAULT_FIELD_NAMES["file_option"] == "file" + assert names["source_line_option"] == "case_line" + + def test_colliding_field_names_are_rejected(self, tmp_path): + # file (report path) and file (source path) cannot share a field. + _write(tmp_path, "[test_reports]\nsource_file_option = 'file'\n") + with pytest.raises(TomlConfigError, match="both name the need field 'file'"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + @pytest.mark.parametrize( + ("key", "name"), + [ + ("file_option", "case"), + ("source_file_option", "result"), + ("source_line_option", "id"), + ], + ) + def test_a_rename_onto_a_fixed_field_is_rejected(self, tmp_path, key, name): + # Every test-case need has these already: the directives would pass + # the keyword twice, the converter overwrite one value with the other. + _write(tmp_path, f"[test_reports]\n{key} = '{name}'\n") + with pytest.raises(TomlConfigError, match=f"{key} = '{name}'"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_need_type_and_case_type_agreeing_is_fine(self, tmp_path): + _write( + tmp_path, + """ + [test_reports.build.needs] + need_type = "check" + + [test_reports.case] + directive = "test-case" + type = "check" + name = "Check" + prefix = "CH_" + color = "#999999" + style = "rectangle" + """, + ) + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + assert section["case"][1] == needs_settings(section)["need_type"] == "check" + + def test_unknown_key_is_reported_but_not_fatal(self, tmp_path): + # ubproject.toml is shared with tools on independent release cadences, + # so a key this reader does not model must not take the build down -- + # but a typo has to be visible, and the key must not be passed on. + _write(tmp_path, "[test_reports]\ndeterministic_id = true\n") + reported = [] + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) + assert section == {} + assert len(reported) == 1 + assert "deterministic_id" in reported[0] + assert "deterministic_case_ids" in reported[0] # supported keys listed + + def test_unknown_key_needs_no_reporter(self, tmp_path): + _write(tmp_path, "[test_reports]\nnope = 1\nfile_option = 'f'\n") + assert load_project_config(tmp_path / DEFAULT_TOML_FILENAME) == { + "file_option": "f" + } + + @pytest.mark.parametrize( + ("key", "value"), + [ + ("property_link_types", '{ request = ["req"] }'), + ("property_link_types", "{ request = 3 }"), + ], + ) + def test_table_values_are_type_checked(self, tmp_path, key, value): + # Without this the value reaches the directives, which fail with a bare + # TypeError on an unhashable field name instead of a config error. + _write(tmp_path, f"[test_reports]\n{key} = {value}\n") + with pytest.raises(TomlConfigError, match=key): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_json_mapping_nesting_stays_free_form(self, tmp_path): + # It mirrors an arbitrary parser mapping, so only the outer table is + # checked -- validating deeper would reject valid configurations. + _write( + tmp_path, + "[test_reports.json_mapping.json_config.testsuite]\nname = 1\n", + ) + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + assert section["json_mapping"] == {"json_config": {"testsuite": {"name": 1}}} + + @pytest.mark.skipif( + hasattr(os, "geteuid") and os.geteuid() == 0, + reason="root reads unreadable files", + ) + @pytest.mark.skipif( + sys.platform == "win32", + reason="os.chmod on Windows only sets the read-only attribute; the file stays readable", + ) + def test_unreadable_file_is_a_config_error(self, tmp_path): + # is_file() succeeding does not mean the open will; an unwrapped + # OSError would surface as a traceback instead of a config error. + config = _write(tmp_path, "[test_reports]\nfile_option = 'f'\n") + config.chmod(0o000) + try: + with pytest.raises(TomlConfigError, match="cannot be read"): + load_project_config(config) + finally: + config.chmod(0o644) + + @pytest.mark.parametrize( + ("key", "value"), + [ + ("file_option", "42"), + ("suite_id_length", '"four"'), # string for int + ("suite_id_length", "true"), # bool must not pass for int + ("deterministic_case_ids", '"yes"'), # string for bool + ("extra_options", '"more_info"'), # a bare string is not an array + ("property_link_types", '["a"]'), + ], + ) + def test_wrong_types_are_rejected(self, tmp_path, key, value): + _write(tmp_path, f"[test_reports]\n{key} = {value}\n") + with pytest.raises(TomlConfigError, match=key): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_invalid_toml_is_rejected(self, tmp_path): + _write(tmp_path, "[test-reports\n") + with pytest.raises(TomlConfigError, match="invalid TOML"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_section_must_be_a_table(self, tmp_path): + _write(tmp_path, "test_reports = 5\n") + with pytest.raises(TomlConfigError, match="must be a table"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_need_type_positional_list_still_works(self, tmp_path): + # The conf.py spelling, so existing projects can copy their lists over + # verbatim. + _write( + tmp_path, + '[test_reports]\ncase = ["test-case", "testcase", "Test-Case", "TC_", "#999999", "rectangle"]\n', + ) + config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + assert config["case"] == [ + "test-case", + "testcase", + "Test-Case", + "TC_", + "#999999", + "rectangle", + ] + + def test_need_type_named_table(self, tmp_path): + # Six bare strings cannot be told apart; the table spelling names them. + _write( + tmp_path, + """ + [test_reports.case] + directive = "test-case" + type = "testcase" + name = "Test-Case" + prefix = "TC_" + color = "#999999" + style = "rectangle" + """, + ) + config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + assert config["case"] == [ + "test-case", + "testcase", + "Test-Case", + "TC_", + "#999999", + "rectangle", + ] + + def test_need_type_table_rejects_partial_and_unknown(self, tmp_path): + _write(tmp_path, '[test_reports.case]\ndirective = "test-case"\n') + with pytest.raises(TomlConfigError, match="missing"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + _write( + tmp_path, + """ + [test_reports.case] + directive = "test-case" + type = "testcase" + name = "Test-Case" + prefix = "TC_" + color = "#999999" + style = "rectangle" + typo = true + """, + ) + with pytest.raises(TomlConfigError, match="unknown typo"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + @pytest.mark.parametrize( + ("toml_source", "problem"), + [ + # A non-string value inside the named table. Without the check the + # value is silently stringified and reaches sphinx-needs. + ( + """ + [test_reports.case] + directive = "test-case" + type = "testcase" + name = "Test-Case" + prefix = "TC_" + color = 999999 + style = "rectangle" + """, + "non-string color", + ), + # A non-string element of the positional list. + ( + '[test_reports]\ncase = ["test-case", "testcase", "Test-Case", "TC_", 999999, "rectangle"]\n', + "exactly 6 strings", + ), + # Too few elements. + ('[test_reports]\ncase = ["test-case", "testcase"]\n', "exactly 6 strings"), + ], + ) + def test_need_type_values_must_be_strings(self, tmp_path, toml_source, problem): + _write(tmp_path, toml_source) + with pytest.raises(TomlConfigError, match=problem): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_relative_paths_anchor_to_the_toml_directory(self, tmp_path): + # The file is self-describing: moving it as a unit keeps its relative + # paths meaningful, and both consumers resolve them identically. + subdir = tmp_path / "config" + subdir.mkdir() + _write( + subdir, + '[test_reports]\nrootdir = "docs"\nreport_template = "templates/report.txt"\n', + name=subdir / DEFAULT_TOML_FILENAME, + ) + config = load_project_config(subdir / DEFAULT_TOML_FILENAME) + assert config["rootdir"] == str(subdir / "docs") + assert config["report_template"] == str(subdir / "templates" / "report.txt") + + @pytest.mark.parametrize( + "suffix", + [ + pytest.param("", id="plain"), + # the two forms ``Path`` normalises away: a round trip through it + # would return ``/a/b`` for ``/a/b/`` and ``/a/b`` for ``/a//b``, + # so these are the cases that tell "left as the string it was" + # from "anchored, and absolute already" + pytest.param(os.sep, id="trailing-separator"), + pytest.param(f"{os.sep}{os.sep}x", id="doubled-separator"), + ], + ) + def test_absolute_paths_stay_untouched(self, tmp_path, suffix): + # a TOML literal string: in a basic string a Windows path's backslashes + # are escape sequences ("\U" starts a unicode escape) and the file is invalid + value = f"{tmp_path}{suffix}" + _write(tmp_path, f"[test_reports]\nrootdir = '{value}'\n") + config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + assert config["rootdir"] == value + + +def _not_utf8(tmp_path): + """A file saved in Latin-1: ``é`` is the lone byte 0xE9, which UTF-8 refuses.""" + config = tmp_path / DEFAULT_TOML_FILENAME + config.write_bytes("[test_reports]\nfile_option = 'café'\n".encode("latin-1")) + return config + + +class TestSharedReaderBoundary: + """ub-project reads the file; its exception never leaves this package. + + Both consumers catch :class:`TomlConfigError` and nothing else, so a + ``ProjectConfigError`` escaping the loader would reach the user as a + traceback. Each case asserts the exact type and that the message is the + shared reader's, word for word. + """ + + def _assert_re_raised(self, config): + with pytest.raises(TomlConfigError) as caught: + load_project_config(config) + assert type(caught.value) is TomlConfigError + cause = caught.value.__cause__ + assert isinstance(cause, ProjectConfigError) + assert str(caught.value) == str(cause) + return str(caught.value) + + def test_the_two_exceptions_are_unrelated(self): + # a subclass either way round would put ub-project's exception on this + # package's public surface + assert not issubclass(TomlConfigError, ProjectConfigError) + assert not issubclass(ProjectConfigError, TomlConfigError) + + def test_the_walk_is_the_shared_reader_s_own(self): + # re-exported, not copied: a local fork of the walk would pass every + # discovery test and drift from the reader the other members use + assert find_project_config is ub_project.find_project_config + + def test_invalid_toml(self, tmp_path): + message = self._assert_re_raised(_write(tmp_path, "[test-reports\n")) + assert message.startswith(f"{tmp_path / DEFAULT_TOML_FILENAME}: invalid TOML: ") + + @pytest.mark.skipif( + hasattr(os, "geteuid") and os.geteuid() == 0, + reason="root reads unreadable files", + ) + @pytest.mark.skipif( + sys.platform == "win32", + reason="os.chmod on Windows only sets the read-only attribute; the file stays readable", + ) + def test_unreadable_file(self, tmp_path): + config = _write(tmp_path, "[test_reports]\nfile_option = 'f'\n") + config.chmod(0o000) + try: + message = self._assert_re_raised(config) + finally: + config.chmod(0o644) + assert message.startswith(f"{config}: cannot be read: ") + + def test_a_file_that_is_not_utf8(self, tmp_path): + # New with ub-project: before it, the decode error escaped the loader + # as a bare UnicodeDecodeError. + config = _not_utf8(tmp_path) + message = self._assert_re_raised(config) + assert message.startswith(f"{config}: not valid UTF-8 TOML: ") + + +class TestDiscovery: + """The upward search that lets both consumers find the same file. + + The search is bounded by the repository root -- the directory holding + ``.git``: a ``pyproject.toml`` on the way up marks a Python distribution, + not the project, and must not end the search. Outside any repository there + is no such root, so the distribution root bounds it instead -- otherwise + the walk reaches the filesystem root and adopts a stranger's file. + """ + + def test_finds_the_file_in_the_starting_directory(self, tmp_path): + config = _write(tmp_path, "[test_reports]\n") + assert find_project_config(tmp_path) == config + + def test_walks_up_to_the_repository_root(self, tmp_path): + (tmp_path / ".git").mkdir() + config = _write(tmp_path, "[test_reports]\n") + deep = tmp_path / "docs" / "source" + deep.mkdir(parents=True) + assert find_project_config(deep) == config + + def test_a_pyproject_toml_beside_conf_py_does_not_end_the_search(self, tmp_path): + # docs/ carrying its own pyproject.toml (its own dependency set) still + # belongs to the project whose shared file sits at the repository root. + (tmp_path / ".git").mkdir() + config = _write(tmp_path, "[test_reports]\n") + docs = tmp_path / "docs" + docs.mkdir() + (docs / "pyproject.toml").write_text("", encoding="utf-8") + assert find_project_config(docs) == config + + def test_walks_past_a_workspace_member_pyproject_toml(self, tmp_path): + # A uv-workspace member: packages//pyproject.toml with the docs + # below it, and one ubproject.toml at the repository root describing + # the whole monorepo. + (tmp_path / ".git").mkdir() + config = _write(tmp_path, "[test_reports]\n") + member = tmp_path / "packages" / "dist" + docs = member / "docs" + docs.mkdir(parents=True) + (member / "pyproject.toml").write_text("", encoding="utf-8") + assert find_project_config(docs) == config + + def test_stops_at_a_nested_repository_without_the_file(self, tmp_path): + # A checkout nested inside another repository (a vendored tree, a + # submodule) must not adopt the outer repository's configuration. + _write(tmp_path, "[test_reports]\n") + inner = tmp_path / "vendor" / "inner" + docs = inner / "docs" + docs.mkdir(parents=True) + (inner / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + assert find_project_config(docs) is None + + def test_stops_at_the_distribution_root_without_a_repository(self, tmp_path): + # An unpacked sdist, a CI artefact directory, an exported docs tree: + # no .git anywhere, so nothing above would end the walk and a + # stranger's file further up would be adopted. The distribution root + # bounds the search instead, so it is not. + _write(tmp_path, "[test_reports]\n") # a stranger's, two levels up + dist = tmp_path / "downloads" / "sphinx-test-reports-1.4.0" + docs = dist / "docs" + docs.mkdir(parents=True) + (dist / "pyproject.toml").write_text("", encoding="utf-8") + assert find_project_config(docs) is None + + def test_the_file_at_the_distribution_root_is_still_found(self, tmp_path): + # The distribution root bounds the search without hiding a file that + # sits on it: an sdist shipping its own ubproject.toml is configured + # by it. + dist = tmp_path / "sphinx-test-reports-1.4.0" + docs = dist / "docs" + docs.mkdir(parents=True) + (dist / "pyproject.toml").write_text("", encoding="utf-8") + config = _write(dist, "[test_reports]\n") + assert find_project_config(docs) == config + + def test_a_repository_marker_outranks_a_distribution_root(self, tmp_path): + # The distribution root is only the fallback boundary. Inside a + # repository the walk still passes a pyproject.toml on the way up -- + # the workspace-member layout above depends on it. + (tmp_path / ".git").mkdir() + config = _write(tmp_path, "[test_reports]\n") + member = tmp_path / "packages" / "dist" + docs = member / "docs" + docs.mkdir(parents=True) + (member / "pyproject.toml").write_text("", encoding="utf-8") + assert find_project_config(docs) == config + + def test_a_fruitless_search_reports_the_distribution_root(self, tmp_path): + dist = tmp_path / "sphinx-test-reports-1.4.0" + docs = dist / "docs" + docs.mkdir(parents=True) + (dist / "pyproject.toml").write_text("", encoding="utf-8") + reported = [] + assert find_project_config(docs, report=reported.append) is None + assert len(reported) == 1 + assert f"distribution root {dist}" in reported[0] + assert "pyproject.toml" in reported[0] + + def test_the_file_wins_over_the_marker_in_one_directory(self, tmp_path): + # The root marker only ends a *fruitless* step; a repository root + # holding the file is the canonical layout and must be found. + config = _write(tmp_path, "[test_reports]\n") + (tmp_path / ".git").mkdir() + assert find_project_config(tmp_path) == config + + def test_missing_file_is_none(self, tmp_path): + (tmp_path / ".git").mkdir() + assert find_project_config(tmp_path) is None + + def test_a_fruitless_search_reports_where_it_ended(self, tmp_path): + # "Not found" must not be silent: the report names the directory whose + # marker ended the search, so a misplaced file can be diagnosed. + (tmp_path / ".git").mkdir() + docs = tmp_path / "docs" + docs.mkdir() + reported = [] + assert find_project_config(docs, report=reported.append) is None + assert len(reported) == 1 + assert str(docs) in reported[0] + assert f"repository root {tmp_path}" in reported[0] + assert ".git" in reported[0] + + def test_a_successful_search_reports_nothing(self, tmp_path): + (tmp_path / ".git").mkdir() + _write(tmp_path, "[test_reports]\n") + reported = [] + find_project_config(tmp_path / "docs", report=reported.append) + assert reported == [] + + def test_a_relative_start_is_searched_from_the_working_directory( + self, tmp_path, monkeypatch + ): + # A converter started with a relative path must still see the parents. + (tmp_path / ".git").mkdir() + config = _write(tmp_path, "[test_reports]\n") + docs = tmp_path / "docs" + docs.mkdir() + monkeypatch.chdir(docs) + assert find_project_config(Path(".")) == config + + def test_a_symlinked_start_walks_the_link_s_parents(self, tmp_path): + # The start is made absolute WITHOUT resolving: a symlinked docs/ + # belongs to the repository it is linked into, not to the one its + # target lives in. The only case here that tells the two apart -- + # tmp_path is already resolved, so every other start is too. + repo = tmp_path / "repo" + (repo / ".git").mkdir(parents=True) + config = _write(repo, "[test_reports]\n") + elsewhere = tmp_path / "elsewhere" + (elsewhere / ".git").mkdir(parents=True) + target = elsewhere / "docs" + target.mkdir() + link = repo / "docs" + try: + link.symlink_to(target, target_is_directory=True) + except OSError: # Windows without the symlink privilege + pytest.skip("creating a symlink needs a privilege this account lacks") + assert find_project_config(link) == config + + +class TestPathAnchoring: + """Relative paths anchor at the TOML file's directory, as given.""" + + def test_anchoring_does_not_resolve_the_given_directory(self, tmp_path): + # The loader leaves the form of the directory it was handed alone -- a + # symlinked path stays symlinked. Whether to resolve it is the + # consumer's call (Sphinx resolves its confdir before the bridge runs), + # not something the loader decides behind its back. + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real, target_is_directory=True) + config = _write(link, "[test_reports]\nrootdir = 'reports'\n") + section = load_project_config(config) + assert section["rootdir"] == str(link / "reports") + + +class TestSphinxFree: + """A consumer without the documentation toolchain can read the section.""" + + def test_projectconfig_imports_without_sphinx(self): + # Importing the module runs the package __init__, so the package must + # not import Sphinx eagerly either -- or a build action that turns + # reports into a needs.json dies with ModuleNotFoundError wherever + # Sphinx is not installed. Checked in a subprocess: this process has + # Sphinx imported already. + code = ( + "import sys\n" + "sys.modules['sphinx'] = None\n" # any `import sphinx...` now fails + "import ub_test_reports.projectconfig\n" + ) + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True + ) + assert result.returncode == 0, result.stderr diff --git a/packages/sphinx-test-reports/tests/test_pytest_plugin.py b/packages/ub-test-reports/tests/test_pytest_plugin.py similarity index 93% rename from packages/sphinx-test-reports/tests/test_pytest_plugin.py rename to packages/ub-test-reports/tests/test_pytest_plugin.py index e05d500e5..5f6831730 100644 --- a/packages/sphinx-test-reports/tests/test_pytest_plugin.py +++ b/packages/ub-test-reports/tests/test_pytest_plugin.py @@ -11,8 +11,8 @@ import pytest -from sphinx_test_reports import pytest_plugin -from sphinx_test_reports.pytest_plugin import ( +from ub_test_reports import pytest_plugin +from ub_test_reports.pytest_plugin import ( Property, apply_test_metadata, clean_source_path, @@ -20,7 +20,7 @@ properties_mapping, ) -PLUGIN = "sphinx_test_reports.pytest_plugin" +PLUGIN = "ub_test_reports.pytest_plugin" #: S-CORE's model, as the docs show it: the profile most of these tests run with. SCORE_PROFILE = """\ @@ -32,7 +32,7 @@ """ DECORATED = """ -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @add_test_properties( partially_verifies=["REQ_1", "REQ_2"], @@ -50,7 +50,7 @@ def test_plain(): RUNTIME = """ import pytest -from sphinx_test_reports.pytest_plugin import apply_test_metadata +from ub_test_reports.pytest_plugin import apply_test_metadata @pytest.mark.parametrize("spec", ["a.rst", "b.rst"]) def test_driven_by_a_file(spec, record_property): @@ -64,7 +64,7 @@ def test_driven_by_a_file(spec, record_property): """ RUNTIME_COMPAT = """ -from sphinx_test_reports.pytest_plugin import apply_test_metadata +from ub_test_reports.pytest_plugin import apply_test_metadata def test_score_style(record_property, record_xml_attribute): apply_test_metadata( @@ -78,7 +78,7 @@ def test_score_style(record_property, record_xml_attribute): SKIPPED = """ import pytest -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @pytest.fixture(scope="module") @@ -118,7 +118,7 @@ def test_runs(): # derives its own fixture from this string by replacing `str(inner)]) == 0`. NESTED = """ import pytest -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @add_test_properties(partially_verifies=["REQ_1"]) @@ -154,7 +154,7 @@ def test_strict(): """ STACKED = """ -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @add_test_properties(test_type="requirements-based", Owner="team-a") @@ -175,7 +175,7 @@ def test_stacked(): """ CUSTOM = """ -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @add_test_properties(satisfies=["REQ_1", "REQ_2"], reviewers=["ann", "bob"], Owner="x") def test_custom(): @@ -412,7 +412,7 @@ def test_a_decorator_may_run_before_configuration_is_read(self, pytester): # nor loses the properties, which are written at setup. pytester.makepyfile( helpers=( - "from sphinx_test_reports.pytest_plugin import add_test_properties\n" + "from ub_test_reports.pytest_plugin import add_test_properties\n" "\n" '@add_test_properties(partially_verifies=["REQ_1"], test_type="interface-test")\n' "def test_from_helper():\n" @@ -558,10 +558,37 @@ def test_the_location_is_applied_without_metadata(self, pytester): assert _properties(case) == {} assert (case.get("file"), case.get("line")) == ("specs/a.rst", "7") + def test_the_location_travels_under_the_documented_wire_names(self, score_model): + # The two names are a documented wire format, not the import path: they did not + # move when the plugin did, and must not change. + recorded = [] + apply_test_metadata( + record_property=lambda name, value: recorded.append((name, value)), + metadata={}, + file="specs/a.rst", + line=7, + ) + assert recorded == [ + ("sphinxcontrib.test_reports:file", "specs/a.rst"), + ("sphinxcontrib.test_reports:line", "7"), + ] + + def test_a_property_under_a_wire_name_is_read_as_the_location(self, pytester): + source = ( + "def test_direct(record_property):\n" + " record_property('sphinxcontrib.test_reports:file', 'specs/b.rst')\n" + " record_property('sphinxcontrib.test_reports:line', '3')\n" + ) + result, root = _run(pytester, source) + result.assert_outcomes(passed=1) + case = _cases(root)["test_direct"] + assert _properties(case) == {} + assert (case.get("file"), case.get("line")) == ("specs/b.rst", "3") + BAD_SHAPE_AND_BROKEN_FIXTURE = """ import pytest -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @pytest.fixture @@ -579,7 +606,7 @@ class TestBadShape: def test_a_bad_shape_errors_the_case_at_setup(self, pytester): result, _root = _run( pytester, - "from sphinx_test_reports.pytest_plugin import add_test_properties\n\n@add_test_properties(test_type=['a', 'b'])\ndef test_shape():\n assert True\n", + "from ub_test_reports.pytest_plugin import add_test_properties\n\n@add_test_properties(test_type=['a', 'b'])\ndef test_shape():\n assert True\n", ) result.assert_outcomes(errors=1) result.stdout.fnmatch_lines(["*TypeError*'test_type' takes a single value*"]) diff --git a/packages/ub-test-reports/tests/test_result_vocabulary.py b/packages/ub-test-reports/tests/test_result_vocabulary.py new file mode 100644 index 000000000..cc0ae09b5 --- /dev/null +++ b/packages/ub-test-reports/tests/test_result_vocabulary.py @@ -0,0 +1,151 @@ +"""The ``result`` field's vocabulary, and the one place it is decided. + +A test case's ``result`` used to be whatever the input spelled it: the JUnit +parser passed the name of the ```` child element through verbatim +(``failure``), the JSON parser passed the report's own value through, and the +two states with no element of their own -- ``passed`` and ``disabled`` -- were +spelled as participles because nothing forced a choice. So one product said +``failure`` for a single case and ``failed`` for the count of them +(``fields.FIELDS``), and the documented value and the documented example +disagreed. + +These tests pin the vocabulary down: every parser maps its input onto the same +participles, and the mapping is the only place that decides. The part-level +``kind`` is deliberately *not* normalised -- it names the XML element the +evidence came from, which is what the rendered evidence heading reports. +""" + +import os + +import pytest + +UTILS = os.path.join(os.path.dirname(__file__), "fixtures") +XML_PATH = os.path.join(UTILS, "xml_data.xml") +JSON_PATH = os.path.join(UTILS, "json_data.json") + +#: The mapping the JSON parser needs, as ``tr_json_mapping`` declares it. +JSON_MAPPING = { + "testsuite": { + "name": (["name"], "unknown"), + "tests": (["tests"], "unknown"), + "errors": (["errors"], "unknown"), + "failures": (["failures"], "unknown"), + "skips": (["skips"], "unknown"), + "passed": (["passed"], "unknown"), + "time": (["time"], "unknown"), + "testcases": (["testcase"], "unknown"), + }, + "testcase": { + "name": (["name"], "unknown"), + "classname": (["classname"], "unknown"), + "file": (["file"], "unknown"), + "line": (["line"], "unknown"), + "time": (["time"], "unknown"), + "result": (["result"], "unknown"), + "type": (["type"], "unknown"), + "text": (["text"], "unknown"), + "message": (["message"], "unknown"), + "system-out": (["system-out"], "unknown"), + }, +} + + +class TestNormalisation: + """One function decides the vocabulary, so both parsers cannot disagree.""" + + def test_the_junit_failure_element_name_becomes_failed(self): + from ub_test_reports.results import normalize_result + + assert normalize_result("failure") == "failed" + + def test_normalising_the_canonical_spelling_changes_nothing(self): + """Normalisation runs on already-canonical values too, so it must be + idempotent -- a JSON report may already spell the result ``failed``.""" + from ub_test_reports.results import normalize_result + + assert normalize_result("failed") == "failed" + + @pytest.mark.parametrize("result", ["passed", "skipped", "error", "disabled"]) + def test_the_other_states_are_already_canonical(self, result): + from ub_test_reports.results import normalize_result + + assert normalize_result(result) == result + + def test_a_vocabulary_this_extension_does_not_know_is_left_alone(self): + """``tr_json_mapping`` points at an arbitrary report, so a project may + feed in states of its own. Rewriting those would break its filters.""" + from ub_test_reports.results import normalize_result + + assert normalize_result("flaky") == "flaky" + + def test_the_canonical_states_are_the_documented_ones(self): + """Ordered, because the declared field description is built from it and + the converter's output has to be byte-stable.""" + from ub_test_reports.results import CANONICAL_RESULTS + + assert CANONICAL_RESULTS == ( + "passed", + "failed", + "error", + "skipped", + "disabled", + ) + + +class TestDeclaredSchema: + """The converter writes the field declarations into the ``needs.json`` it + produces, so that a consumer which never loads this extension -- a schema + check, a metamodel validator -- learns the fields from the file. Naming the + states in the ``result`` description tells it the field's domain too. + """ + + def test_the_result_declaration_names_every_state(self): + from ub_test_reports.fields import declaration + from ub_test_reports.results import CANONICAL_RESULTS + + _, description = declaration("result") + + assert all(state in description for state in CANONICAL_RESULTS), description + + +class TestJUnitParser: + """The JUnit dialect is where the old spelling came from.""" + + def test_a_failure_child_yields_the_failed_result(self): + from ub_test_reports.junitparser import JUnitParser + + suite = JUnitParser(XML_PATH).parse()[0] + + assert suite["testcases"][2]["result"] == "failed" + + def test_a_result_part_keeps_the_name_of_its_xml_element(self): + """``kind`` reports which element the evidence came from -- the + converter capitalises it into the evidence heading -- so it stays the + XML name even though ``result`` no longer is.""" + from ub_test_reports.junitparser import JUnitParser + + suite = JUnitParser(XML_PATH).parse()[0] + + assert suite["testcases"][2]["parts"][0]["kind"] == "failure" + + +class TestJsonParser: + """The JSON parser's API is documented as being in sync with the JUnit + parser's, so the same report content has to produce the same result.""" + + def test_the_failure_spelling_in_a_json_report_is_normalised(self): + from ub_test_reports.jsonparser import JsonParser + + parser = JsonParser(JSON_PATH, json_mapping=JSON_MAPPING) + suite = parser.parse()[0] + + assert suite["testcases"][0]["result"] == "failed" + + def test_the_results_needing_no_normalisation_are_untouched(self): + from ub_test_reports.jsonparser import JsonParser + + parser = JsonParser(JSON_PATH, json_mapping=JSON_MAPPING) + suite = parser.parse()[0] + + assert suite["testcases"][1]["result"] == "passed" + assert suite["testcases"][2]["result"] == "skipped" From c665f8ac7a6c4a3cda87faea0f29e9b21e74d30d Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Thu, 1 Oct 2026 02:28:41 +0200 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=94=A7=20CI=20for=20the=20core:=20too?= =?UTF-8?q?lchain-free=20rewritten,=20the=20plugin-floor=20job,=20coverage?= =?UTF-8?q?,=20labels,=20RTD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit toolchain-free is now "Toolchain-free (ub-test-reports, ub-project)": the property it fences belongs to a distribution, not an extra. It builds the core's sdist and wheel as the release does and checks them (top level exactly `ub_test_reports` plus dist-info, `schemas/JUnit.xsd` and `py.typed` shipped, no `tests/` or `docs/` in the sdist), keeps the extension's hatchling two-package fence unchanged (it only reads archives), installs the core's wheel with `[pytest]`, pytest-xdist and ub-project into a Python 3.11 environment with no toolchain, and runs the core's whole suite there -- no marker, no file list. plugin-floor: the same environment with pytest pinned to the oldest each floor Python takes (7.0.1 on 3.11, 7.3.2 on 3.12), the pin asserted, the suite run from the package directory. Added to `check`. The Extensions cells run the core's suite with coverage under a new `ub-test-reports` Codecov flag; the Lint type-gate canary probes the core; the labelers and both issue forms know `ub-test-reports`; and the extension's .readthedocs.yaml installs the core from the checkout before the extension, so pip never looks for an unreleased core on PyPI. --- .github/ISSUE_TEMPLATE/bug_report.yml | 1 + .github/ISSUE_TEMPLATE/feature_request.yml | 1 + .github/issue-labeler.yml | 8 +- .github/labeler.yml | 3 + .github/workflows/ci.yaml | 167 +++++++++++++----- .github/workflows/test-extensions.yaml | 18 ++ codecov.yml | 15 +- .../sphinx-test-reports/.readthedocs.yaml | 17 +- 8 files changed, 177 insertions(+), 53 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 63714c48e..79bd240ba 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -19,6 +19,7 @@ body: - sphinx-mounts - sphinx-codelinks - sphinx-test-reports + - ub-test-reports - ub-project - the repository (workflows, CI, release, docker, tooling) validations: diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index bf7ac5606..84abbad55 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -14,6 +14,7 @@ body: - sphinx-mounts - sphinx-codelinks - sphinx-test-reports + - ub-test-reports - ub-project - the repository (workflows, CI, release, docker, tooling) validations: diff --git a/.github/issue-labeler.yml b/.github/issue-labeler.yml index 914d152ba..075f2f32c 100644 --- a/.github/issue-labeler.yml +++ b/.github/issue-labeler.yml @@ -7,8 +7,10 @@ # patterns case-insensitive, like the workflow's `contains()` gate, so the two agree. 'pkg: sphinx-needs': - '/### Package[^#]*\bsphinx-needs\b/i' -# No two of `sphinx-needs`, `sphinx-mounts`, `sphinx-codelinks`, `sphinx-test-reports` and -# `ub-project` contains another as a \b-delimited word, so no two patterns can match one choice. Every +# No two of `sphinx-needs`, `sphinx-mounts`, `sphinx-codelinks`, `sphinx-test-reports`, +# `ub-test-reports` and `ub-project` contains another as a \b-delimited word, so no two patterns +# can match one choice (`ub-test-reports` is not a word inside `sphinx-test-reports`: the +# `-` before `test` is a boundary, but the pattern needs `ub-` there). Every # pattern has to be present before its option is offered by a form, though: the workflow # runs with `sync-labels: 1`, so a pattern that is missing when its option is picked means # the label is synced away rather than merely not added @@ -18,6 +20,8 @@ - '/### Package[^#]*\bsphinx-codelinks\b/i' 'pkg: sphinx-test-reports': - '/### Package[^#]*\bsphinx-test-reports\b/i' +'pkg: ub-test-reports': + - '/### Package[^#]*\bub-test-reports\b/i' 'pkg: ub-project': - '/### Package[^#]*\bub-project\b/i' 'pkg: workspace': diff --git a/.github/labeler.yml b/.github/labeler.yml index 8732546d3..df9215093 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -22,6 +22,9 @@ 'pkg: sphinx-needs-testkit': - changed-files: - any-glob-to-any-file: 'packages/sphinx-needs-testkit/**' +'pkg: ub-test-reports': + - changed-files: + - any-glob-to-any-file: 'packages/ub-test-reports/**' 'pkg: ub-project': - changed-files: - any-glob-to-any-file: 'packages/ub-project/**' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 532540524..875b35ae9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -61,7 +61,7 @@ jobs: # ONE probe at a time, and one per package: with both planted at once a gate that had # dropped one package would still go red on the other and look healthy. run: | - probes="packages/sphinx-needs/src/sphinx_needs/_ty_gate_probe.py packages/sphinx-mounts/src/sphinx_mounts/_ty_gate_probe.py packages/sphinx-codelinks/src/sphinx_codelinks/_ty_gate_probe.py packages/sphinx-test-reports/src/sphinx_test_reports/_ty_gate_probe.py packages/ub-project/src/ub_project/_ty_gate_probe.py" + probes="packages/sphinx-needs/src/sphinx_needs/_ty_gate_probe.py packages/sphinx-mounts/src/sphinx_mounts/_ty_gate_probe.py packages/sphinx-codelinks/src/sphinx_codelinks/_ty_gate_probe.py packages/sphinx-test-reports/src/sphinx_test_reports/_ty_gate_probe.py packages/ub-test-reports/src/ub_test_reports/_ty_gate_probe.py packages/ub-project/src/ub_project/_ty_gate_probe.py" # shellcheck disable=SC2064 trap "rm -f $probes" EXIT for probe in $probes; do @@ -459,23 +459,23 @@ jobs: working-directory: packages/sphinx-test-reports toolchain-free: - name: Toolchain-free (sphinx-test-reports, ub-project) + name: Toolchain-free (ub-test-reports, ub-project) runs-on: ubuntu-latest permissions: contents: read - # sphinx-test-reports' converter (`test-reports`) and its pytest plugin are meant to run - # where the documentation toolchain is NOT installed -- that is why Sphinx and - # sphinx-needs are an EXTRA of that package rather than dependencies. This job is the - # only thing that keeps it true: it installs the package with the dependencies it - # declares, asserts that sphinx, sphinx_needs and docutils are all absent, and runs the - # modules that do not need a build. + # ub-test-reports -- the converter (`test-reports`), the pytest plugin, the parsers and the + # `[test_reports]` model -- is a DISTRIBUTION that must run where the documentation + # toolchain is NOT installed: a build action and a test run have none. Nothing else keeps + # that true. This job installs the core's built wheel with the dependencies it declares, + # asserts that sphinx, sphinx_needs and docutils are all absent, and runs its WHOLE suite + # there: a module-level or a function-body import of the toolchain fails it. # # The environment is on Python 3.11, the workspace floor: both packages it runs claim # `>=3.11`, and no other cell runs either of them there -- the matrix cells vary the # sphinx series on the pinned interpreter -- so the job that proves "no toolchain" # proves the floor too. # - # ubproject, the shared reader for `ubproject.toml`, is installed into the SAME + # ub-project, the shared reader for `ubproject.toml`, is installed into the SAME # environment and its whole suite runs here: its contract is to import nothing outside # the standard library, so that every tool reading the file -- this converter included # -- can use it without the toolchain. The assertion below runs after both installs, so @@ -489,10 +489,9 @@ jobs: # groups are asked for. The environment is therefore built OUTSIDE the project with # `uv pip install --no-sources`, exactly as `release.yaml`'s compat cell builds its own. # - # It replaces the `toolchain_free` session of the noxfile the import retired. That - # noxfile's other lane, `plugin_floor` (the plugin on the oldest pytest of each Python), - # is deliberately NOT here: it belongs with the release that makes the plugin a shipped - # surface of its own. + # The extension, sphinx-test-reports, has hard Sphinx dependencies now, so nothing of it + # runs here; only its artefact fence does, because that fence reads archives and needs + # no toolchain. The plugin's oldest-pytest lane is `plugin-floor` below. steps: - uses: actions/checkout@v7 - uses: astral-sh/setup-uv@v10.2.0 @@ -500,8 +499,48 @@ jobs: enable-cache: true cache-suffix: toolchain-free cache-python: true - - name: Build the sdist and wheel as the release does, and check both - # The fence on the ARTEFACTS. Every other job installs this member editable, which + - name: Build ub-test-reports' sdist and wheel as the release does, and check both + # The fence on the core's ARTEFACTS, in `release.yaml`'s shape (no `--wheel`: the + # wheel is built FROM the sdist). flit ships one module, so what is checked is that it + # is the right one and complete -- every tracked file under `src/`, `schemas/JUnit.xsd` + # (which the XML validation reads at run time) and `py.typed` among them -- that the + # wheel's top level is exactly `ub_test_reports` and its dist-info, and that the sdist + # carries neither `tests/` (with its fixture copies) nor `docs/` + run: | + uv build --package ub-test-reports --no-sources -o /tmp/core-dist + git ls-files packages/ub-test-reports/src | uv run --no-project python -c " + import pathlib, sys, tarfile, zipfile + dist = pathlib.Path('/tmp/core-dist') + (wheel,) = dist.glob('*.whl') + (sdist,) = dist.glob('*.tar.gz') + wanted = {line.strip().split('/src/', 1)[1] for line in sys.stdin if line.strip()} + errors = [] + for name in ('ub_test_reports/__init__.py', 'ub_test_reports/schemas/JUnit.xsd', 'ub_test_reports/py.typed'): + if name not in wanted: + errors.append(f'{name} is not tracked under src/') + with tarfile.open(sdist) as tar: + members = [m.name.split('/', 1)[1] for m in tar.getmembers() if m.isfile()] + in_src = {name.removeprefix('src/') for name in members if name.startswith('src/')} + errors += [f'missing from {sdist.name}: src/{name}' for name in sorted(wanted - in_src)] + errors += [f'{sdist.name} ships {name}' for name in sorted(members) + if name.startswith(('tests/', 'docs/'))] + shipped = set(zipfile.ZipFile(wheel).namelist()) + errors += [f'missing from {wheel.name}: {name}' for name in sorted(wanted - shipped)] + top = {name.split('/', 1)[0] for name in shipped} + dist_info = {name for name in top if name.endswith('.dist-info')} + if len(dist_info) != 1 or top - dist_info != {'ub_test_reports'}: + errors.append(f'{wheel.name} top level is {sorted(top)}, not exactly ub_test_reports and one dist-info') + licences = sorted(n.split('/licenses/', 1)[1] for n in shipped if '.dist-info/licenses/' in n) + if licences != ['LICENSE']: + errors.append(f'{wheel.name} ships licence files {licences}, not exactly LICENSE') + print(f'{sdist.name}: {len(members)} files; {wheel.name}: {len(shipped)} entries; {len(wanted)} tracked files under src/') + sys.exit('\\n'.join(errors) if errors else 0) + " + - name: Build sphinx-test-reports' sdist and wheel as the release does, and check both + # The fence on the extension's ARTEFACTS. It only reads archives, so it needs no + # toolchain and lives here rather than in a job of its own; nothing below installs + # this wheel -- the extension needs Sphinx, and the release's compat cell walks and + # tests it. Every other job installs this member editable, which # reads `src/`, so a build configuration that dropped a package -- the # `sphinxcontrib/test_reports/` aliases most of all, which flit would drop without a # word -- would leave the whole suite green over a broken release. So this builds in @@ -558,12 +597,13 @@ jobs: sys.exit('\\n'.join(errors) if errors else 0) " - name: Build an environment with no documentation toolchain in it - # from the wheel checked above, so the modules below -- the aliases among them -- - # run against the artefact, not against the checkout + # from the core's wheel checked above, so its suite runs against the artefact, not + # against the checkout. `pytest-xdist` so that the plugin's three xdist tests run + # here instead of skipping on `importorskip` run: | uv venv --python 3.11 /tmp/toolchain-free - wheel=$(ls /tmp/str-dist/*.whl) - uv pip install --python /tmp/toolchain-free --no-sources "${wheel}[pytest]" packages/ub-project + wheel=$(ls /tmp/core-dist/*.whl) + uv pip install --python /tmp/toolchain-free --no-sources "${wheel}[pytest]" pytest-xdist packages/ub-project - name: Assert the toolchain really is absent # the fence's fence. Without this the job would still pass with Sphinx installed, # and would be testing nothing it claims to test. @@ -573,31 +613,75 @@ jobs: present = [m for m in ('sphinx', 'sphinx_needs', 'docutils') if importlib.util.find_spec(m)] sys.exit(f'toolchain installed: {present}' if present else 0) " - - name: Run the modules that need no documentation build - # From the REPOSITORY ROOT, so pytest's rootdir is this repository and the root - # `[tool.pytest.ini_options]` applies -- which is where the `toolchain` marker is - # registered, so `-m "not toolchain"` deselects rather than warning. - run: | - /tmp/toolchain-free/bin/python -m pytest -q -m "not toolchain" \ - packages/sphinx-test-reports/tests/test_aliases.py \ - packages/sphinx-test-reports/tests/test_cli_config.py \ - packages/sphinx-test-reports/tests/test_cli_convert.py \ - packages/sphinx-test-reports/tests/test_identity.py \ - packages/sphinx-test-reports/tests/test_junit_parser.py \ - packages/sphinx-test-reports/tests/test_junit_parser_gtest.py \ - packages/sphinx-test-reports/tests/test_needs_export.py \ - packages/sphinx-test-reports/tests/test_project_config.py \ - packages/sphinx-test-reports/tests/test_pytest_plugin.py \ - packages/sphinx-test-reports/tests/test_result_vocabulary.py \ - packages/sphinx-test-reports/tests/test_toolchain.py + - name: Run the ub-test-reports suite + # WHOLE, with no marker and no file list: every test of the core must pass without the + # toolchain. From the repository root, so the root `[tool.pytest.ini_options]` applies. + # The installed wheel is what is imported -- `src/` is never on `sys.path` under + # `--import-mode=importlib` -- so this tests the artefact. `-rs` prints any skip. + run: /tmp/toolchain-free/bin/python -m pytest -q -rs packages/ub-test-reports/tests - name: Run the ub-project suite - # A step of its own so that a red check names the suite. (One run over both would - # collect today -- measured -- but only because this suite has no `conftest.py`: - # add one and it resolves to `tests.conftest` beside test-reports', the collision - # the root `testpaths` comment describes.) The installed wheel is what is imported - # -- `src/` is never on `sys.path` -- so this also tests the artefact + # A step of its own, and it has to be: both suites have a `tests/__init__.py`, so one + # run over both resolves this suite beside ub-test-reports' `tests/conftest.py` under + # the one module name `tests.conftest` and stops with "Plugin already registered under + # a different name" (measured). A step per suite also names the red one. The installed + # wheel is what is imported -- `src/` is never on `sys.path` -- so this also tests the + # artefact run: /tmp/toolchain-free/bin/python -m pytest -q packages/ub-project/tests + plugin-floor: + name: Plugin floor (pytest ${{ matrix.pytest }}, Python ${{ matrix.python }}) + runs-on: ubuntu-latest + permissions: + contents: read + # ub-test-reports' pytest plugin declares `pytest>=7.0` (its `pytest` extra), and every + # other job runs the newest pytest. This runs the core's whole suite on the oldest pytest + # each floor Python can take: 7.0.1 on 3.11, and 7.3.2 on 3.12, the first pytest that runs + # there at all. Same environment shape as `toolchain-free` -- the built wheel, no + # toolchain -- with pytest pinned. + # + # From the PACKAGE directory, as the release's compat cell runs a suite: run from the + # repository root, pytest 7.3.2 loads the conftest of every root `testpaths` entry -- + # sphinx-needs' among them -- and dies on its imports (measured). `pytest-xdist` is not + # pinned: the newest (3.8.0, measured) declares `pytest>=7.0` and passes on both pins, so + # the plugin's three xdist tests run rather than skip. + strategy: + fail-fast: false + matrix: + include: + - python: "3.11" + pytest: "7.0.1" + - python: "3.12" + pytest: "7.3.2" + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.2.0 + with: + enable-cache: true + cache-suffix: plugin-floor-${{ matrix.python }} + cache-python: true + - name: Build an environment with the oldest pytest and no documentation toolchain + run: | + uv build --package ub-test-reports --no-sources --wheel -o /tmp/core-dist + uv venv --python ${{ matrix.python }} /tmp/plugin-floor + wheel=$(ls /tmp/core-dist/*.whl) + uv pip install --python /tmp/plugin-floor --no-sources "${wheel}[pytest]" \ + "pytest==${{ matrix.pytest }}" pytest-xdist packages/ub-project + - name: Assert the pin, and that the toolchain is absent + # the pin's positive control: a resolver that moved pytest would make this job test + # the newest pytest again while still reporting the floor + run: | + /tmp/plugin-floor/bin/python -c " + import importlib.util, sys + import pytest + if pytest.__version__ != '${{ matrix.pytest }}': + sys.exit(f'pytest {pytest.__version__} is installed, not ${{ matrix.pytest }}') + present = [m for m in ('sphinx', 'sphinx_needs', 'docutils') if importlib.util.find_spec(m)] + sys.exit(f'toolchain installed: {present}' if present else 0) + " + - name: Run the ub-test-reports suite + working-directory: packages/ub-test-reports + run: /tmp/plugin-floor/bin/python -m pytest -q -rs tests + bazel: name: Mounts Bazel integration runs-on: ubuntu-latest @@ -677,6 +761,7 @@ jobs: - docs-codelinks - docs-reports - toolchain-free + - plugin-floor - bazel - smoke-needs diff --git a/.github/workflows/test-extensions.yaml b/.github/workflows/test-extensions.yaml index 4119ed7ca..db9eb76e1 100644 --- a/.github/workflows/test-extensions.yaml +++ b/.github/workflows/test-extensions.yaml @@ -165,6 +165,24 @@ jobs: flags: reports files: ./reports.xml fail_ci_if_error: true + - name: "ub-test-reports: pytest" + # sphinx-test-reports' Sphinx-free core: the converter, the pytest plugin, the parsers + # and the `[test_reports]` model. Here for every cell's platform and interpreter -- + # Windows above all, where the converter's path handling and the plugin's source + # locations are the platform-sensitive part -- and for coverage under a flag of its + # own; `toolchain-free` in ci.yaml stays the run that proves it imports no Sphinx. + # Same `if:` as the steps above + if: ${{ !cancelled() && steps.prepare.outcome == 'success' }} + run: uv run --no-sync pytest -v packages/ub-test-reports/tests --cov=ub_test_reports --cov-report=xml:ub-test-reports.xml --cov-report=term-missing + - name: "ub-test-reports: upload to Codecov" + if: inputs.upload-coverage && github.event.pull_request.head.repo.full_name == github.repository && github.repository == 'useblocks/sphinx-needs' && github.actor != 'dependabot[bot]' + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} + name: ub-test-reports-pytests + flags: ub-test-reports + files: ./ub-test-reports.xml + fail_ci_if_error: true - name: "ub-project: pytest" # The shared `ubproject.toml` reader, which every extension above is to depend on. # Its suite takes under a second and needs nothing this cell does not already have; diff --git a/codecov.yml b/codecov.yml index 12ed7f29d..5dfc9c0c5 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,7 +1,6 @@ -# One repository, three packages, three coverage uploads -- so three flags with targets of -# their own. Without them all three would land on the `default` status and a young -# package's coverage would be averaged into a mature one's 80%, moving a gate nobody -# decided to move. +# One repository, one coverage upload per package -- so one flag each, with a target of its +# own. Without them every upload would land on the `default` status and a young package's +# coverage would be averaged into a mature one's 80%, moving a gate nobody decided to move. coverage: status: project: @@ -32,6 +31,14 @@ coverage: flags: [reports] target: auto threshold: 0.5% + # ub-test-reports, sphinx-test-reports' Sphinx-free core, uploaded by the same cell with + # `flags: ub-test-reports`. `target: auto` because its statements MOVED out of + # `reports` when the core was split off -- that flag's number moves with them, and + # neither number says anything about a change in coverage on that pull request + ub-test-reports: + flags: [ub-test-reports] + target: auto + threshold: 0.5% patch: default: target: 67% diff --git a/packages/sphinx-test-reports/.readthedocs.yaml b/packages/sphinx-test-reports/.readthedocs.yaml index 1924ce12d..e7732e306 100644 --- a/packages/sphinx-test-reports/.readthedocs.yaml +++ b/packages/sphinx-test-reports/.readthedocs.yaml @@ -72,14 +72,19 @@ sphinx: python: install: - # The workspace sibling FIRST, from this checkout. Read the Docs runs these entries in - # order with pip, and a member installed on its own would resolve `sphinx-needs` from - # PyPI -- the released version, not the one beside it -- so a docs build here could - # pass or fail on code the workspace does not have. With the sibling already installed - # the member's requirement is satisfied and pip fetches nothing (tight tracking keeps - # the checkout's version inside the member's floor and cap). + # The workspace siblings FIRST, from this checkout. Read the Docs runs these entries in + # order with pip, and a member installed on its own would resolve `sphinx-needs` and + # `ub-test-reports` from PyPI -- the released versions, not the ones beside it -- so a + # docs build here could pass or fail on code the workspace does not have, and between a + # core change and its release pip finds no matching `ub-test-reports` at all (measured: + # "Could not find a version that satisfies the requirement ub-test-reports<2,>=1.0.0.dev0"). + # With the siblings already installed the member's requirements are satisfied and pip + # fetches neither (tight tracking keeps the checkout's versions inside the member's floors + # and caps). - method: pip path: packages/sphinx-needs + - method: pip + path: packages/ub-test-reports - method: pip path: packages/sphinx-test-reports extra_requirements: From d85cf9a65c56cec0dc9106fb8ccbda923ba467d4 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Thu, 1 Oct 2026 02:31:12 +0200 Subject: [PATCH 4/9] =?UTF-8?q?=F0=9F=91=8C=20ub-test-reports:=20test=20th?= =?UTF-8?q?at=20validate()=20reads=20the=20shipped=20JUnit=20schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `schemas/JUnit.xsd` is package data whose only reader is `JUnitParser.validate()`, and nothing called it: a wheel that shipped without the schema passed the whole suite (measured while proving the toolchain-free artefact check). Two direct tests now load it -- a conforming report validates, pytest's own report does not -- so the suite run against a built wheel fails when the schema is missing. --- .../tests/test_junit_parser.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/packages/ub-test-reports/tests/test_junit_parser.py b/packages/ub-test-reports/tests/test_junit_parser.py index 428de4c21..94fee78ba 100644 --- a/packages/ub-test-reports/tests/test_junit_parser.py +++ b/packages/ub-test-reports/tests/test_junit_parser.py @@ -240,3 +240,48 @@ def test_error_counts_are_taken_from_the_testsuite(): assert suite["errors"] == 2 assert suite["failures"] == 0 assert suite["passed"] == 1 + + +#: The smallest report the shipped Apache Ant JUnit schema accepts: every required +#: attribute of `` and ``, and the four child elements in order. +CONFORMING_REPORT = """\ + + + + + + + +""" + + +class TestSchemaValidation: + """`validate()` reads `schemas/JUnit.xsd` from the installed package. + + The schema is package data, so it is what an artefact check must not lose: these + tests are the ones that fail when a built wheel ships without it. + """ + + def test_the_shipped_schema_accepts_a_conforming_report(self, tmp_path): + from pathlib import Path + + from ub_test_reports.junitparser import JUnitParser + + report = tmp_path / "conforming.xml" + report.write_text(CONFORMING_REPORT, encoding="utf-8") + parser = JUnitParser(str(report)) + + assert Path(parser.junit_xsd_path).is_file() + assert parser.validate() is True + + def test_the_shipped_schema_rejects_a_report_without_its_required_attributes( + self, + ): + from ub_test_reports.junitparser import JUnitParser + + # pytest's report has no `hostname` or `timestamp` on its `` + parser = JUnitParser(xml_pytest_path) + + assert parser.validate() is False + assert len(parser.xmlschema.error_log) > 0 From 329007392439ddc05aaa4582eb7c40de3942e553 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Thu, 1 Oct 2026 02:38:53 +0200 Subject: [PATCH 5/9] =?UTF-8?q?=F0=9F=93=9A=20Docs,=20changelogs,=20AGENTS?= =?UTF-8?q?=20and=20README=20for=20the=20split?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One documentation site for both distributions: the index groups the pages into "The Sphinx extension", "Without Sphinx (ub-test-reports)" and support, and says the version shown is the extension's. ub-test-reports' changelog is included as a page of its own. `docs/conf.py` reads the version from the installed distribution instead of a literal. install.rst describes two distributions and three install lines, with a 3.0.0 note; its paragraph on Sphinx-Needs 6.0.1 and the load-time check, false already, now states the real floors. cli.rst, pytest.rst and the `[test_reports]` section of configuration.rst name ub-test-reports where the code now lives. The extension's Unreleased changelog is rewritten where the split makes it false and gains the split's own entries: ub-test-reports exists; `pip install sphinx-test-reports` brings Sphinx, Sphinx-Needs and docutils again (and the fix for a CI or Bazel action that installed it for the command: install ub-test-reports); `[sphinx]` and `[pytest]` kept until 4.0; the old module names need the extension; the plugin's printed names; the load-time check and compat-requirements.txt gone. AGENTS.md: the extension's describes the extension alone; ub-test-reports' holds the core's rules; the root's table, naming, ub-project and command paragraphs follow. README.md lists ub-test-reports and ub-project. --- AGENTS.md | 30 ++-- README.md | 4 +- packages/sphinx-test-reports/AGENTS.md | 142 +++++++----------- .../sphinx-test-reports/docs/changelog.rst | 101 ++++++++++--- packages/sphinx-test-reports/docs/cli.rst | 15 +- packages/sphinx-test-reports/docs/conf.py | 10 +- .../docs/configuration.rst | 5 + packages/sphinx-test-reports/docs/index.rst | 23 ++- packages/sphinx-test-reports/docs/install.rst | 45 +++--- packages/sphinx-test-reports/docs/pytest.rst | 28 ++-- .../docs/ub-test-reports-changelog.rst | 13 ++ packages/ub-test-reports/AGENTS.md | 22 ++- 12 files changed, 265 insertions(+), 173 deletions(-) create mode 100644 packages/sphinx-test-reports/docs/ub-test-reports-changelog.rst diff --git a/AGENTS.md b/AGENTS.md index f85f91646..efa4cdf69 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,8 @@ the shim. | its documentation | `packages/sphinx-needs/docs/` (changelog: `docs/changelog.rst`) | | sphinx-mounts' behaviour, tests, documentation | `packages/sphinx-mounts/{src/sphinx_mounts,tests,docs}/` — start at [`packages/sphinx-mounts/AGENTS.md`](packages/sphinx-mounts/AGENTS.md) | | sphinx-codelinks' behaviour, tests, documentation | `packages/sphinx-codelinks/{src/sphinx_codelinks,tests,docs}/` — start at [`packages/sphinx-codelinks/AGENTS.md`](packages/sphinx-codelinks/AGENTS.md) | -| sphinx-test-reports' behaviour, tests, documentation | `packages/sphinx-test-reports/{src/sphinx_test_reports,tests,docs}/` — start at [`packages/sphinx-test-reports/AGENTS.md`](packages/sphinx-test-reports/AGENTS.md) | +| sphinx-test-reports' behaviour (the Sphinx extension), tests, documentation | `packages/sphinx-test-reports/{src/sphinx_test_reports,tests,docs}/` — start at [`packages/sphinx-test-reports/AGENTS.md`](packages/sphinx-test-reports/AGENTS.md); its `docs/` are ub-test-reports' too | +| ub-test-reports' behaviour (the converter, the pytest plugin, the parsers, the `[test_reports]` model — no Sphinx), tests | `packages/ub-test-reports/{src/ub_test_reports,tests}/` — start at [`packages/ub-test-reports/AGENTS.md`](packages/ub-test-reports/AGENTS.md) | | how every tool reads `ubproject.toml` — finding, anchoring, `[variants]`, the variant-data merge | `packages/ub-project/{src/ub_project,tests,design}/` — start at [`packages/ub-project/AGENTS.md`](packages/ub-project/AGENTS.md) | | the fixtures, helpers and renderer resolution three suites share | `packages/sphinx-needs-testkit/` — a member this repository never publishes, installed through the root's `test` group and loaded by each suite's `tests/conftest.py` as a pytest plugin | | the three conformance corpora | `packages/sphinx-needs/tests/conformance/` (needflow) and `packages/sphinx-mounts/tests/fixtures/variant_condition_conformance.toml` (variant conditions), whose repository of record is ubCode, and `packages/ub-project/tests/fixtures/ubproject_reading_conformance.toml` (reading `ubproject.toml`), whose record is THIS repository and which ubCode is to vendor — all shared byte-for-byte; do not reformat any of them (`.gitattributes` plus the yamlfmt and taplo excludes protect them) | @@ -37,7 +38,7 @@ the shim. | the PlantUML renderer | `vendor/plantuml/` — `pin.toml` (version + sha256, the one place either is written), the committed `plantuml-.jar` it names, and a `README.md`. `uv run poe verify-plantuml` fences the two against each other | | CI | `.github/workflows/`, and `.github/scripts/` for the three checks that must run *inside* a CI environment | | the docker image | `docker/` — a repository-level deliverable, like the workflows | -| Read the Docs | sphinx-needs: `.readthedocs.yml`, and it stays at the root under that exact name — the configuration path applies to every version, so moving it makes older tags unbuildable. sphinx-mounts: `packages/sphinx-mounts/.readthedocs.yaml`, sphinx-codelinks: `packages/sphinx-codelinks/.readthedocs.yaml`, and sphinx-test-reports: `packages/sphinx-test-reports/.readthedocs.yaml`, each of which its own RTD project points at; every path inside those is relative to the REPOSITORY root, not to the file | +| Read the Docs | sphinx-needs: `.readthedocs.yml`, and it stays at the root under that exact name — the configuration path applies to every version, so moving it makes older tags unbuildable. sphinx-mounts: `packages/sphinx-mounts/.readthedocs.yaml`, sphinx-codelinks: `packages/sphinx-codelinks/.readthedocs.yaml`, and sphinx-test-reports: `packages/sphinx-test-reports/.readthedocs.yaml`, each of which its own RTD project points at; every path inside those is relative to the REPOSITORY root, not to the file. ub-test-reports has no RTD project: it is documented on sphinx-test-reports' site, whose yaml installs it from the checkout first. ub-project has no docs site | **`tools/` is the workspace's tooling — a virtual member, never released, whose manifest declares the tooling's dependencies; `.github/scripts/` keeps only the checks that must @@ -75,17 +76,18 @@ rejects on upload, for the by-hand path. nothing**: standard library only, no Sphinx, fenced by its own `tests/test_imports.py` and by CI's `toolchain-free` job, which runs its suite where Sphinx is not installed. Its contract is its conformance corpus (ubCode is to vendor it) plus `design/reading-contract.md`; -it decides no policy — discovery, warnings and `-D` stay with each consumer. No member -depends on it yet: every `--no-sources` gate resolves a consumer from the index, so the -consumers arrive one pull request each after its first release, and from then on each -release of it re-floors all of them. +it decides no policy — discovery, warnings and `-D` stay with each consumer. sphinx-needs, +sphinx-mounts, sphinx-codelinks and ub-test-reports depend on it, so each release of it +re-floors all of them (`propagate_floors.py`). **Naming: `sphinx-*` is a Sphinx extension; `ub-*` is a useblocks package that is not one** -— a tool (such as the future `ub-test-reports`) or a library (such as `ub-project`, import -`ub_project`). The name does not say which of the two a `ub-*` package is; its README and +— a tool (such as `ub-test-reports`, import `ub_test_reports`) or a library (such as +`ub-project`, import `ub_project`). The name does not say which of the two a `ub-*` package is; its README and classifiers do. A `ub-*` library says in its README's first line that it is a library for the sphinx-needs family which the extensions pull in, and carries no `Framework :: Sphinx` -classifier. Its poe tasks keep the whole name (`test-ub-project`): only `sphinx-` is dropped. +classifier; a `ub-*` tool says it is a tool for the family, and carries none either. Their +poe tasks keep the whole name (`test-ub-project`, `test-ub-test-reports`): only `sphinx-` is +dropped. ## Commands @@ -96,6 +98,7 @@ uv run poe test-needs -k # trailing words are appended to the task' uv run poe test-mounts # the sphinx-mounts suite (bazel tests deselected) uv run poe test-codelinks # the sphinx-codelinks suite (adds the libclang group) uv run poe test-reports # the sphinx-test-reports suite +uv run poe test-ub-test-reports # the ub-test-reports suite (no Sphinx needed) uv run poe lint # every prek hook over the whole tree uv run poe typecheck # ty over both packages, against the oldest supported sphinx uv run poe typecheck-js-needs # tsc over the vendored needstable.js (needs node) @@ -110,6 +113,7 @@ uv run poe check-workspace # the manifests agree with each other (Lin uv run poe release-plan # what is pending, in what order (advice; exits 0) uv run poe bump --bump minor # stamp a release: version, literals, floors, lock, changelog uv run poe import-check-needs # import the wheel against PyPI-resolved dependencies +uv run poe import-check-ub-test-reports # the same for ub-test-reports (with its pytest extra) uv run poe import-check-codelinks # the same for sphinx-codelinks (with its libclang extra) uv run --frozen --no-sync pytest tools/tests -q # the tooling's own tests UV_PYTHON=3.12 uv run --no-sync poe test-needs-sphinx8 # one CI matrix cell @@ -169,7 +173,7 @@ and the testkit all are. The sdist is ≈7.4 MB rather than 28. **sphinx-test-reports is the two cases at once**: its SUITE needs neither renderer (no test document carries a rendering directive, and the dead 8.6 MB jar its tests used to carry left -with the import), while `docs-reports` DOES render — 13 `needflow` directives — so it needs +with the import; ub-test-reports' suite needs none either), while `docs-reports` DOES render — 13 `needflow` directives — so it needs `java` and `dot`, and its `.readthedocs.yaml` keeps `apt_packages` where both SIBLING extensions' have none. (sphinx-needs' own docs render too, and more: 48 needflow directives.) @@ -251,12 +255,12 @@ the rootdir, so the tasks carry `--ignore=performance` instead of naming `tests` the task's own command would be *added* to yours rather than replaced by it.) **A bare `pytest` at the root collects sphinx-needs' suite and the tooling's — not -sphinx-mounts', sphinx-codelinks', sphinx-test-reports' or ub-project's.** Their `tests` directories are deliberately +sphinx-mounts', sphinx-codelinks', sphinx-test-reports', ub-test-reports' or ub-project's.** Their `tests` directories are deliberately absent from `testpaths`: every package ships a `tests/__init__.py`, so under `--import-mode=importlib` every `conftest.py` resolves to the module name `tests.conftest` and a rootdir-invoked pytest refuses the second outright — listing one there collects *nothing*, rather than more. Run those suites through `poe test-mounts`, -`poe test-codelinks`, `poe test-reports` and `poe test-ub-project` (which cd into the package), the way CI does +`poe test-codelinks`, `poe test-reports`, `poe test-ub-test-reports` and `poe test-ub-project` (which cd into the package), the way CI does with an explicit path. The Lint job's "Check a bare root pytest still collects" step is what keeps the list honest. @@ -457,7 +461,7 @@ removed* (`error-on-warning` makes an unused suppression an error), never by loo Every issue and pull request carries one or more `pkg:` labels naming what it concerns: `pkg: ` (today `pkg: sphinx-needs`, `pkg: sphinx-mounts`, -`pkg: sphinx-codelinks`, `pkg: sphinx-test-reports`, `pkg: sphinx-needs-testkit` and `pkg: ub-project`) or `pkg: workspace` for the repository +`pkg: sphinx-codelinks`, `pkg: sphinx-test-reports`, `pkg: sphinx-needs-testkit`, `pkg: ub-test-reports` and `pkg: ub-project`) or `pkg: workspace` for the repository itself — workflows, CI, release, docker, tooling, the workspace root. Pull requests get theirs automatically from the paths they touch (`.github/labeler.yml`); the issue forms set it from their "Package" dropdown (`.github/issue-labeler.yml`). **An issue created diff --git a/README.md b/README.md index 59559ecd0..75ba10b73 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,9 @@ definitions and the CI workflows. | [`packages/sphinx-needs`](packages/sphinx-needs) | [`sphinx-needs`](https://pypi.org/project/sphinx-needs/) | the Sphinx extension for managing requirements and specifications — [documentation](https://sphinx-needs.readthedocs.io), [README](packages/sphinx-needs/README.rst) | | [`packages/sphinx-mounts`](packages/sphinx-mounts) | [`sphinx-mounts`](https://pypi.org/project/sphinx-mounts/) | the Sphinx extension that mounts external source trees into a build without copying or symlinking — [documentation](https://sphinx-mounts.useblocks.com), [README](packages/sphinx-mounts/README.md) | | [`packages/sphinx-codelinks`](packages/sphinx-codelinks) | [`sphinx-codelinks`](https://pypi.org/project/sphinx-codelinks/) | fast source-code traceability for sphinx-needs — it scans source files for comment markers, turns them into needs, and links documentation to exact source lines — [documentation](https://codelinks.useblocks.com), [README](packages/sphinx-codelinks/README.md) | -| [`packages/sphinx-test-reports`](packages/sphinx-test-reports) | [`sphinx-test-reports`](https://pypi.org/project/sphinx-test-reports/) | test results as needs: JUnit/ctest/googletest XML and tox-envreport JSON become needs in a build, and a `test-reports` command turns the same reports into a `needs.json` without running Sphinx — [documentation](https://sphinx-test-reports.readthedocs.io), [README](packages/sphinx-test-reports/README.rst) | +| [`packages/sphinx-test-reports`](packages/sphinx-test-reports) | [`sphinx-test-reports`](https://pypi.org/project/sphinx-test-reports/) | test results as needs: the Sphinx extension in which JUnit/ctest/googletest XML and tox-envreport JSON become needs in a build — [documentation](https://sphinx-test-reports.readthedocs.io), [README](packages/sphinx-test-reports/README.rst) | +| [`packages/ub-test-reports`](packages/ub-test-reports) | `ub-test-reports` (not yet on PyPI) | the Sphinx-free half of test reports, which sphinx-test-reports depends on: the `test-reports` command that turns reports into a `needs.json` without running Sphinx, the pytest plugin, the parsers — documented on sphinx-test-reports' site, [README](packages/ub-test-reports/README.rst) | +| [`packages/ub-project`](packages/ub-project) | [`ub-project`](https://pypi.org/project/ub-project/) | the shared reader for `ubproject.toml` and its variant data, which the other packages depend on — no documentation site, [README](packages/ub-project/README.rst) | ## Why one repository, and why still several packages diff --git a/packages/sphinx-test-reports/AGENTS.md b/packages/sphinx-test-reports/AGENTS.md index ab9219592..a1a62c7a5 100644 --- a/packages/sphinx-test-reports/AGENTS.md +++ b/packages/sphinx-test-reports/AGENTS.md @@ -8,50 +8,37 @@ not of the workspace. ## Project Overview -sphinx-test-reports turns test results into needs. It has **three surfaces, and only one of -them is a Sphinx extension** — which is the single most important thing to know about this -package, because it shapes the manifest, the CI and the split that is coming: - -- **the extension** — `test-file`, `test-suite`, `test-case`, `test-report`, `test-results` - and `test-env` directives, which read JUnit / ctest / googletest XML and tox-envreport - JSON and create sphinx-needs items from them, plus the `tr_link` dynamic function; -- **the converter** — a `test-reports` console script that turns the same reports into a - `needs.json` **without running Sphinx at all**; -- **the pytest plugin** — `sphinx_test_reports.pytest_plugin`, which writes the XML - shape the extension reads, including per-case properties for traceability. - -So **Sphinx and sphinx-needs are an `[project.optional-dependencies]` extra, not -dependencies**: `pip install sphinx-test-reports` gets you `lxml`, `ub-project` and the -last two surfaces; `pip install "sphinx-test-reports[sphinx]"` gets you the extension. The -published wheel's `Requires-Dist` is those two alone. Two things in this repository exist -because of that — the `toolchain-free` CI job and this package's `compat-requirements.txt` -— and both are described below. +sphinx-test-reports is **the Sphinx extension, and only that**: the `test-file`, +`test-suite`, `test-case`, `test-report`, `test-results` and `test-env` directives, which read +JUnit / ctest / googletest XML and tox-envreport JSON and create sphinx-needs items from +them, plus the `tr_link` dynamic function. **Its dependencies are hard** — Sphinx, docutils, +sphinx-needs and `ub-test-reports`. The converter (`test-reports`), the pytest plugin, the +parsers, the result vocabulary, the deterministic IDs and the `[test_reports]` model are +**ub-test-reports** ([`packages/ub-test-reports/`](../ub-test-reports/AGENTS.md)), which +runs without Sphinx and which this package depends on; a change to any of those belongs +there. `[sphinx]` (empty) and `[pytest]` (a pass-through to `ub-test-reports[pytest]`) stay +as extras only so that 2.0.0's install lines keep working until 4.0. ## Package structure ```text -pyproject.toml # `[project]`, `[project.urls]`, `[project.scripts]` and the - # hatch build tables. NOT ruff, ty, pytest or dependency - # groups: those are the root's, and check (7) refuses them here -compat-requirements.txt # released deps the compat cell needs -- see "Releasing" below +pyproject.toml # `[project]`, `[project.urls]` and the hatch build tables. NOT + # ruff, ty, pytest or dependency groups: those are the root's, + # and check (7) refuses them here .readthedocs.yaml # this package's RTD project; its paths are REPOSITORY-root relative AUTHORS · LICENSE · README.rst design/ # import-commit-map.txt: old hash -> new hash for the 2026-09 import src/sphinxcontrib/test_reports/ # the pre-3.0 name: four warning aliases, removed in 4.0 src/sphinx_test_reports/ -├── __init__.py # `__version__`, and the lazy `setup` re-export -├── test_reports.py # the extension entry point: directives, config values, fields -├── cli.py # the `test-reports` converter command -├── pytest_plugin.py # the pytest plugin -├── junitparser.py · jsonparser.py · results.py · identity.py · fields.py -│ # the toolchain-free core: parsers, the result vocabulary, the -│ # deterministic case IDs, the one field table both writers share -├── projectconfig.py # the `[test_reports]` ubproject.toml model, read through ub-project -├── needs_export.py · remote.py · config.py · environment.py · exceptions.py · toolchain.py +├── __init__.py # `__version__` FIRST, then the eager `setup` import -- the order is +│ # load-bearing: `test_reports` imports `__version__` from here +├── test_reports.py # the extension entry point: directives, config values, the bridge +│ # that applies ub-test-reports' `[test_reports]` model +├── config.py · environment.py · exceptions.py ├── directives/ # one module per directive, all inheriting TestCommonDirective ├── functions/ # `tr_link`, a sphinx-needs dynamic function -├── css/ · schemas/JUnit.xsd +├── css/ └── directives/test_report_template.txt # the DEFAULT tr_report_template -- it SHIPS tests/ # `tests/__init__.py` is why this path is not in the root testpaths @@ -66,24 +53,27 @@ The package was `sphinxcontrib.test_reports` until 3.0. `src/sphinxcontrib/test_ keeps exactly four old names working until 4.0: the package as a Sphinx extension, which warns through Sphinx's logger (type `test_reports.deprecated`) and loads the real extension with `app.setup_extension`, and `pytest_plugin`, `junitparser` and `jsonparser`, one file -each, which put the REAL module into `sys.modules` under the old name with one -`FutureWarning` per process. **Do not add a finder or a catch-all**: every other old name is meant to -fail as a plain `ImportError`, and `tests/test_aliases.py` walks the real package -to hold that. **There is no `src/sphinxcontrib/__init__.py`, and there must never be +each, which put the REAL module -- `ub_test_reports.`, in the core -- into +`sys.modules` under the old name with one `FutureWarning` per process. **Do not add a finder +or a catch-all**: every other old name is meant to fail as a plain `ImportError`, and +`tests/test_aliases.py` walks BOTH real packages, this one and ub-test-reports, to hold that +-- so a module added to either is covered without anyone remembering the file. **There is no `src/sphinxcontrib/__init__.py`, and there must never be one**: `sphinxcontrib` is a PEP 420 namespace other distributions install into. ### hatchling, and the fence on the built artefacts This is the one member that builds with hatchling: its wheel ships two top-level packages, and flit ships one and drops the other without a word. An editable install reads `src/`, so -a build configuration that lost the aliases would leave every test green. The -`toolchain-free` job is therefore where the artefacts are checked. It builds in the -release's shape -- the sdist, then the wheel FROM the sdist, so the sdist's include list -bounds what the wheel ships -- and fails when either lacks a tracked file under `src/`, -when the sdist's files outside `src/` are not exactly its metadata files, or when the -wheel's top level is anything but the two packages and its dist-info, it ships anything -under `sphinxcontrib/` but `test_reports/`, or a licence file other than `LICENSE`; then it -runs its modules against that wheel. A new top-level package needs a line in +a build configuration that lost the aliases would leave every test green. CI's +`toolchain-free` job is therefore where the artefacts are checked -- the job is named for +ub-test-reports, whose suite it runs without Sphinx, and this fence lives there because it +only reads archives. It builds in the release's shape -- the sdist, then the wheel FROM the +sdist, so the sdist's include list bounds what the wheel ships -- and fails when either lacks +a tracked file under `src/`, when the sdist's files outside `src/` are not exactly its +metadata files, or when the wheel's top level is anything but the two packages and its +dist-info, it ships anything under `sphinxcontrib/` but `test_reports/`, or a licence file +other than `LICENSE`. Nothing installs that wheel there: it needs the toolchain, and the +release's compat cell is what walks and tests it. A new top-level package needs a line in `[tool.hatch.build.targets.wheel]` AND `[tool.hatch.build.targets.sdist]`. ### The suite needs no renderer; the DOCS need two @@ -119,43 +109,21 @@ The short name is **`reports`**, not `test-reports`: the naming rule takes the d name minus its `sphinx-` prefix, which here would collide with the task verb and give `test-test-reports`. -### One test drives an in-process pytest session, and the default environment breaks it - -`tests/test_pytest_plugin.py`'s `NESTED` fixture starts a `pytest.main()` *inside* a -`pytester` session. In a developer's default `.venv` that process has every installed plugin -loaded, and **pytest-playwright** — from the root `js` group, which the default `dev` group -includes — keeps a module-global soft-assertion scope, so the nested session dies with -*nested soft assertion scopes are not supported*. Every CI cell syncs -`--no-default-groups --group test --group sphinx-N`, where the plugin is absent. The fixture -therefore passes `-p no:playwright`, and **that line has to stay one line**: a sibling test -builds its own fixture from the same string by replacing the literal `str(inner)]) == 0`. - -This is the shape to remember: **run the suite in the default `.venv` AND in a cell.** Green -in one proves nothing about the other. - ### The old name still appears in the tree, on purpose -Three fixture files in `tests/doc_test/utils/` carry paths like -`file="sphinxcontrib/test_reports/junitparser.py"`: **test data** describing a historical -pytest run, not paths anything opens. The docs' `classname` examples match that data, and +Fixture files in `tests/doc_test/utils/` (and their copies under ub-test-reports' +`tests/fixtures/`) carry paths like `file="sphinxcontrib/test_reports/junitparser.py"`: +**test data** describing a historical pytest run, not paths anything opens. The docs' `classname` examples match that data, and the pytest plugin's reserved `user_properties` names (`sphinxcontrib.test_reports:file`, `:line`) are documented wire names. None of them is an import path, so none moved with the package; a rename `sed` over the tree would corrupt them silently. -### `ubproject.toml` is read through `ub-project` - -`projectconfig.py` finds, loads and anchors the file through `ub-project`, the workspace's -shared reader (a runtime dependency, standard library only): `find_project_config` is its -walk re-exported, and `load_toml` and `anchor` do the reading and the joining. What stays -here is this package's policy — the `[test_reports]` keys, their types, the normalisation, -unknown keys warned rather than fatal — and **`TomlConfigError`, which is still the only -exception either consumer catches**: `load_project_config` re-raises ub-project's -`ProjectConfigError` as it with the same message, and must never be made a subclass of it. -The walk stops at the first `ubproject.toml`, else at `.git`, and only where no `.git` -exists anywhere above, at `pyproject.toml` — so a member's `packages//pyproject.toml` -never ends it, and a repository-root `ubproject.toml` (there is none today) would be picked -up by every consumer under `packages/`. `packages/ub-project/design/reading-contract.md` is -the specification; a change the walk or the anchoring needs belongs there, not here. +### `ubproject.toml` is read by ub-test-reports + +The `[test_reports]` model -- keys, types, normalisation, `TomlConfigError` -- is +`ub_test_reports.projectconfig`, read through `ub-project`; the core's `AGENTS.md` has its +rules. What is here is the bridge in `test_reports.py` that applies it to the `tr_*` values +at `config-inited`, and `-D` precedence over it. ## Testing @@ -167,21 +135,17 @@ The suite spawns Sphinx builds in three places, and all three go through `tests/test_subprocess_fence.py` is what keeps that true; it reads SOURCE, because a site that spawns the bare word passes in every environment where `PATH` happens to be right. -**`toolchain-free` (a CI job in `ci.yaml`, not a cell) is the fence that keeps the converter -and the plugin importable with no documentation toolchain.** It cannot be a `uv sync` cell, -structurally: the root's `[project] dependencies` name every member, those are installed in -every environment, and sphinx-needs declares sphinx at runtime — so every environment this -root can produce has Sphinx in it. The job builds one outside the project with -`uv pip install --no-sources` of the wheel it has just built and checked, asserts `sphinx`, -`sphinx_needs` and `docutils` are all absent, and runs the toolchain-free modules with -`-m "not toolchain"`. The `toolchain` marker itself lives in the ROOT's `markers` list. +**This suite always has Sphinx**, and every test in it may use it: the converter's and the +plugin's tests are ub-test-reports' and run without the toolchain there. A test of a +Sphinx-free module does not belong here — not even split by a marker, which this package no +longer has. ## Releasing -`compat-requirements.txt` names `sphinx` and `sphinx-needs`, and unlike both siblings' it -covers the package's **own runtime** requirements rather than a test-only need — because they -are optional. The compat cell installs a bare wheel path with no extras, so without that file -the `import_check` walk fails `6 of 25 modules` on `sphinx_needs` before pytest even starts. +**ub-test-reports releases first.** This package's floor on the core is tight-tracked, so +the core's release pull request (`poe bump ub-test-reports …`) rewrites it, and every +release gate here resolves the core from PyPI: `poe import-check-reports`, the release +plan and the compat cell are red until the core version this tree names is published. ## What the move into this workspace cost, deliberately @@ -193,8 +157,8 @@ Recorded here because none of it is visible in a diff: - **Five ruff rule families left**: `FURB`, `PERF`, `PGH`, `PIE`, `SLF`, which this package enabled and the root's shared set does not. All five are at zero violations today, which is exactly why the loss would otherwise be silent. -- **`plugin_floor`** — the plugin against the oldest pytest of each Python — has no - replacement yet. It returns with the release that makes the plugin a shipped surface. +- **`plugin_floor`** — the plugin against the oldest pytest of each Python — returned as + `ci.yaml`'s `plugin-floor` job when the plugin moved to ub-test-reports. - **mypy left for ty**, with the whole package checked rather than the 15-entry `exclude` the mypy configuration carried. - **Beyond those five families, the root `extend-ignore`s `B904`, `ICN001`, `ISC004` and diff --git a/packages/sphinx-test-reports/docs/changelog.rst b/packages/sphinx-test-reports/docs/changelog.rst index c682e2dfc..8822d3790 100644 --- a/packages/sphinx-test-reports/docs/changelog.rst +++ b/packages/sphinx-test-reports/docs/changelog.rst @@ -6,13 +6,67 @@ Changelog Unreleased ---------- +The Sphinx-free half is its own distribution, ub-test-reports +............................................................. + +- ‼️ The ``test-reports`` converter, the pytest plugin, the parsers, the result + vocabulary, the deterministic case IDs and the ``[test_reports]`` model of + ``ubproject.toml`` are now **ub-test-reports** 1.0.0, a distribution of their own with no + Sphinx in it, imported as ``ub_test_reports``. They moved unchanged:: + + pip install ub-test-reports # the test-reports command + pip install "ub-test-reports[pytest]" # and the pytest plugin + + sphinx-test-reports is the Sphinx extension alone, and depends on ub-test-reports, so + both still arrive with it. + +- ‼️ **pip install sphinx-test-reports brings Sphinx, Sphinx-Needs, docutils and + ub-test-reports again.** This reverses 2.0.0's install-footprint change: the extension's + dependencies are hard dependencies now, not the ``sphinx`` extra. A CI job or a Bazel + action that installed sphinx-test-reports only for the ``test-reports`` command or the + pytest plugin now gets the whole documentation toolchain -- and a resolver conflict + wherever that environment pins another Sphinx. Install ``ub-test-reports`` there instead: + it is the same command and the same plugin, without the toolchain. + +- The ``sphinx`` extra is accepted and ignored until 4.0 -- ``pip install + "sphinx-test-reports[sphinx]"`` installs exactly what the bare line does -- and the + ``pytest`` extra passes through to ``ub-test-reports[pytest]`` until 4.0, so 2.0.0's + documented install lines keep working. + +- ‼️ **The pytest plugin is** ``-p ub_test_reports.pytest_plugin``. Its pluggy registration + name is now ``ub_test_reports.xml_shape`` and the configuration warnings it issues start + with ``ub_test_reports.pytest_plugin:`` (2.0.0: ``sphinxcontrib.test_reports.xml_shape`` + and ``sphinxcontrib.test_reports.pytest_plugin:``); a warning filter matching that text + needs the new prefix. The two ``user_properties`` wire names it reserves do not change. + +- ‼️ **The old module names need the extension.** ``sphinxcontrib.test_reports.junitparser``, + ``.jsonparser`` and ``.pytest_plugin`` (below) are shipped by sphinx-test-reports, so they + resolve only where it is installed -- which now means with Sphinx. On 2.0.0 a Sphinx-free + environment could ``pip install sphinx-test-reports`` and import them; on 3.0 such an + environment installs ``ub-test-reports`` and imports ``ub_test_reports.junitparser`` (and + so on). With only ub-test-reports installed the old names fail as a plain + ``ModuleNotFoundError: No module named 'sphinxcontrib'``. + +- 🔧 The load-time toolchain check is gone. It existed because the toolchain was an opt-in + extra pip never saw; as hard dependencies, pip resolves the floors itself. An environment + whose Sphinx-Needs is downgraded below the floor AFTER installing now fails with a + traceback from inside a directive rather than a one-line error naming the install line; + ``pip check`` (or ``uv pip check``) names the conflict. The extension itself still refuses + a Sphinx older than 7.4 when it loads, with Sphinx's own version error. With the check went + ``compat-requirements.txt``, which the release's compatibility cell needed only while the + toolchain was optional. + +- The modules that moved were ``sphinx_test_reports.`` only in this unreleased + version -- 2.0.0 shipped them as ``sphinxcontrib.test_reports.`` -- so no alias is + kept for the ``sphinx_test_reports`` spelling; the old names that do keep working are the + four below. + The import name moves ..................... -- ♻️ The package is imported as ``sphinx_test_reports`` now, not - ``sphinxcontrib.test_reports``: the extension is ``extensions = ["sphinx_test_reports"]`` - and the pytest plugin is ``-p sphinx_test_reports.pytest_plugin``. Nothing else about - either changes. +- ♻️ The extension is imported as ``sphinx_test_reports`` now, not + ``sphinxcontrib.test_reports``: ``extensions = ["sphinx_test_reports"]``. The pytest + plugin and the parsers moved further, to ``ub_test_reports`` (above). Four old names keep working until **4.0**, each with a warning that names its replacement. 4.0 turns all four into errors that say the same thing. Under pytest's @@ -28,7 +82,8 @@ The import name moves loads the extension once. - **The pytest plugin**, ``-p sphinxcontrib.test_reports.pytest_plugin`` (and the same name in ``addopts``, ``PYTEST_PLUGINS`` or a ``conftest.py``'s ``pytest_plugins``), - loads the real plugin and raises a ``FutureWarning`` at start-up -- once per process, so + loads the real plugin, ``ub_test_reports.pytest_plugin``, and raises a ``FutureWarning`` + at start-up -- once per process, so once more for each pytest-xdist worker. A filter written against the old name -- ``ignore::sphinxcontrib.test_reports.pytest_plugin.TestReportsConfigWarning`` -- still matches, because resolving it imports the alias. Change it together with the ``-p`` @@ -38,7 +93,8 @@ The import name moves ``pytest_plugins``, stops pytest with "Plugin already registered under a different name"; keep one. - **The parsers**, ``sphinxcontrib.test_reports.junitparser`` and - ``sphinxcontrib.test_reports.jsonparser``, are the real modules under the old name, + ``sphinxcontrib.test_reports.jsonparser``, are the real modules -- + ``ub_test_reports.junitparser`` and ``ub_test_reports.jsonparser`` -- under the old name, with a ``FutureWarning`` (once per process) that points at the ``import`` statement naming them -- at ``importlib`` itself when they are loaded with ``importlib.import_module``. Their classes, and a ``mock.patch`` target through the old @@ -58,8 +114,8 @@ The import name moves ordinary ``ImportError`` (``ModuleNotFoundError`` for an ``import`` statement): ``projectconfig``, ``identity``, ``results``, the directives and the rest were never documented as an API. The module is the same under - the new name -- ``sphinxcontrib.test_reports.projectconfig`` is - ``sphinx_test_reports.projectconfig``. + its new name -- ``sphinxcontrib.test_reports.projectconfig`` is + ``ub_test_reports.projectconfig``, and the directives are under ``sphinx_test_reports``. The two ``user_properties`` names the pytest plugin reserves for its location override, ``sphinxcontrib.test_reports:file`` and ``sphinxcontrib.test_reports:line``, are wire @@ -89,9 +145,10 @@ New and Improved - **Release tags** are prefixed: ``sphinx-test-reports-v2.0.0`` rather than ``2.0.0``. Six of the ten historical bare names collided with existing Sphinx-Needs releases, so the prefix is load-bearing rather than tidy. - - **The distribution does not change.** It is still ``sphinx-test-reports``, and the - ``test-reports`` command is still the same command. The import name does change, in - this same release: see *The import name moves* above. + - **The distribution keeps its name.** It is still ``sphinx-test-reports``, and the + ``test-reports`` command is still the same command, now shipped by ub-test-reports, + which it depends on. The import names do change, in this same release: see the two + sections above. - 🔧 The shipped default ``tr_report_template`` ends with a ``literalinclude`` of itself, by a path relative to the including document. **That is still broken for your project** @@ -119,16 +176,15 @@ New and Improved two lines below a slice of the missing value, which raised first. It now raises ``TestReportFileNotSetError`` like every other configuration mistake. -- ⬆️ The ``sphinx`` extra now requires docutils 0.21 or newer, the floor the whole +- ⬆️ The extension now requires docutils 0.21 or newer, as a dependency, the floor the whole Sphinx-Needs workspace declares and type-checks against (previously whatever Sphinx - accepted, which is 0.20 for Sphinx 7.4 through 9.0). The extension checks the extra's - floors when it loads, so it now refuses docutils 0.20 with the install line; the - ``test-reports`` command and the pytest plugin still install no docutils at all. + accepted, which is 0.20 for Sphinx 7.4 through 9.0). The ``test-reports`` command and the + pytest plugin, which are ub-test-reports, install no docutils at all. - ♻️ The ``ubproject.toml`` reader is now `ub-project `__, - the shared reader every useblocks tool uses for the file, and a new runtime dependency - (standard library only, so the ``test-reports`` command and the pytest plugin still run - without Sphinx). Nothing changes in behaviour: the walk up to the repository root, the + the shared reader every useblocks tool uses for the file, and a runtime dependency of + ub-test-reports, where the ``[test_reports]`` model now lives (standard library only, so + the ``test-reports`` command and the pytest plugin still run without Sphinx). Nothing changes in behaviour: the walk up to the repository root, the anchoring of relative paths at the file's directory and every message are as before. One failure that used to escape as a traceback is now reported like the others: a file that is not UTF-8 is a configuration error naming the file. @@ -140,16 +196,17 @@ What the move costs, stated rather than left to the CI diff suite against Sphinx-Needs 6.0.1, 6.3.0, 7.0.0, 8.0.0 and 8.5.0; in the workspace the suite runs against the sibling in the tree, across Sphinx 7.4, 8.2 and 9.1 instead. With it, **the declared floor narrows from** ``sphinx-needs>=6.0.1`` **to** - ``sphinx-needs>=8.5.0,<9`` in the ``sphinx`` extra (and from ``>=6`` in ``docs``). That is + ``sphinx-needs>=8.5.0,<9`` (and from ``>=6`` in ``docs``). That is the workspace's tight-tracking policy for a dependency on a sibling, and it is enforced; it means this release supports a narrower range of Sphinx-Needs than 2.0.0 did. - **Five ruff rule families are no longer enforced here** -- ``FURB``, ``PERF``, ``PGH``, ``PIE`` and ``SLF`` -- because the workspace has one shared rule set and they are not in it. All five were at zero violations, so nothing changed in the code; what changed is that a new violation would no longer be caught. -- **The** ``plugin_floor`` **lane is gone for now**: the pytest plugin is no longer tested - against the oldest pytest of each Python. The ``toolchain_free`` lane survives, as a CI - job that installs the package with no documentation toolchain at all and asserts it. +- **Both retired nox lanes have CI jobs**, now that the plugin is ub-test-reports': the + ``toolchain_free`` lane is a job that installs ub-test-reports with no documentation + toolchain at all, asserts it, and runs that package's whole suite; the ``plugin_floor`` + lane runs the same suite on pytest 7.0.1 (Python 3.11) and 7.3.2 (Python 3.12). - **mypy is replaced by ty**, which checks the whole package -- the mypy configuration excluded fifteen modules. - **Beyond those five families, four rules this package enabled are now ignored** diff --git a/packages/sphinx-test-reports/docs/cli.rst b/packages/sphinx-test-reports/docs/cli.rst index 97c1b9ba7..d3b752208 100644 --- a/packages/sphinx-test-reports/docs/cli.rst +++ b/packages/sphinx-test-reports/docs/cli.rst @@ -4,7 +4,7 @@ Command line interface ====================== .. versionadded:: 2.0.0 -``Sphinx-Test-Reports`` ships a ``test-reports`` command that converts +``ub-test-reports`` ships a ``test-reports`` command that converts test-result XML into a ``needs.json`` **without running Sphinx**. Why this exists: parsing test results inside a documentation build couples the @@ -13,10 +13,15 @@ be cached by a build system, and the data is unavailable to anything else. The CLI splits the computation out; the documentation build only imports the result. The command imports no Sphinx code at all, so it can run as a build action in an -environment that has no documentation toolchain installed. Installed as -``pip install sphinx-test-reports`` -- without the ``sphinx`` extra the -:doc:`extension needs ` -- the package brings a single dependency, -``lxml``. +environment that has no documentation toolchain installed. Install it as +``pip install ub-test-reports``, whose dependencies are ``lxml`` and ``ub-project`` +(see :doc:`/install`); sphinx-test-reports depends on it, so the command is also there +wherever the extension is. + +.. versionchanged:: 3.0.0 + The command is shipped by ub-test-reports, not sphinx-test-reports. On 2.0.0 the bare + ``pip install sphinx-test-reports`` was the way to get it without Sphinx; that line now + brings the documentation toolchain. Converting a report ------------------- diff --git a/packages/sphinx-test-reports/docs/conf.py b/packages/sphinx-test-reports/docs/conf.py index c16046eac..4ed7b0299 100644 --- a/packages/sphinx-test-reports/docs/conf.py +++ b/packages/sphinx-test-reports/docs/conf.py @@ -8,6 +8,7 @@ # -- Path setup -------------------------------------------------------------- import datetime +import importlib.metadata import os import shutil @@ -35,10 +36,13 @@ copyright = f"team useblocks, 2017-{now.year}" author = "team useblocks" +# The full version, including alpha/beta/rc tags: the EXTENSION's, read from the installed +# distribution so it cannot drift from the manifest. ub-test-reports, which this site +# documents too, has its own version line; the index page says so +release = importlib.metadata.version("sphinx-test-reports") # The short X.Y version -version = "2.0" -# The full version, including alpha/beta/rc tags -release = "2.0.0" +_release = Version(release) +version = f"{_release.major}.{_release.minor}" needs_id_regex = ".*" needs_css = "dark.css" diff --git a/packages/sphinx-test-reports/docs/configuration.rst b/packages/sphinx-test-reports/docs/configuration.rst index da24e88f7..45c32a31a 100644 --- a/packages/sphinx-test-reports/docs/configuration.rst +++ b/packages/sphinx-test-reports/docs/configuration.rst @@ -421,6 +421,11 @@ shared file other useblocks tooling (sphinx-needs, sphinx-codelinks, sphinx-mounts, ubCode) reads. It describes the project once, so every tool acting on it works from the same settings instead of each restating them. +The section's model -- its keys, their types, how they are normalised and which are +rejected -- belongs to ub-test-reports (``ub_test_reports.projectconfig``), so the +:ref:`build needs command ` and the extension read it identically; the extension +applies it to the ``tr_*`` values below when a build starts. + .. code-block:: toml [test_reports] diff --git a/packages/sphinx-test-reports/docs/index.rst b/packages/sphinx-test-reports/docs/index.rst index 4341bf59e..8a85cdaf1 100644 --- a/packages/sphinx-test-reports/docs/index.rst +++ b/packages/sphinx-test-reports/docs/index.rst @@ -69,18 +69,35 @@ As example, here is a shorten list of tests results from the Sphinx-pytest examp Content ------- +Test reports come in two distributions: **sphinx-test-reports**, the Sphinx extension, and +**ub-test-reports**, the converter, the pytest plugin and the parsers, which run without +Sphinx and which the extension depends on. The version this site shows is the +extension's; ub-test-reports has its own, and its own changelog below. + .. toctree:: :maxdepth: 2 + :caption: The Sphinx extension install directives/index configuration - cli - pytest - parsers filter functions examples/index + +.. toctree:: + :maxdepth: 2 + :caption: Without Sphinx (ub-test-reports) + + cli + pytest + parsers + ub-test-reports-changelog + +.. toctree:: + :maxdepth: 2 + :caption: Support + support changelog diff --git a/packages/sphinx-test-reports/docs/install.rst b/packages/sphinx-test-reports/docs/install.rst index 68518772d..beaee0ae1 100644 --- a/packages/sphinx-test-reports/docs/install.rst +++ b/packages/sphinx-test-reports/docs/install.rst @@ -3,26 +3,28 @@ Installation ============ -The package has three consumers, and each installs a different part of it. +Test reports come in two distributions, and each consumer installs the one it needs. -A documentation project needs the Sphinx extension, and with it the -documentation toolchain -- Sphinx and -`Sphinx-Needs `_ -- which is -the ``sphinx`` extra of the package:: +A documentation project needs the Sphinx extension, **sphinx-test-reports**. It brings the +documentation toolchain -- Sphinx, docutils and +`Sphinx-Needs `_ -- and ub-test-reports +with it:: - pip install "sphinx-test-reports[sphinx]" + pip install sphinx-test-reports + +A build action that only runs the :ref:`test-reports command `, which turns test +results into a ``needs.json`` without a Sphinx build, installs **ub-test-reports**, which +has no Sphinx in it (its dependencies are ``lxml`` and ``ub-project``):: + + pip install ub-test-reports A test runner that should write the XML shape the extension reads installs the -:ref:`pytest plugin ` as the ``pytest`` extra, which adds pytest +:ref:`pytest plugin `, ub-test-reports' ``pytest`` extra, which adds pytest and nothing of the documentation toolchain:: - pip install "sphinx-test-reports[pytest]" + pip install "ub-test-reports[pytest]" -A build action that only runs the :ref:`test-reports command `, which -turns test results into a ``needs.json`` without a Sphinx build, installs the -bare package, whose single dependency is ``lxml``:: - - pip install sphinx-test-reports +ub-test-reports has its own version number and :doc:`changelog `. .. versionchanged:: 2.0.0 ``pip install sphinx-test-reports`` -- without an extra -- no longer @@ -30,12 +32,17 @@ bare package, whose single dependency is ``lxml``:: ``sphinx`` extra to its install line; a test runner or a build action that has no documentation toolchain no longer gets one. -The ``sphinx`` extra also states the supported versions: Sphinx 7.4 and -Sphinx-Needs 6.0.1 or later. An extra is opt-in, so a project that keeps -installing the bare package into an environment holding an older toolchain -would never be told by ``pip``; the extension therefore checks the installed -versions when Sphinx loads it and stops the build with a message naming the -install line above. +.. versionchanged:: 3.0.0 + The converter, the pytest plugin and the parsers moved to their own distribution, + ub-test-reports, and ``pip install sphinx-test-reports`` installs Sphinx, docutils and + Sphinx-Needs again: they are dependencies of the extension, not an extra. A test runner + or a build action that installed sphinx-test-reports for the command or the plugin + installs ub-test-reports instead. The ``sphinx`` extra is accepted and ignored, and the + ``pytest`` extra installs ``ub-test-reports[pytest]``, until 4.0. + +The extension supports Sphinx 7.4 or later, docutils 0.21 or later and Sphinx-Needs 8.5 +(``>=8.5.0,<9``); ``pip`` resolves those as it installs it, and the extension itself +refuses an older Sphinx when it loads. After that the extension must be added to the ``conf.py`` file:: diff --git a/packages/sphinx-test-reports/docs/pytest.rst b/packages/sphinx-test-reports/docs/pytest.rst index 3d7c24369..203418242 100644 --- a/packages/sphinx-test-reports/docs/pytest.rst +++ b/packages/sphinx-test-reports/docs/pytest.rst @@ -11,7 +11,7 @@ Under pytest's default ``junit_family = xunit2`` no ```` carries the of ``tr_deterministic_case_ids``). Under ``xunit1`` pytest writes them, but counts the line from 0, keeps Bazel's runfiles prefix, and has no way to point a case at the file that drove it. And nothing records the requirements a test -verifies. ``Sphinx-Test-Reports`` ships a small pytest plugin that takes care +verifies. ``ub-test-reports`` ships a small pytest plugin that takes care of both. The plugin is generic: which properties exist, what they are called in the XML @@ -21,20 +21,22 @@ is the worked example below. Enabling it ----------- -The plugin is the ``pytest`` extra of the package: ``pip install -"sphinx-test-reports[pytest]"`` installs it without the documentation toolchain -(see :doc:`/install`). Enable it in the pytest configuration: +The plugin is part of ub-test-reports, the Sphinx-free distribution, and its ``pytest`` +extra brings pytest: ``pip install "ub-test-reports[pytest]"`` installs it without the +documentation toolchain (see :doc:`/install`). Enable it in the pytest configuration: .. code-block:: ini # pytest.ini / pyproject.toml [tool.pytest.ini_options] - addopts = -p sphinx_test_reports.pytest_plugin + addopts = -p ub_test_reports.pytest_plugin junit_family = xunit1 .. versionchanged:: 3.0.0 - The module is ``sphinx_test_reports.pytest_plugin``. The pre-3.0 name, - ``sphinxcontrib.test_reports.pytest_plugin``, still loads the same plugin - until 4.0, with a ``FutureWarning`` at start-up asking for the new one. That + The module is ``ub_test_reports.pytest_plugin``, in the ub-test-reports distribution; + ``pip install "sphinx-test-reports[pytest]"`` still installs it, until 4.0. The pre-3.0 + name, ``sphinxcontrib.test_reports.pytest_plugin``, still loads the same plugin until + 4.0 wherever sphinx-test-reports is installed, with a ``FutureWarning`` at start-up + asking for the new one. That warning comes before pytest installs its own filters, so only Python's own options silence it: ``PYTHONWARNINGS=ignore::FutureWarning``, which pytest-xdist's workers inherit, or @@ -62,7 +64,7 @@ Nothing else changes for tests that do not use the decorator below. The start-up notice is a ``TestReportsConfigWarning``. A project that turns warnings into errors (``filterwarnings = error``, ``-W error``) gets it as a clean usage error instead; -``ignore::sphinx_test_reports.pytest_plugin.TestReportsConfigWarning`` +``ignore::ub_test_reports.pytest_plugin.TestReportsConfigWarning`` silences it. Declaring the properties @@ -108,7 +110,7 @@ plugin this one was ported from: # pytest.ini [pytest] - addopts = -p sphinx_test_reports.pytest_plugin + addopts = -p ub_test_reports.pytest_plugin junit_family = xunit1 test_reports_properties = partially_verifies = PartiallyVerifies, list @@ -120,7 +122,7 @@ plugin this one was ported from: # pyproject.toml [tool.pytest.ini_options] - addopts = "-p sphinx_test_reports.pytest_plugin" + addopts = "-p ub_test_reports.pytest_plugin" junit_family = "xunit1" test_reports_properties = [ "partially_verifies = PartiallyVerifies, list", @@ -160,7 +162,7 @@ With the S-CORE model declared: .. code-block:: python - from sphinx_test_reports.pytest_plugin import add_test_properties + from ub_test_reports.pytest_plugin import add_test_properties @add_test_properties( partially_verifies=["REQ_1", "REQ_2"], @@ -234,7 +236,7 @@ of at the test function: .. code-block:: python - from sphinx_test_reports.pytest_plugin import apply_test_metadata + from ub_test_reports.pytest_plugin import apply_test_metadata @pytest.mark.parametrize("spec", SPECS) def test_spec(spec, record_property): diff --git a/packages/sphinx-test-reports/docs/ub-test-reports-changelog.rst b/packages/sphinx-test-reports/docs/ub-test-reports-changelog.rst new file mode 100644 index 000000000..6b3babea9 --- /dev/null +++ b/packages/sphinx-test-reports/docs/ub-test-reports-changelog.rst @@ -0,0 +1,13 @@ +:hide-navigation: + +ub-test-reports changelog +========================= + +ub-test-reports, the converter and the pytest plugin, has its own version line; this is +its changelog, ``packages/ub-test-reports/docs/changelog.rst``. + +.. The included file's own label and `Changelog` title are skipped: `poe bump` needs the + title there, and this page has its heading above. + +.. include:: ../../ub-test-reports/docs/changelog.rst + :start-line: 4 diff --git a/packages/ub-test-reports/AGENTS.md b/packages/ub-test-reports/AGENTS.md index c77c71141..4c32bf363 100644 --- a/packages/ub-test-reports/AGENTS.md +++ b/packages/ub-test-reports/AGENTS.md @@ -27,12 +27,17 @@ uv run poe build-ub-test-reports # sdist + wheel into dist/ub-test-repo - **Nothing here may import Sphinx, sphinx-needs or docutils** — not at module level and not in a function body, because the converter runs as a build action and the plugin - inside a test run, and neither has a documentation toolchain. A default `.venv` cannot - catch a violation: every environment the workspace root produces has Sphinx in it. CI's - `toolchain-free` job is the fence — it installs the built wheel where the toolchain is - absent and runs the whole suite there. + inside a test run, and neither has a documentation toolchain. Every environment the + workspace root produces has Sphinx in it, so the default `.venv` sees only a module-level + import in the converter's import chain (a subprocess test lists `sys.modules`); a + function-body import passes there. CI's `toolchain-free` job is the fence — it installs + the built wheel where the toolchain is absent and runs the whole suite there. - **`ub-project` is its `ubproject.toml` reader.** Finding, loading and anchoring the file - come from there; what stays here is the `[test_reports]` policy and `TomlConfigError`. + come from there (`packages/ub-project/design/reading-contract.md` is the specification); + what stays here is the `[test_reports]` policy -- keys, types, normalisation, unknown keys + warned rather than fatal -- and **`TomlConfigError`, the only exception either consumer + catches**: `load_project_config` re-raises ub-project's `ProjectConfigError` as it, with + the same message, and it must never be made a subclass of it. - **The `[test_reports]` model is a parity surface**: ubCode reads the same table and is held to the same behaviour. A behaviour change in it — keys, types, normalisation, defaults — says so in the changelog, so ubCode can follow. @@ -48,5 +53,12 @@ uv run poe build-ub-test-reports # sdist + wheel into dist/ub-test-repo suite reads nothing from another member's tree and runs from the installed wheel; some of them also exist under sphinx-test-reports' `tests/doc_test/utils/`, where its test projects and docs read them. Neither copy ships in an sdist. +- **One plugin test drives an in-process pytest session, and the default `.venv` breaks + it.** `tests/test_pytest_plugin.py`'s `NESTED` source starts `pytest.main()` inside a + `pytester` session, where every installed plugin loads; pytest-playwright (the root `js` + group, in the default `dev` group) refuses the nested soft-assertion scope. So it passes + `-p no:playwright`, and that line has to stay ONE line: a sibling test builds its own + source from it by replacing the literal `str(inner)]) == 0`. Run the suite in the default + `.venv` AND in a cell — green in one proves nothing about the other. - **Tests build paths with `Path`** and read and write text with an explicit `encoding`, so that the suite holds on Windows too. From 88344b204d6c3a380a8cd7e426ff9889ef39e36e Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Thu, 1 Oct 2026 02:42:14 +0200 Subject: [PATCH 6/9] =?UTF-8?q?=F0=9F=91=8C=20Read=20the=20Docs:=20build?= =?UTF-8?q?=20sphinx-test-reports'=20site=20when=20ub-test-reports=20chang?= =?UTF-8?q?es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The site documents ub-test-reports, includes its changelog and installs it from the checkout, so a pull request that touches only packages/ub-test-reports/ changes what the site builds. The skip filter in post_checkout now looks at both packages. --- packages/sphinx-test-reports/.readthedocs.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/sphinx-test-reports/.readthedocs.yaml b/packages/sphinx-test-reports/.readthedocs.yaml index e7732e306..19507df1e 100644 --- a/packages/sphinx-test-reports/.readthedocs.yaml +++ b/packages/sphinx-test-reports/.readthedocs.yaml @@ -27,8 +27,8 @@ build: jobs: post_checkout: # Every pull request against this repository builds every Read the Docs project it - # has. A pull request that leaves `packages/sphinx-test-reports/` untouched has - # nothing for this one to build, so cancel it. Exit code 183 is Read the Docs' + # has. A pull request that leaves `packages/sphinx-test-reports/` and + # `packages/ub-test-reports/` untouched has nothing for this one to build, so cancel it. Exit code 183 is Read the Docs' # documented "skip" code # (https://docs.readthedocs.com/platform/stable/guides/build/skip-build.html): the # build is recorded as `cancelled`, and the commit status sent to GitHub is `success` @@ -43,8 +43,9 @@ build: # pull request did not touch it. That is the safe direction; it never skips a build # that was needed. # - # `packages/sphinx-test-reports/` is ALMOST the complete set of inputs, and the one - # exception is deliberate: `docs/conf.py` also reads `vendor/plantuml/pin.toml` and + # The two packages are ALMOST the complete set of inputs -- ub-test-reports is one because + # this site documents it, includes its changelog, and installs it from the checkout + # above -- and the one exception is deliberate: `docs/conf.py` also reads `vendor/plantuml/pin.toml` and # the jar beside it. A bump there changes only which renderer version draws the # diagrams, never whether the build succeeds, so it is not worth widening the filter # and rebuilding this project for every workspace-level commit. @@ -56,8 +57,8 @@ build: echo "origin/master is not available in this clone; building to be safe" exit 0 fi - if git diff --quiet origin/master -- packages/sphinx-test-reports/; then - echo "nothing under packages/sphinx-test-reports/ changed; skipping this build (exit 183)" + if git diff --quiet origin/master -- packages/sphinx-test-reports/ packages/ub-test-reports/; then + echo "nothing under packages/sphinx-test-reports/ or packages/ub-test-reports/ changed; skipping this build (exit 183)" exit 183 fi From 518c9f578bcc269bc0b47f5974f4e9600ceae72e Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Thu, 1 Oct 2026 03:19:05 +0200 Subject: [PATCH 7/9] =?UTF-8?q?=F0=9F=A7=AA=20Review=20fixes:=20a=20static?= =?UTF-8?q?=20no-toolchain=20test,=20the=20printed=20names=20pinned,=20xdi?= =?UTF-8?q?st=20fenced?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ub-test-reports' tests/test_imports.py walks every module's source and refuses an import statement naming sphinx, sphinx_needs, docutils, sphinx_test_reports or sphinxcontrib at any depth -- function bodies, `try` arms and `TYPE_CHECKING` included -- plus string-spelled `importlib.import_module`/`__import__`. It runs in every environment and does not depend on a test reaching the line. The plugin's warning prefix and hook registration name, which the changelog documents as changed from 2.0.0, are pinned by one pytester run. The alias test now imports the old pytest_plugin name too, and is renamed for what it proves: the alias chain does not load Sphinx. CI: toolchain-free and plugin-floor fail when pytest-xdist is missing, instead of skipping the plugin's three xdist tests green. --- .github/workflows/ci.yaml | 19 ++++-- .../sphinx-test-reports/tests/test_aliases.py | 6 +- .../ub-test-reports/tests/test_imports.py | 65 +++++++++++++++++++ .../tests/test_pytest_plugin.py | 22 +++++++ 4 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 packages/ub-test-reports/tests/test_imports.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 875b35ae9..1bc5b8029 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -604,14 +604,19 @@ jobs: uv venv --python 3.11 /tmp/toolchain-free wheel=$(ls /tmp/core-dist/*.whl) uv pip install --python /tmp/toolchain-free --no-sources "${wheel}[pytest]" pytest-xdist packages/ub-project - - name: Assert the toolchain really is absent + - name: Assert the toolchain really is absent, and pytest-xdist present # the fence's fence. Without this the job would still pass with Sphinx installed, - # and would be testing nothing it claims to test. + # and would be testing nothing it claims to test. pytest-xdist the other way round: + # without it the plugin's three xdist tests skip on `importorskip` and the job + # stays green run: | /tmp/toolchain-free/bin/python -c " import importlib.util, sys present = [m for m in ('sphinx', 'sphinx_needs', 'docutils') if importlib.util.find_spec(m)] - sys.exit(f'toolchain installed: {present}' if present else 0) + if present: + sys.exit(f'toolchain installed: {present}') + if importlib.util.find_spec('xdist') is None: + sys.exit(\"pytest-xdist is not installed: the plugin's three xdist tests would skip\") " - name: Run the ub-test-reports suite # WHOLE, with no marker and no file list: every test of the core must pass without the @@ -666,15 +671,19 @@ jobs: wheel=$(ls /tmp/core-dist/*.whl) uv pip install --python /tmp/plugin-floor --no-sources "${wheel}[pytest]" \ "pytest==${{ matrix.pytest }}" pytest-xdist packages/ub-project - - name: Assert the pin, and that the toolchain is absent + - name: Assert the pin, pytest-xdist present, and the toolchain absent # the pin's positive control: a resolver that moved pytest would make this job test - # the newest pytest again while still reporting the floor + # the newest pytest again while still reporting the floor. pytest-xdist: a resolver + # that could no longer place one next to the pinned pytest would make the plugin's + # three xdist tests skip, green run: | /tmp/plugin-floor/bin/python -c " import importlib.util, sys import pytest if pytest.__version__ != '${{ matrix.pytest }}': sys.exit(f'pytest {pytest.__version__} is installed, not ${{ matrix.pytest }}') + if importlib.util.find_spec('xdist') is None: + sys.exit(\"pytest-xdist is not installed: the plugin's three xdist tests would skip\") present = [m for m in ('sphinx', 'sphinx_needs', 'docutils') if importlib.util.find_spec(m)] sys.exit(f'toolchain installed: {present}' if present else 0) " diff --git a/packages/sphinx-test-reports/tests/test_aliases.py b/packages/sphinx-test-reports/tests/test_aliases.py index d0b96d1df..68c99412d 100644 --- a/packages/sphinx-test-reports/tests/test_aliases.py +++ b/packages/sphinx-test-reports/tests/test_aliases.py @@ -200,7 +200,10 @@ def test_the_warning_is_visible_under_default_filters(tmp_path, module: str) -> assert f"user_code.py:1: FutureWarning: {OLD}.{module} has moved" in result.stderr -def test_the_aliases_import_without_sphinx(tmp_path) -> None: +def test_the_aliases_import_without_loading_sphinx(tmp_path) -> None: + # This suite always has Sphinx installed, so what it proves is that the alias chain + # does not LOAD it: `-p sphinxcontrib.test_reports.pytest_plugin` must not pull the + # toolchain into every pytest run, which is what the lazy package root is for. result = _run_user_code( tmp_path, f"""\ @@ -208,6 +211,7 @@ def test_the_aliases_import_without_sphinx(tmp_path) -> None: import {OLD} import {OLD}.junitparser import {OLD}.jsonparser + import {OLD}.pytest_plugin loaded = sorted(m for m in ("sphinx", "sphinx_needs", "docutils") if m in sys.modules) assert not loaded, loaded """, diff --git a/packages/ub-test-reports/tests/test_imports.py b/packages/ub-test-reports/tests/test_imports.py new file mode 100644 index 000000000..c60c01fef --- /dev/null +++ b/packages/ub-test-reports/tests/test_imports.py @@ -0,0 +1,65 @@ +"""No module of ub-test-reports names the documentation toolchain, at any depth. + +A static walk, not an import: it reads every module's source, so it refuses an import +statement -- at module level, in a function body, in a ``try`` arm or under +``TYPE_CHECKING`` -- whether or not any test reaches that line, and in every environment, +the default one (which has Sphinx installed) included. String-spelled +``importlib.import_module("...")`` and ``__import__("...")`` calls are read too. What a +static walk cannot see -- a dependency that drags Sphinx in, an import spelled some other +way -- is what CI's ``toolchain-free`` job is for. +""" + +import ast +from pathlib import Path + +import pytest + +import ub_test_reports + +#: Top-level names this package must never import: the documentation toolchain, and the +#: extension, which depends on this package and not the other way round. +FORBIDDEN = { + "sphinx", + "sphinx_needs", + "docutils", + "sphinx_test_reports", + "sphinxcontrib", +} + +ROOT = Path(ub_test_reports.__file__).parent +MODULES = sorted(ROOT.rglob("*.py")) + + +def _imported_names(node: ast.AST) -> list[str]: + """The absolute module names *node* imports, if it is an import of any spelling.""" + if isinstance(node, ast.Import): + return [alias.name for alias in node.names] + if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + return [node.module] + if ( + isinstance(node, ast.Call) + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + and ast.unparse(node.func).endswith(("import_module", "__import__")) + ): + return [node.args[0].value] + return [] + + +def test_the_walk_sees_the_whole_package() -> None: + # the fence's fence: a walk of the wrong directory would pass every module vacuously + names = {path.relative_to(ROOT).as_posix() for path in MODULES} + assert {"__init__.py", "cli.py", "jsonparser.py", "pytest_plugin.py"} <= names + + +@pytest.mark.parametrize("path", MODULES, ids=lambda p: p.relative_to(ROOT).as_posix()) +def test_no_module_imports_the_toolchain(path: Path) -> None: + tree = ast.parse(path.read_text(encoding="utf-8")) + leaks = [ + f"{path.name}:{node.lineno}: {name}" + for node in ast.walk(tree) + for name in _imported_names(node) + if name.split(".")[0] in FORBIDDEN + ] + assert leaks == [] diff --git a/packages/ub-test-reports/tests/test_pytest_plugin.py b/packages/ub-test-reports/tests/test_pytest_plugin.py index 5f6831730..c6d32ed74 100644 --- a/packages/ub-test-reports/tests/test_pytest_plugin.py +++ b/packages/ub-test-reports/tests/test_pytest_plugin.py @@ -5,6 +5,7 @@ ``test_reports_properties`` ini option; S-CORE's is the profile most tests use. """ +import re import subprocess import sys import xml.etree.ElementTree as ET @@ -337,6 +338,27 @@ def test_xunit2_is_warned_about_at_configure_time(self, pytester): result.stdout.fnmatch_lines(["*junit_family is 'xunit2'*xunit1*"]) assert _cases(root)["test_plain"].get("file") is None + def test_the_names_it_prints_are_ub_test_reports(self, pytester): + # Both are documented in the changelog as changed from 2.0.0's + # `sphinxcontrib.test_reports...`: the prefix of the warnings it issues (what a + # user's warning filter matches) and the name its hook object is registered under. + source = ( + "def test_registered(request):\n" + " assert request.config.pluginmanager.has_plugin('ub_test_reports.xml_shape')\n" + ) + result, _ = _run(pytester, source, family="xunit2") + result.assert_outcomes(passed=1) + messages = [ + line.split("TestReportsConfigWarning: ", 1)[1] + for line in result.stdout.lines + if "TestReportsConfigWarning: " in line + ] + assert messages + assert all( + re.match(r"^ub_test_reports\.pytest_plugin: ", message) + for message in messages + ), messages + class TestPropertyModel: """The model is pytest configuration; the plugin ships no names of its own.""" From 7e3ffb5a32ba0a1c8386974851239d03efed2ac5 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Thu, 1 Oct 2026 03:23:04 +0200 Subject: [PATCH 8/9] =?UTF-8?q?=F0=9F=93=9A=20Review=20fixes:=20the=20chan?= =?UTF-8?q?gelog's=20downgrade=20and=20installer=20claims,=20a=20changelog?= =?UTF-8?q?=20link,=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The changelog no longer says a below-floor Sphinx-Needs fails with a traceback: measured, it runs untested and nothing fails, which is what it now says; `pip check` still names the conflict. A new entry says the `test-reports` command belongs to ub-test-reports, so `pipx install` and `uv tool install sphinx-test-reports` find no command and should name ub-test-reports. The core's entry says its printed names changed. ub-test-reports' changelog is linked from the site rather than included, so its release labels can never collide with the extension's. AGENTS: the core's no-toolchain rule names the static test and what the toolchain-free job adds; the extension's Releasing paragraph says the site must build this repository before the core's first tag. Comments in ci.yaml, release.yaml and two tests corrected. --- .github/workflows/ci.yaml | 27 ++++++++++--------- .github/workflows/release.yaml | 7 ++--- .../sphinx-test-reports/.readthedocs.yaml | 4 +-- packages/sphinx-test-reports/AGENTS.md | 5 +++- .../sphinx-test-reports/docs/changelog.rst | 17 +++++++++--- packages/sphinx-test-reports/docs/index.rst | 4 +-- packages/sphinx-test-reports/docs/install.rst | 3 ++- .../docs/ub-test-reports-changelog.rst | 13 --------- .../tests/test_cli_convert.py | 12 +-------- packages/ub-test-reports/AGENTS.md | 12 +++++---- packages/ub-test-reports/docs/changelog.rst | 5 ++-- tools/tests/test_check_workspace.py | 4 +-- 12 files changed, 55 insertions(+), 58 deletions(-) delete mode 100644 packages/sphinx-test-reports/docs/ub-test-reports-changelog.rst diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1bc5b8029..8b5ed48e9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -466,9 +466,12 @@ jobs: # ub-test-reports -- the converter (`test-reports`), the pytest plugin, the parsers and the # `[test_reports]` model -- is a DISTRIBUTION that must run where the documentation # toolchain is NOT installed: a build action and a test run have none. Nothing else keeps - # that true. This job installs the core's built wheel with the dependencies it declares, - # asserts that sphinx, sphinx_needs and docutils are all absent, and runs its WHOLE suite - # there: a module-level or a function-body import of the toolchain fails it. + # that true at run time. This job installs the core's built wheel with the dependencies it + # declares, asserts that sphinx, sphinx_needs and docutils are all absent, and runs its + # WHOLE suite there. The suite's own `tests/test_imports.py` refuses any import STATEMENT + # naming the toolchain, in every environment; what this job adds is what a static walk + # cannot see -- a dependency that drags Sphinx in, an import spelled dynamically -- and + # the plugin's pytester subprocess runs, all without the toolchain. # # The environment is on Python 3.11, the workspace floor: both packages it runs claim # `>=3.11`, and no other cell runs either of them there -- the matrix cells vary the @@ -625,12 +628,12 @@ jobs: # `--import-mode=importlib` -- so this tests the artefact. `-rs` prints any skip. run: /tmp/toolchain-free/bin/python -m pytest -q -rs packages/ub-test-reports/tests - name: Run the ub-project suite - # A step of its own, and it has to be: both suites have a `tests/__init__.py`, so one - # run over both resolves this suite beside ub-test-reports' `tests/conftest.py` under - # the one module name `tests.conftest` and stops with "Plugin already registered under - # a different name" (measured). A step per suite also names the red one. The installed - # wheel is what is imported -- `src/` is never on `sys.path` -- so this also tests the - # artefact + # A step of its own so that a red check names the suite. (One run over both would + # collect today -- measured -- but only because this suite has no `conftest.py`: add + # one and it resolves to `tests.conftest` beside ub-test-reports', and pytest stops + # with "Plugin already registered under a different name", measured too.) The + # installed wheel is what is imported -- `src/` is never on `sys.path` -- so this also + # tests the artefact run: /tmp/toolchain-free/bin/python -m pytest -q packages/ub-project/tests plugin-floor: @@ -644,9 +647,9 @@ jobs: # there at all. Same environment shape as `toolchain-free` -- the built wheel, no # toolchain -- with pytest pinned. # - # From the PACKAGE directory, as the release's compat cell runs a suite: run from the - # repository root, pytest 7.3.2 loads the conftest of every root `testpaths` entry -- - # sphinx-needs' among them -- and dies on its imports (measured). `pytest-xdist` is not + # From the package directory -- pytest still finds the root's `pyproject.toml` + # configuration there -- because from the root pytest 7.3.2 loads the conftest of every + # `testpaths` entry (sphinx-needs') and dies on its imports (measured). `pytest-xdist` is not # pinned: the newest (3.8.0, measured) declares `pytest>=7.0` and passes on both pins, so # the plugin's three xdist tests run rather than skip. strategy: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2efe6b6dd..19488198c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,11 +4,12 @@ name: Release # `-v`, where is a directory name under `packages/`. # # The filename is load-bearing and must not change: every publishable member's PyPI project -# (sphinx-needs, sphinx-mounts, sphinx-codelinks, sphinx-test-reports, ub-project) names -# `release.yaml` in its trusted-publisher entry. +# (sphinx-needs, sphinx-mounts, sphinx-codelinks, sphinx-test-reports, ub-project, +# ub-test-reports) names `release.yaml` in its trusted-publisher entry -- ub-test-reports' is +# created, as a pending publisher, before its first tag. # # The leading `*` in the tag filter is deliberately loose -- GitHub's tag filters are globs, -# not regexes, and cannot express "one of these four names" -- and the `plan` job is what +# not regexes, and cannot express "one of these names" -- and the `plan` job is what # refuses everything else. That order matters: GitHub *creates* an environment that a # workflow names and that does not exist, with no protection rules at all, so # `environment: pypi-${{ needs.plan.outputs.dist }}` is a fence only for the names that diff --git a/packages/sphinx-test-reports/.readthedocs.yaml b/packages/sphinx-test-reports/.readthedocs.yaml index 19507df1e..2ce96e3b8 100644 --- a/packages/sphinx-test-reports/.readthedocs.yaml +++ b/packages/sphinx-test-reports/.readthedocs.yaml @@ -44,8 +44,8 @@ build: # that was needed. # # The two packages are ALMOST the complete set of inputs -- ub-test-reports is one because - # this site documents it, includes its changelog, and installs it from the checkout - # above -- and the one exception is deliberate: `docs/conf.py` also reads `vendor/plantuml/pin.toml` and + # this site documents it and installs it from the checkout below -- and the one + # exception is deliberate: `docs/conf.py` also reads `vendor/plantuml/pin.toml` and # the jar beside it. A bump there changes only which renderer version draws the # diagrams, never whether the build succeeds, so it is not worth widening the filter # and rebuilding this project for every workspace-level commit. diff --git a/packages/sphinx-test-reports/AGENTS.md b/packages/sphinx-test-reports/AGENTS.md index a1a62c7a5..b8f1a7884 100644 --- a/packages/sphinx-test-reports/AGENTS.md +++ b/packages/sphinx-test-reports/AGENTS.md @@ -145,7 +145,10 @@ longer has. **ub-test-reports releases first.** This package's floor on the core is tight-tracked, so the core's release pull request (`poe bump ub-test-reports …`) rewrites it, and every release gate here resolves the core from PyPI: `poe import-check-reports`, the release -plan and the compat cell are red until the core version this tree names is published. +plan and the compat cell are red until the core version this tree names is published. The +core's documentation lives on this package's site, so that site's Read the Docs project +must build THIS repository (`packages/sphinx-test-reports/.readthedocs.yaml`) before the +core's first tag — until then its PyPI page links a site with no page about it. ## What the move into this workspace cost, deliberately diff --git a/packages/sphinx-test-reports/docs/changelog.rst b/packages/sphinx-test-reports/docs/changelog.rst index 8822d3790..e5d9d1000 100644 --- a/packages/sphinx-test-reports/docs/changelog.rst +++ b/packages/sphinx-test-reports/docs/changelog.rst @@ -28,6 +28,15 @@ The Sphinx-free half is its own distribution, ub-test-reports wherever that environment pins another Sphinx. Install ``ub-test-reports`` there instead: it is the same command and the same plugin, without the toolchain. +- ‼️ **The** ``test-reports`` **command belongs to ub-test-reports now**, so a tool installer + that takes the command from the package you name finds none in sphinx-test-reports: + ``pipx install sphinx-test-reports`` fails with "No apps associated with package + sphinx-test-reports", and ``uv tool install sphinx-test-reports`` with "No executables are + provided by package sphinx-test-reports". Name ``ub-test-reports`` there (``pipx install + ub-test-reports``, ``uv tool install ub-test-reports``), and in anything else that looks + the script up in the installing package's own metadata. ``pip install + sphinx-test-reports`` still puts ``test-reports`` on the path, through the dependency. + - The ``sphinx`` extra is accepted and ignored until 4.0 -- ``pip install "sphinx-test-reports[sphinx]"`` installs exactly what the bare line does -- and the ``pytest`` extra passes through to ``ub-test-reports[pytest]`` until 4.0, so 2.0.0's @@ -49,10 +58,10 @@ The Sphinx-free half is its own distribution, ub-test-reports - 🔧 The load-time toolchain check is gone. It existed because the toolchain was an opt-in extra pip never saw; as hard dependencies, pip resolves the floors itself. An environment - whose Sphinx-Needs is downgraded below the floor AFTER installing now fails with a - traceback from inside a directive rather than a one-line error naming the install line; - ``pip check`` (or ``uv pip check``) names the conflict. The extension itself still refuses - a Sphinx older than 7.4 when it loads, with Sphinx's own version error. With the check went + whose Sphinx-Needs is downgraded below the floor AFTER installing is no longer refused: + the extension does not check, so a below-floor Sphinx-Needs runs untested (it may work, it + may fail anywhere); ``pip check`` (or ``uv pip check``) names the conflict. The extension + itself still refuses a Sphinx older than 7.4 when it loads. With the check went ``compat-requirements.txt``, which the release's compatibility cell needed only while the toolchain was optional. diff --git a/packages/sphinx-test-reports/docs/index.rst b/packages/sphinx-test-reports/docs/index.rst index 8a85cdaf1..1d6011b78 100644 --- a/packages/sphinx-test-reports/docs/index.rst +++ b/packages/sphinx-test-reports/docs/index.rst @@ -72,7 +72,8 @@ Content Test reports come in two distributions: **sphinx-test-reports**, the Sphinx extension, and **ub-test-reports**, the converter, the pytest plugin and the parsers, which run without Sphinx and which the extension depends on. The version this site shows is the -extension's; ub-test-reports has its own, and its own changelog below. +extension's; ub-test-reports has its own version and its own +`changelog `__. .. toctree:: :maxdepth: 2 @@ -92,7 +93,6 @@ extension's; ub-test-reports has its own, and its own changelog below. cli pytest parsers - ub-test-reports-changelog .. toctree:: :maxdepth: 2 diff --git a/packages/sphinx-test-reports/docs/install.rst b/packages/sphinx-test-reports/docs/install.rst index beaee0ae1..cd0af6c12 100644 --- a/packages/sphinx-test-reports/docs/install.rst +++ b/packages/sphinx-test-reports/docs/install.rst @@ -24,7 +24,8 @@ and nothing of the documentation toolchain:: pip install "ub-test-reports[pytest]" -ub-test-reports has its own version number and :doc:`changelog `. +ub-test-reports has its own version number and +`changelog `__. .. versionchanged:: 2.0.0 ``pip install sphinx-test-reports`` -- without an extra -- no longer diff --git a/packages/sphinx-test-reports/docs/ub-test-reports-changelog.rst b/packages/sphinx-test-reports/docs/ub-test-reports-changelog.rst deleted file mode 100644 index 6b3babea9..000000000 --- a/packages/sphinx-test-reports/docs/ub-test-reports-changelog.rst +++ /dev/null @@ -1,13 +0,0 @@ -:hide-navigation: - -ub-test-reports changelog -========================= - -ub-test-reports, the converter and the pytest plugin, has its own version line; this is -its changelog, ``packages/ub-test-reports/docs/changelog.rst``. - -.. The included file's own label and `Changelog` title are skipped: `poe bump` needs the - title there, and this page has its heading above. - -.. include:: ../../ub-test-reports/docs/changelog.rst - :start-line: 4 diff --git a/packages/sphinx-test-reports/tests/test_cli_convert.py b/packages/sphinx-test-reports/tests/test_cli_convert.py index 1832d24c2..09ccd645d 100644 --- a/packages/sphinx-test-reports/tests/test_cli_convert.py +++ b/packages/sphinx-test-reports/tests/test_cli_convert.py @@ -1,14 +1,4 @@ -"""Tests for the Sphinx-free ``test-reports build needs`` CLI (TR-A). - -This is the keystone of the build-system story: a test-XML to needs.json -conversion that runs as a build action *outside* Sphinx, so the docs build only -imports the result. Two properties are load-bearing and therefore tested -explicitly rather than assumed: - -* the CLI must not import Sphinx -- otherwise a Bazel action pulls the whole - documentation toolchain into the test-result conversion; -* the output must be byte-stable, because it is a cached build artifact and - qualification evidence. +"""The ``test-reports build needs`` CLI's output, inside Sphinx (TR-A). This is the half that needs Sphinx: the converter's output checked against sphinx-needs' schema and imported into a build. The converter's own tests are ub-test-reports', diff --git a/packages/ub-test-reports/AGENTS.md b/packages/ub-test-reports/AGENTS.md index 4c32bf363..805a7c996 100644 --- a/packages/ub-test-reports/AGENTS.md +++ b/packages/ub-test-reports/AGENTS.md @@ -27,11 +27,13 @@ uv run poe build-ub-test-reports # sdist + wheel into dist/ub-test-repo - **Nothing here may import Sphinx, sphinx-needs or docutils** — not at module level and not in a function body, because the converter runs as a build action and the plugin - inside a test run, and neither has a documentation toolchain. Every environment the - workspace root produces has Sphinx in it, so the default `.venv` sees only a module-level - import in the converter's import chain (a subprocess test lists `sys.modules`); a - function-body import passes there. CI's `toolchain-free` job is the fence — it installs - the built wheel where the toolchain is absent and runs the whole suite there. + inside a test run, and neither has a documentation toolchain. `tests/test_imports.py` + refuses any import STATEMENT naming the toolchain (or the extension), at any depth and in + every environment, whether or not a test reaches the line; three subprocess tests also + check that importing the CLI's chain, the plugin and `projectconfig` loads none of it. + CI's `toolchain-free` job is the fence for what a static walk cannot see — a dependency + that drags Sphinx in, an import spelled dynamically — and for the plugin's subprocess runs: + it installs the built wheel where the toolchain is absent and runs the whole suite there. - **`ub-project` is its `ubproject.toml` reader.** Finding, loading and anchoring the file come from there (`packages/ub-project/design/reading-contract.md` is the specification); what stays here is the `[test_reports]` policy -- keys, types, normalisation, unknown keys diff --git a/packages/ub-test-reports/docs/changelog.rst b/packages/ub-test-reports/docs/changelog.rst index be7050f07..a0ec36852 100644 --- a/packages/ub-test-reports/docs/changelog.rst +++ b/packages/ub-test-reports/docs/changelog.rst @@ -8,5 +8,6 @@ Unreleased - 1.0.0 is the first release: the converter, the pytest plugin, the parsers, the result vocabulary, the deterministic IDs and the ``[test_reports]`` model, moved out of - sphinx-test-reports 2.0.0 unchanged; the plugin is ``-p ub_test_reports.pytest_plugin``; - its wire names are unchanged. + sphinx-test-reports 2.0.0 unchanged; the plugin is ``-p ub_test_reports.pytest_plugin``, + and its warning prefix and pluggy registration name now say ``ub_test_reports`` (the + extension's changelog lists both); its wire names are unchanged. diff --git a/tools/tests/test_check_workspace.py b/tools/tests/test_check_workspace.py index 7cd254452..40dbcf386 100644 --- a/tools/tests/test_check_workspace.py +++ b/tools/tests/test_check_workspace.py @@ -919,8 +919,8 @@ def test_only_a_real_import_counts(workspace, capsys, line) -> None: def test_a_declaration_in_an_extra_satisfies_an_import(workspace, capsys) -> None: - """sphinx-test-reports' shape: its core install must stay docutils-free, so the - floor lives in the extra that installs the Sphinx toolchain.""" + """The shape sphinx-test-reports 2.0.0 had: its core install had to stay + docutils-free, so the floor lived in the extra that installed the Sphinx toolchain.""" root = workspace( { "acme-reports": { From c2d6354a116d056abfc27e5693a1b522f215b3ce Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Thu, 1 Oct 2026 03:38:27 +0200 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=93=9A=20Say=20what=20the=20static=20?= =?UTF-8?q?import=20walk=20and=20the=20toolchain-free=20job=20each=20catch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validation round of the split review: the static test's docstring now names the import spellings it reads (a positional string literal) and the ones it cannot; the toolchain-free job's header and the core's AGENTS.md add the qualifier that the job sees a dynamic import only on a line a test reaches; the three subprocess tests load no Sphinx, not "none of the toolchain"; the quoted uv message keeps its backticks. --- .github/workflows/ci.yaml | 5 +++-- packages/sphinx-test-reports/docs/changelog.rst | 2 +- packages/ub-test-reports/AGENTS.md | 7 ++++--- packages/ub-test-reports/tests/test_imports.py | 9 +++++---- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8b5ed48e9..062ecb1e2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -470,8 +470,9 @@ jobs: # declares, asserts that sphinx, sphinx_needs and docutils are all absent, and runs its # WHOLE suite there. The suite's own `tests/test_imports.py` refuses any import STATEMENT # naming the toolchain, in every environment; what this job adds is what a static walk - # cannot see -- a dependency that drags Sphinx in, an import spelled dynamically -- and - # the plugin's pytester subprocess runs, all without the toolchain. + # cannot see -- a dependency that drags Sphinx in, an import spelled dynamically -- on + # the lines a test reaches, and the plugin's pytester subprocess runs, all without the + # toolchain. # # The environment is on Python 3.11, the workspace floor: both packages it runs claim # `>=3.11`, and no other cell runs either of them there -- the matrix cells vary the diff --git a/packages/sphinx-test-reports/docs/changelog.rst b/packages/sphinx-test-reports/docs/changelog.rst index e5d9d1000..8fb87c270 100644 --- a/packages/sphinx-test-reports/docs/changelog.rst +++ b/packages/sphinx-test-reports/docs/changelog.rst @@ -32,7 +32,7 @@ The Sphinx-free half is its own distribution, ub-test-reports that takes the command from the package you name finds none in sphinx-test-reports: ``pipx install sphinx-test-reports`` fails with "No apps associated with package sphinx-test-reports", and ``uv tool install sphinx-test-reports`` with "No executables are - provided by package sphinx-test-reports". Name ``ub-test-reports`` there (``pipx install + provided by package \`sphinx-test-reports\`". Name ``ub-test-reports`` there (``pipx install ub-test-reports``, ``uv tool install ub-test-reports``), and in anything else that looks the script up in the installing package's own metadata. ``pip install sphinx-test-reports`` still puts ``test-reports`` on the path, through the dependency. diff --git a/packages/ub-test-reports/AGENTS.md b/packages/ub-test-reports/AGENTS.md index 805a7c996..b646864bd 100644 --- a/packages/ub-test-reports/AGENTS.md +++ b/packages/ub-test-reports/AGENTS.md @@ -30,10 +30,11 @@ uv run poe build-ub-test-reports # sdist + wheel into dist/ub-test-repo inside a test run, and neither has a documentation toolchain. `tests/test_imports.py` refuses any import STATEMENT naming the toolchain (or the extension), at any depth and in every environment, whether or not a test reaches the line; three subprocess tests also - check that importing the CLI's chain, the plugin and `projectconfig` loads none of it. + check that importing the CLI's chain, the plugin and `projectconfig` loads no Sphinx. CI's `toolchain-free` job is the fence for what a static walk cannot see — a dependency - that drags Sphinx in, an import spelled dynamically — and for the plugin's subprocess runs: - it installs the built wheel where the toolchain is absent and runs the whole suite there. + that drags Sphinx in, an import spelled dynamically — on the lines a test reaches, and for + the plugin's subprocess runs: it installs the built wheel where the toolchain is absent and + runs the whole suite there. - **`ub-project` is its `ubproject.toml` reader.** Finding, loading and anchoring the file come from there (`packages/ub-project/design/reading-contract.md` is the specification); what stays here is the `[test_reports]` policy -- keys, types, normalisation, unknown keys diff --git a/packages/ub-test-reports/tests/test_imports.py b/packages/ub-test-reports/tests/test_imports.py index c60c01fef..a0f8796e3 100644 --- a/packages/ub-test-reports/tests/test_imports.py +++ b/packages/ub-test-reports/tests/test_imports.py @@ -3,10 +3,11 @@ A static walk, not an import: it reads every module's source, so it refuses an import statement -- at module level, in a function body, in a ``try`` arm or under ``TYPE_CHECKING`` -- whether or not any test reaches that line, and in every environment, -the default one (which has Sphinx installed) included. String-spelled -``importlib.import_module("...")`` and ``__import__("...")`` calls are read too. What a -static walk cannot see -- a dependency that drags Sphinx in, an import spelled some other -way -- is what CI's ``toolchain-free`` job is for. +the default one (which has Sphinx installed) included. ``importlib.import_module("...")`` +and ``__import__("...")`` calls are read too, when the module name is a positional string +literal (a keyword argument, a variable, an f-string or the ``package`` argument are not). +What a static walk cannot see -- a dependency that drags Sphinx in, an import spelled some +other way -- is what CI's ``toolchain-free`` job is for, on the lines a test reaches. """ import ast