diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 63714c48e..79bd240ba 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -19,6 +19,7 @@ body: - sphinx-mounts - sphinx-codelinks - sphinx-test-reports + - ub-test-reports - ub-project - the repository (workflows, CI, release, docker, tooling) validations: diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index bf7ac5606..84abbad55 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -14,6 +14,7 @@ body: - sphinx-mounts - sphinx-codelinks - sphinx-test-reports + - ub-test-reports - ub-project - the repository (workflows, CI, release, docker, tooling) validations: diff --git a/.github/issue-labeler.yml b/.github/issue-labeler.yml index 914d152ba..075f2f32c 100644 --- a/.github/issue-labeler.yml +++ b/.github/issue-labeler.yml @@ -7,8 +7,10 @@ # patterns case-insensitive, like the workflow's `contains()` gate, so the two agree. 'pkg: sphinx-needs': - '/### Package[^#]*\bsphinx-needs\b/i' -# No two of `sphinx-needs`, `sphinx-mounts`, `sphinx-codelinks`, `sphinx-test-reports` and -# `ub-project` contains another as a \b-delimited word, so no two patterns can match one choice. Every +# No two of `sphinx-needs`, `sphinx-mounts`, `sphinx-codelinks`, `sphinx-test-reports`, +# `ub-test-reports` and `ub-project` contains another as a \b-delimited word, so no two patterns +# can match one choice (`ub-test-reports` is not a word inside `sphinx-test-reports`: the +# `-` before `test` is a boundary, but the pattern needs `ub-` there). Every # pattern has to be present before its option is offered by a form, though: the workflow # runs with `sync-labels: 1`, so a pattern that is missing when its option is picked means # the label is synced away rather than merely not added @@ -18,6 +20,8 @@ - '/### Package[^#]*\bsphinx-codelinks\b/i' 'pkg: sphinx-test-reports': - '/### Package[^#]*\bsphinx-test-reports\b/i' +'pkg: ub-test-reports': + - '/### Package[^#]*\bub-test-reports\b/i' 'pkg: ub-project': - '/### Package[^#]*\bub-project\b/i' 'pkg: workspace': diff --git a/.github/labeler.yml b/.github/labeler.yml index 8732546d3..df9215093 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -22,6 +22,9 @@ 'pkg: sphinx-needs-testkit': - changed-files: - any-glob-to-any-file: 'packages/sphinx-needs-testkit/**' +'pkg: ub-test-reports': + - changed-files: + - any-glob-to-any-file: 'packages/ub-test-reports/**' 'pkg: ub-project': - changed-files: - any-glob-to-any-file: 'packages/ub-project/**' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 532540524..062ecb1e2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -61,7 +61,7 @@ jobs: # ONE probe at a time, and one per package: with both planted at once a gate that had # dropped one package would still go red on the other and look healthy. run: | - probes="packages/sphinx-needs/src/sphinx_needs/_ty_gate_probe.py packages/sphinx-mounts/src/sphinx_mounts/_ty_gate_probe.py packages/sphinx-codelinks/src/sphinx_codelinks/_ty_gate_probe.py packages/sphinx-test-reports/src/sphinx_test_reports/_ty_gate_probe.py packages/ub-project/src/ub_project/_ty_gate_probe.py" + probes="packages/sphinx-needs/src/sphinx_needs/_ty_gate_probe.py packages/sphinx-mounts/src/sphinx_mounts/_ty_gate_probe.py packages/sphinx-codelinks/src/sphinx_codelinks/_ty_gate_probe.py packages/sphinx-test-reports/src/sphinx_test_reports/_ty_gate_probe.py packages/ub-test-reports/src/ub_test_reports/_ty_gate_probe.py packages/ub-project/src/ub_project/_ty_gate_probe.py" # shellcheck disable=SC2064 trap "rm -f $probes" EXIT for probe in $probes; do @@ -459,23 +459,27 @@ jobs: working-directory: packages/sphinx-test-reports toolchain-free: - name: Toolchain-free (sphinx-test-reports, ub-project) + name: Toolchain-free (ub-test-reports, ub-project) runs-on: ubuntu-latest permissions: contents: read - # sphinx-test-reports' converter (`test-reports`) and its pytest plugin are meant to run - # where the documentation toolchain is NOT installed -- that is why Sphinx and - # sphinx-needs are an EXTRA of that package rather than dependencies. This job is the - # only thing that keeps it true: it installs the package with the dependencies it - # declares, asserts that sphinx, sphinx_needs and docutils are all absent, and runs the - # modules that do not need a build. + # ub-test-reports -- the converter (`test-reports`), the pytest plugin, the parsers and the + # `[test_reports]` model -- is a DISTRIBUTION that must run where the documentation + # toolchain is NOT installed: a build action and a test run have none. Nothing else keeps + # that true at run time. This job installs the core's built wheel with the dependencies it + # declares, asserts that sphinx, sphinx_needs and docutils are all absent, and runs its + # WHOLE suite there. The suite's own `tests/test_imports.py` refuses any import STATEMENT + # naming the toolchain, in every environment; what this job adds is what a static walk + # cannot see -- a dependency that drags Sphinx in, an import spelled dynamically -- on + # the lines a test reaches, and the plugin's pytester subprocess runs, all without the + # toolchain. # # The environment is on Python 3.11, the workspace floor: both packages it runs claim # `>=3.11`, and no other cell runs either of them there -- the matrix cells vary the # sphinx series on the pinned interpreter -- so the job that proves "no toolchain" # proves the floor too. # - # ubproject, the shared reader for `ubproject.toml`, is installed into the SAME + # ub-project, the shared reader for `ubproject.toml`, is installed into the SAME # environment and its whole suite runs here: its contract is to import nothing outside # the standard library, so that every tool reading the file -- this converter included # -- can use it without the toolchain. The assertion below runs after both installs, so @@ -489,10 +493,9 @@ jobs: # groups are asked for. The environment is therefore built OUTSIDE the project with # `uv pip install --no-sources`, exactly as `release.yaml`'s compat cell builds its own. # - # It replaces the `toolchain_free` session of the noxfile the import retired. That - # noxfile's other lane, `plugin_floor` (the plugin on the oldest pytest of each Python), - # is deliberately NOT here: it belongs with the release that makes the plugin a shipped - # surface of its own. + # The extension, sphinx-test-reports, has hard Sphinx dependencies now, so nothing of it + # runs here; only its artefact fence does, because that fence reads archives and needs + # no toolchain. The plugin's oldest-pytest lane is `plugin-floor` below. steps: - uses: actions/checkout@v7 - uses: astral-sh/setup-uv@v10.2.0 @@ -500,8 +503,48 @@ jobs: enable-cache: true cache-suffix: toolchain-free cache-python: true - - name: Build the sdist and wheel as the release does, and check both - # The fence on the ARTEFACTS. Every other job installs this member editable, which + - name: Build ub-test-reports' sdist and wheel as the release does, and check both + # The fence on the core's ARTEFACTS, in `release.yaml`'s shape (no `--wheel`: the + # wheel is built FROM the sdist). flit ships one module, so what is checked is that it + # is the right one and complete -- every tracked file under `src/`, `schemas/JUnit.xsd` + # (which the XML validation reads at run time) and `py.typed` among them -- that the + # wheel's top level is exactly `ub_test_reports` and its dist-info, and that the sdist + # carries neither `tests/` (with its fixture copies) nor `docs/` + run: | + uv build --package ub-test-reports --no-sources -o /tmp/core-dist + git ls-files packages/ub-test-reports/src | uv run --no-project python -c " + import pathlib, sys, tarfile, zipfile + dist = pathlib.Path('/tmp/core-dist') + (wheel,) = dist.glob('*.whl') + (sdist,) = dist.glob('*.tar.gz') + wanted = {line.strip().split('/src/', 1)[1] for line in sys.stdin if line.strip()} + errors = [] + for name in ('ub_test_reports/__init__.py', 'ub_test_reports/schemas/JUnit.xsd', 'ub_test_reports/py.typed'): + if name not in wanted: + errors.append(f'{name} is not tracked under src/') + with tarfile.open(sdist) as tar: + members = [m.name.split('/', 1)[1] for m in tar.getmembers() if m.isfile()] + in_src = {name.removeprefix('src/') for name in members if name.startswith('src/')} + errors += [f'missing from {sdist.name}: src/{name}' for name in sorted(wanted - in_src)] + errors += [f'{sdist.name} ships {name}' for name in sorted(members) + if name.startswith(('tests/', 'docs/'))] + shipped = set(zipfile.ZipFile(wheel).namelist()) + errors += [f'missing from {wheel.name}: {name}' for name in sorted(wanted - shipped)] + top = {name.split('/', 1)[0] for name in shipped} + dist_info = {name for name in top if name.endswith('.dist-info')} + if len(dist_info) != 1 or top - dist_info != {'ub_test_reports'}: + errors.append(f'{wheel.name} top level is {sorted(top)}, not exactly ub_test_reports and one dist-info') + licences = sorted(n.split('/licenses/', 1)[1] for n in shipped if '.dist-info/licenses/' in n) + if licences != ['LICENSE']: + errors.append(f'{wheel.name} ships licence files {licences}, not exactly LICENSE') + print(f'{sdist.name}: {len(members)} files; {wheel.name}: {len(shipped)} entries; {len(wanted)} tracked files under src/') + sys.exit('\\n'.join(errors) if errors else 0) + " + - name: Build sphinx-test-reports' sdist and wheel as the release does, and check both + # The fence on the extension's ARTEFACTS. It only reads archives, so it needs no + # toolchain and lives here rather than in a job of its own; nothing below installs + # this wheel -- the extension needs Sphinx, and the release's compat cell walks and + # tests it. Every other job installs this member editable, which # reads `src/`, so a build configuration that dropped a package -- the # `sphinxcontrib/test_reports/` aliases most of all, which flit would drop without a # word -- would leave the whole suite green over a broken release. So this builds in @@ -558,46 +601,100 @@ jobs: sys.exit('\\n'.join(errors) if errors else 0) " - name: Build an environment with no documentation toolchain in it - # from the wheel checked above, so the modules below -- the aliases among them -- - # run against the artefact, not against the checkout + # from the core's wheel checked above, so its suite runs against the artefact, not + # against the checkout. `pytest-xdist` so that the plugin's three xdist tests run + # here instead of skipping on `importorskip` run: | uv venv --python 3.11 /tmp/toolchain-free - wheel=$(ls /tmp/str-dist/*.whl) - uv pip install --python /tmp/toolchain-free --no-sources "${wheel}[pytest]" packages/ub-project - - name: Assert the toolchain really is absent + wheel=$(ls /tmp/core-dist/*.whl) + uv pip install --python /tmp/toolchain-free --no-sources "${wheel}[pytest]" pytest-xdist packages/ub-project + - name: Assert the toolchain really is absent, and pytest-xdist present # the fence's fence. Without this the job would still pass with Sphinx installed, - # and would be testing nothing it claims to test. + # and would be testing nothing it claims to test. pytest-xdist the other way round: + # without it the plugin's three xdist tests skip on `importorskip` and the job + # stays green run: | /tmp/toolchain-free/bin/python -c " import importlib.util, sys present = [m for m in ('sphinx', 'sphinx_needs', 'docutils') if importlib.util.find_spec(m)] - sys.exit(f'toolchain installed: {present}' if present else 0) + if present: + sys.exit(f'toolchain installed: {present}') + if importlib.util.find_spec('xdist') is None: + sys.exit(\"pytest-xdist is not installed: the plugin's three xdist tests would skip\") " - - name: Run the modules that need no documentation build - # From the REPOSITORY ROOT, so pytest's rootdir is this repository and the root - # `[tool.pytest.ini_options]` applies -- which is where the `toolchain` marker is - # registered, so `-m "not toolchain"` deselects rather than warning. - run: | - /tmp/toolchain-free/bin/python -m pytest -q -m "not toolchain" \ - packages/sphinx-test-reports/tests/test_aliases.py \ - packages/sphinx-test-reports/tests/test_cli_config.py \ - packages/sphinx-test-reports/tests/test_cli_convert.py \ - packages/sphinx-test-reports/tests/test_identity.py \ - packages/sphinx-test-reports/tests/test_junit_parser.py \ - packages/sphinx-test-reports/tests/test_junit_parser_gtest.py \ - packages/sphinx-test-reports/tests/test_needs_export.py \ - packages/sphinx-test-reports/tests/test_project_config.py \ - packages/sphinx-test-reports/tests/test_pytest_plugin.py \ - packages/sphinx-test-reports/tests/test_result_vocabulary.py \ - packages/sphinx-test-reports/tests/test_toolchain.py + - name: Run the ub-test-reports suite + # WHOLE, with no marker and no file list: every test of the core must pass without the + # toolchain. From the repository root, so the root `[tool.pytest.ini_options]` applies. + # The installed wheel is what is imported -- `src/` is never on `sys.path` under + # `--import-mode=importlib` -- so this tests the artefact. `-rs` prints any skip. + run: /tmp/toolchain-free/bin/python -m pytest -q -rs packages/ub-test-reports/tests - name: Run the ub-project suite # A step of its own so that a red check names the suite. (One run over both would - # collect today -- measured -- but only because this suite has no `conftest.py`: - # add one and it resolves to `tests.conftest` beside test-reports', the collision - # the root `testpaths` comment describes.) The installed wheel is what is imported - # -- `src/` is never on `sys.path` -- so this also tests the artefact + # collect today -- measured -- but only because this suite has no `conftest.py`: add + # one and it resolves to `tests.conftest` beside ub-test-reports', and pytest stops + # with "Plugin already registered under a different name", measured too.) The + # installed wheel is what is imported -- `src/` is never on `sys.path` -- so this also + # tests the artefact run: /tmp/toolchain-free/bin/python -m pytest -q packages/ub-project/tests + plugin-floor: + name: Plugin floor (pytest ${{ matrix.pytest }}, Python ${{ matrix.python }}) + runs-on: ubuntu-latest + permissions: + contents: read + # ub-test-reports' pytest plugin declares `pytest>=7.0` (its `pytest` extra), and every + # other job runs the newest pytest. This runs the core's whole suite on the oldest pytest + # each floor Python can take: 7.0.1 on 3.11, and 7.3.2 on 3.12, the first pytest that runs + # there at all. Same environment shape as `toolchain-free` -- the built wheel, no + # toolchain -- with pytest pinned. + # + # From the package directory -- pytest still finds the root's `pyproject.toml` + # configuration there -- because from the root pytest 7.3.2 loads the conftest of every + # `testpaths` entry (sphinx-needs') and dies on its imports (measured). `pytest-xdist` is not + # pinned: the newest (3.8.0, measured) declares `pytest>=7.0` and passes on both pins, so + # the plugin's three xdist tests run rather than skip. + strategy: + fail-fast: false + matrix: + include: + - python: "3.11" + pytest: "7.0.1" + - python: "3.12" + pytest: "7.3.2" + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.2.0 + with: + enable-cache: true + cache-suffix: plugin-floor-${{ matrix.python }} + cache-python: true + - name: Build an environment with the oldest pytest and no documentation toolchain + run: | + uv build --package ub-test-reports --no-sources --wheel -o /tmp/core-dist + uv venv --python ${{ matrix.python }} /tmp/plugin-floor + wheel=$(ls /tmp/core-dist/*.whl) + uv pip install --python /tmp/plugin-floor --no-sources "${wheel}[pytest]" \ + "pytest==${{ matrix.pytest }}" pytest-xdist packages/ub-project + - name: Assert the pin, pytest-xdist present, and the toolchain absent + # the pin's positive control: a resolver that moved pytest would make this job test + # the newest pytest again while still reporting the floor. pytest-xdist: a resolver + # that could no longer place one next to the pinned pytest would make the plugin's + # three xdist tests skip, green + run: | + /tmp/plugin-floor/bin/python -c " + import importlib.util, sys + import pytest + if pytest.__version__ != '${{ matrix.pytest }}': + sys.exit(f'pytest {pytest.__version__} is installed, not ${{ matrix.pytest }}') + if importlib.util.find_spec('xdist') is None: + sys.exit(\"pytest-xdist is not installed: the plugin's three xdist tests would skip\") + present = [m for m in ('sphinx', 'sphinx_needs', 'docutils') if importlib.util.find_spec(m)] + sys.exit(f'toolchain installed: {present}' if present else 0) + " + - name: Run the ub-test-reports suite + working-directory: packages/ub-test-reports + run: /tmp/plugin-floor/bin/python -m pytest -q -rs tests + bazel: name: Mounts Bazel integration runs-on: ubuntu-latest @@ -677,6 +774,7 @@ jobs: - docs-codelinks - docs-reports - toolchain-free + - plugin-floor - bazel - smoke-needs diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2efe6b6dd..19488198c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,11 +4,12 @@ name: Release # `-v`, where is a directory name under `packages/`. # # The filename is load-bearing and must not change: every publishable member's PyPI project -# (sphinx-needs, sphinx-mounts, sphinx-codelinks, sphinx-test-reports, ub-project) names -# `release.yaml` in its trusted-publisher entry. +# (sphinx-needs, sphinx-mounts, sphinx-codelinks, sphinx-test-reports, ub-project, +# ub-test-reports) names `release.yaml` in its trusted-publisher entry -- ub-test-reports' is +# created, as a pending publisher, before its first tag. # # The leading `*` in the tag filter is deliberately loose -- GitHub's tag filters are globs, -# not regexes, and cannot express "one of these four names" -- and the `plan` job is what +# not regexes, and cannot express "one of these names" -- and the `plan` job is what # refuses everything else. That order matters: GitHub *creates* an environment that a # workflow names and that does not exist, with no protection rules at all, so # `environment: pypi-${{ needs.plan.outputs.dist }}` is a fence only for the names that diff --git a/.github/workflows/test-extensions.yaml b/.github/workflows/test-extensions.yaml index 4119ed7ca..db9eb76e1 100644 --- a/.github/workflows/test-extensions.yaml +++ b/.github/workflows/test-extensions.yaml @@ -165,6 +165,24 @@ jobs: flags: reports files: ./reports.xml fail_ci_if_error: true + - name: "ub-test-reports: pytest" + # sphinx-test-reports' Sphinx-free core: the converter, the pytest plugin, the parsers + # and the `[test_reports]` model. Here for every cell's platform and interpreter -- + # Windows above all, where the converter's path handling and the plugin's source + # locations are the platform-sensitive part -- and for coverage under a flag of its + # own; `toolchain-free` in ci.yaml stays the run that proves it imports no Sphinx. + # Same `if:` as the steps above + if: ${{ !cancelled() && steps.prepare.outcome == 'success' }} + run: uv run --no-sync pytest -v packages/ub-test-reports/tests --cov=ub_test_reports --cov-report=xml:ub-test-reports.xml --cov-report=term-missing + - name: "ub-test-reports: upload to Codecov" + if: inputs.upload-coverage && github.event.pull_request.head.repo.full_name == github.repository && github.repository == 'useblocks/sphinx-needs' && github.actor != 'dependabot[bot]' + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} + name: ub-test-reports-pytests + flags: ub-test-reports + files: ./ub-test-reports.xml + fail_ci_if_error: true - name: "ub-project: pytest" # The shared `ubproject.toml` reader, which every extension above is to depend on. # Its suite takes under a second and needs nothing this cell does not already have; diff --git a/AGENTS.md b/AGENTS.md index f85f91646..efa4cdf69 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,8 @@ the shim. | its documentation | `packages/sphinx-needs/docs/` (changelog: `docs/changelog.rst`) | | sphinx-mounts' behaviour, tests, documentation | `packages/sphinx-mounts/{src/sphinx_mounts,tests,docs}/` — start at [`packages/sphinx-mounts/AGENTS.md`](packages/sphinx-mounts/AGENTS.md) | | sphinx-codelinks' behaviour, tests, documentation | `packages/sphinx-codelinks/{src/sphinx_codelinks,tests,docs}/` — start at [`packages/sphinx-codelinks/AGENTS.md`](packages/sphinx-codelinks/AGENTS.md) | -| sphinx-test-reports' behaviour, tests, documentation | `packages/sphinx-test-reports/{src/sphinx_test_reports,tests,docs}/` — start at [`packages/sphinx-test-reports/AGENTS.md`](packages/sphinx-test-reports/AGENTS.md) | +| sphinx-test-reports' behaviour (the Sphinx extension), tests, documentation | `packages/sphinx-test-reports/{src/sphinx_test_reports,tests,docs}/` — start at [`packages/sphinx-test-reports/AGENTS.md`](packages/sphinx-test-reports/AGENTS.md); its `docs/` are ub-test-reports' too | +| ub-test-reports' behaviour (the converter, the pytest plugin, the parsers, the `[test_reports]` model — no Sphinx), tests | `packages/ub-test-reports/{src/ub_test_reports,tests}/` — start at [`packages/ub-test-reports/AGENTS.md`](packages/ub-test-reports/AGENTS.md) | | how every tool reads `ubproject.toml` — finding, anchoring, `[variants]`, the variant-data merge | `packages/ub-project/{src/ub_project,tests,design}/` — start at [`packages/ub-project/AGENTS.md`](packages/ub-project/AGENTS.md) | | the fixtures, helpers and renderer resolution three suites share | `packages/sphinx-needs-testkit/` — a member this repository never publishes, installed through the root's `test` group and loaded by each suite's `tests/conftest.py` as a pytest plugin | | the three conformance corpora | `packages/sphinx-needs/tests/conformance/` (needflow) and `packages/sphinx-mounts/tests/fixtures/variant_condition_conformance.toml` (variant conditions), whose repository of record is ubCode, and `packages/ub-project/tests/fixtures/ubproject_reading_conformance.toml` (reading `ubproject.toml`), whose record is THIS repository and which ubCode is to vendor — all shared byte-for-byte; do not reformat any of them (`.gitattributes` plus the yamlfmt and taplo excludes protect them) | @@ -37,7 +38,7 @@ the shim. | the PlantUML renderer | `vendor/plantuml/` — `pin.toml` (version + sha256, the one place either is written), the committed `plantuml-.jar` it names, and a `README.md`. `uv run poe verify-plantuml` fences the two against each other | | CI | `.github/workflows/`, and `.github/scripts/` for the three checks that must run *inside* a CI environment | | the docker image | `docker/` — a repository-level deliverable, like the workflows | -| Read the Docs | sphinx-needs: `.readthedocs.yml`, and it stays at the root under that exact name — the configuration path applies to every version, so moving it makes older tags unbuildable. sphinx-mounts: `packages/sphinx-mounts/.readthedocs.yaml`, sphinx-codelinks: `packages/sphinx-codelinks/.readthedocs.yaml`, and sphinx-test-reports: `packages/sphinx-test-reports/.readthedocs.yaml`, each of which its own RTD project points at; every path inside those is relative to the REPOSITORY root, not to the file | +| Read the Docs | sphinx-needs: `.readthedocs.yml`, and it stays at the root under that exact name — the configuration path applies to every version, so moving it makes older tags unbuildable. sphinx-mounts: `packages/sphinx-mounts/.readthedocs.yaml`, sphinx-codelinks: `packages/sphinx-codelinks/.readthedocs.yaml`, and sphinx-test-reports: `packages/sphinx-test-reports/.readthedocs.yaml`, each of which its own RTD project points at; every path inside those is relative to the REPOSITORY root, not to the file. ub-test-reports has no RTD project: it is documented on sphinx-test-reports' site, whose yaml installs it from the checkout first. ub-project has no docs site | **`tools/` is the workspace's tooling — a virtual member, never released, whose manifest declares the tooling's dependencies; `.github/scripts/` keeps only the checks that must @@ -75,17 +76,18 @@ rejects on upload, for the by-hand path. nothing**: standard library only, no Sphinx, fenced by its own `tests/test_imports.py` and by CI's `toolchain-free` job, which runs its suite where Sphinx is not installed. Its contract is its conformance corpus (ubCode is to vendor it) plus `design/reading-contract.md`; -it decides no policy — discovery, warnings and `-D` stay with each consumer. No member -depends on it yet: every `--no-sources` gate resolves a consumer from the index, so the -consumers arrive one pull request each after its first release, and from then on each -release of it re-floors all of them. +it decides no policy — discovery, warnings and `-D` stay with each consumer. sphinx-needs, +sphinx-mounts, sphinx-codelinks and ub-test-reports depend on it, so each release of it +re-floors all of them (`propagate_floors.py`). **Naming: `sphinx-*` is a Sphinx extension; `ub-*` is a useblocks package that is not one** -— a tool (such as the future `ub-test-reports`) or a library (such as `ub-project`, import -`ub_project`). The name does not say which of the two a `ub-*` package is; its README and +— a tool (such as `ub-test-reports`, import `ub_test_reports`) or a library (such as +`ub-project`, import `ub_project`). The name does not say which of the two a `ub-*` package is; its README and classifiers do. A `ub-*` library says in its README's first line that it is a library for the sphinx-needs family which the extensions pull in, and carries no `Framework :: Sphinx` -classifier. Its poe tasks keep the whole name (`test-ub-project`): only `sphinx-` is dropped. +classifier; a `ub-*` tool says it is a tool for the family, and carries none either. Their +poe tasks keep the whole name (`test-ub-project`, `test-ub-test-reports`): only `sphinx-` is +dropped. ## Commands @@ -96,6 +98,7 @@ uv run poe test-needs -k # trailing words are appended to the task' uv run poe test-mounts # the sphinx-mounts suite (bazel tests deselected) uv run poe test-codelinks # the sphinx-codelinks suite (adds the libclang group) uv run poe test-reports # the sphinx-test-reports suite +uv run poe test-ub-test-reports # the ub-test-reports suite (no Sphinx needed) uv run poe lint # every prek hook over the whole tree uv run poe typecheck # ty over both packages, against the oldest supported sphinx uv run poe typecheck-js-needs # tsc over the vendored needstable.js (needs node) @@ -110,6 +113,7 @@ uv run poe check-workspace # the manifests agree with each other (Lin uv run poe release-plan # what is pending, in what order (advice; exits 0) uv run poe bump --bump minor # stamp a release: version, literals, floors, lock, changelog uv run poe import-check-needs # import the wheel against PyPI-resolved dependencies +uv run poe import-check-ub-test-reports # the same for ub-test-reports (with its pytest extra) uv run poe import-check-codelinks # the same for sphinx-codelinks (with its libclang extra) uv run --frozen --no-sync pytest tools/tests -q # the tooling's own tests UV_PYTHON=3.12 uv run --no-sync poe test-needs-sphinx8 # one CI matrix cell @@ -169,7 +173,7 @@ and the testkit all are. The sdist is ≈7.4 MB rather than 28. **sphinx-test-reports is the two cases at once**: its SUITE needs neither renderer (no test document carries a rendering directive, and the dead 8.6 MB jar its tests used to carry left -with the import), while `docs-reports` DOES render — 13 `needflow` directives — so it needs +with the import; ub-test-reports' suite needs none either), while `docs-reports` DOES render — 13 `needflow` directives — so it needs `java` and `dot`, and its `.readthedocs.yaml` keeps `apt_packages` where both SIBLING extensions' have none. (sphinx-needs' own docs render too, and more: 48 needflow directives.) @@ -251,12 +255,12 @@ the rootdir, so the tasks carry `--ignore=performance` instead of naming `tests` the task's own command would be *added* to yours rather than replaced by it.) **A bare `pytest` at the root collects sphinx-needs' suite and the tooling's — not -sphinx-mounts', sphinx-codelinks', sphinx-test-reports' or ub-project's.** Their `tests` directories are deliberately +sphinx-mounts', sphinx-codelinks', sphinx-test-reports', ub-test-reports' or ub-project's.** Their `tests` directories are deliberately absent from `testpaths`: every package ships a `tests/__init__.py`, so under `--import-mode=importlib` every `conftest.py` resolves to the module name `tests.conftest` and a rootdir-invoked pytest refuses the second outright — listing one there collects *nothing*, rather than more. Run those suites through `poe test-mounts`, -`poe test-codelinks`, `poe test-reports` and `poe test-ub-project` (which cd into the package), the way CI does +`poe test-codelinks`, `poe test-reports`, `poe test-ub-test-reports` and `poe test-ub-project` (which cd into the package), the way CI does with an explicit path. The Lint job's "Check a bare root pytest still collects" step is what keeps the list honest. @@ -457,7 +461,7 @@ removed* (`error-on-warning` makes an unused suppression an error), never by loo Every issue and pull request carries one or more `pkg:` labels naming what it concerns: `pkg: ` (today `pkg: sphinx-needs`, `pkg: sphinx-mounts`, -`pkg: sphinx-codelinks`, `pkg: sphinx-test-reports`, `pkg: sphinx-needs-testkit` and `pkg: ub-project`) or `pkg: workspace` for the repository +`pkg: sphinx-codelinks`, `pkg: sphinx-test-reports`, `pkg: sphinx-needs-testkit`, `pkg: ub-test-reports` and `pkg: ub-project`) or `pkg: workspace` for the repository itself — workflows, CI, release, docker, tooling, the workspace root. Pull requests get theirs automatically from the paths they touch (`.github/labeler.yml`); the issue forms set it from their "Package" dropdown (`.github/issue-labeler.yml`). **An issue created diff --git a/README.md b/README.md index 59559ecd0..75ba10b73 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,9 @@ definitions and the CI workflows. | [`packages/sphinx-needs`](packages/sphinx-needs) | [`sphinx-needs`](https://pypi.org/project/sphinx-needs/) | the Sphinx extension for managing requirements and specifications — [documentation](https://sphinx-needs.readthedocs.io), [README](packages/sphinx-needs/README.rst) | | [`packages/sphinx-mounts`](packages/sphinx-mounts) | [`sphinx-mounts`](https://pypi.org/project/sphinx-mounts/) | the Sphinx extension that mounts external source trees into a build without copying or symlinking — [documentation](https://sphinx-mounts.useblocks.com), [README](packages/sphinx-mounts/README.md) | | [`packages/sphinx-codelinks`](packages/sphinx-codelinks) | [`sphinx-codelinks`](https://pypi.org/project/sphinx-codelinks/) | fast source-code traceability for sphinx-needs — it scans source files for comment markers, turns them into needs, and links documentation to exact source lines — [documentation](https://codelinks.useblocks.com), [README](packages/sphinx-codelinks/README.md) | -| [`packages/sphinx-test-reports`](packages/sphinx-test-reports) | [`sphinx-test-reports`](https://pypi.org/project/sphinx-test-reports/) | test results as needs: JUnit/ctest/googletest XML and tox-envreport JSON become needs in a build, and a `test-reports` command turns the same reports into a `needs.json` without running Sphinx — [documentation](https://sphinx-test-reports.readthedocs.io), [README](packages/sphinx-test-reports/README.rst) | +| [`packages/sphinx-test-reports`](packages/sphinx-test-reports) | [`sphinx-test-reports`](https://pypi.org/project/sphinx-test-reports/) | test results as needs: the Sphinx extension in which JUnit/ctest/googletest XML and tox-envreport JSON become needs in a build — [documentation](https://sphinx-test-reports.readthedocs.io), [README](packages/sphinx-test-reports/README.rst) | +| [`packages/ub-test-reports`](packages/ub-test-reports) | `ub-test-reports` (not yet on PyPI) | the Sphinx-free half of test reports, which sphinx-test-reports depends on: the `test-reports` command that turns reports into a `needs.json` without running Sphinx, the pytest plugin, the parsers — documented on sphinx-test-reports' site, [README](packages/ub-test-reports/README.rst) | +| [`packages/ub-project`](packages/ub-project) | [`ub-project`](https://pypi.org/project/ub-project/) | the shared reader for `ubproject.toml` and its variant data, which the other packages depend on — no documentation site, [README](packages/ub-project/README.rst) | ## Why one repository, and why still several packages diff --git a/codecov.yml b/codecov.yml index 12ed7f29d..5dfc9c0c5 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,7 +1,6 @@ -# One repository, three packages, three coverage uploads -- so three flags with targets of -# their own. Without them all three would land on the `default` status and a young -# package's coverage would be averaged into a mature one's 80%, moving a gate nobody -# decided to move. +# One repository, one coverage upload per package -- so one flag each, with a target of its +# own. Without them every upload would land on the `default` status and a young package's +# coverage would be averaged into a mature one's 80%, moving a gate nobody decided to move. coverage: status: project: @@ -32,6 +31,14 @@ coverage: flags: [reports] target: auto threshold: 0.5% + # ub-test-reports, sphinx-test-reports' Sphinx-free core, uploaded by the same cell with + # `flags: ub-test-reports`. `target: auto` because its statements MOVED out of + # `reports` when the core was split off -- that flag's number moves with them, and + # neither number says anything about a change in coverage on that pull request + ub-test-reports: + flags: [ub-test-reports] + target: auto + threshold: 0.5% patch: default: target: 67% diff --git a/packages/sphinx-test-reports/.readthedocs.yaml b/packages/sphinx-test-reports/.readthedocs.yaml index 1924ce12d..2ce96e3b8 100644 --- a/packages/sphinx-test-reports/.readthedocs.yaml +++ b/packages/sphinx-test-reports/.readthedocs.yaml @@ -27,8 +27,8 @@ build: jobs: post_checkout: # Every pull request against this repository builds every Read the Docs project it - # has. A pull request that leaves `packages/sphinx-test-reports/` untouched has - # nothing for this one to build, so cancel it. Exit code 183 is Read the Docs' + # has. A pull request that leaves `packages/sphinx-test-reports/` and + # `packages/ub-test-reports/` untouched has nothing for this one to build, so cancel it. Exit code 183 is Read the Docs' # documented "skip" code # (https://docs.readthedocs.com/platform/stable/guides/build/skip-build.html): the # build is recorded as `cancelled`, and the commit status sent to GitHub is `success` @@ -43,7 +43,8 @@ build: # pull request did not touch it. That is the safe direction; it never skips a build # that was needed. # - # `packages/sphinx-test-reports/` is ALMOST the complete set of inputs, and the one + # The two packages are ALMOST the complete set of inputs -- ub-test-reports is one because + # this site documents it and installs it from the checkout below -- and the one # exception is deliberate: `docs/conf.py` also reads `vendor/plantuml/pin.toml` and # the jar beside it. A bump there changes only which renderer version draws the # diagrams, never whether the build succeeds, so it is not worth widening the filter @@ -56,8 +57,8 @@ build: echo "origin/master is not available in this clone; building to be safe" exit 0 fi - if git diff --quiet origin/master -- packages/sphinx-test-reports/; then - echo "nothing under packages/sphinx-test-reports/ changed; skipping this build (exit 183)" + if git diff --quiet origin/master -- packages/sphinx-test-reports/ packages/ub-test-reports/; then + echo "nothing under packages/sphinx-test-reports/ or packages/ub-test-reports/ changed; skipping this build (exit 183)" exit 183 fi @@ -72,14 +73,19 @@ sphinx: python: install: - # The workspace sibling FIRST, from this checkout. Read the Docs runs these entries in - # order with pip, and a member installed on its own would resolve `sphinx-needs` from - # PyPI -- the released version, not the one beside it -- so a docs build here could - # pass or fail on code the workspace does not have. With the sibling already installed - # the member's requirement is satisfied and pip fetches nothing (tight tracking keeps - # the checkout's version inside the member's floor and cap). + # The workspace siblings FIRST, from this checkout. Read the Docs runs these entries in + # order with pip, and a member installed on its own would resolve `sphinx-needs` and + # `ub-test-reports` from PyPI -- the released versions, not the ones beside it -- so a + # docs build here could pass or fail on code the workspace does not have, and between a + # core change and its release pip finds no matching `ub-test-reports` at all (measured: + # "Could not find a version that satisfies the requirement ub-test-reports<2,>=1.0.0.dev0"). + # With the siblings already installed the member's requirements are satisfied and pip + # fetches neither (tight tracking keeps the checkout's versions inside the member's floors + # and caps). - method: pip path: packages/sphinx-needs + - method: pip + path: packages/ub-test-reports - method: pip path: packages/sphinx-test-reports extra_requirements: diff --git a/packages/sphinx-test-reports/AGENTS.md b/packages/sphinx-test-reports/AGENTS.md index ab9219592..b8f1a7884 100644 --- a/packages/sphinx-test-reports/AGENTS.md +++ b/packages/sphinx-test-reports/AGENTS.md @@ -8,50 +8,37 @@ not of the workspace. ## Project Overview -sphinx-test-reports turns test results into needs. It has **three surfaces, and only one of -them is a Sphinx extension** — which is the single most important thing to know about this -package, because it shapes the manifest, the CI and the split that is coming: - -- **the extension** — `test-file`, `test-suite`, `test-case`, `test-report`, `test-results` - and `test-env` directives, which read JUnit / ctest / googletest XML and tox-envreport - JSON and create sphinx-needs items from them, plus the `tr_link` dynamic function; -- **the converter** — a `test-reports` console script that turns the same reports into a - `needs.json` **without running Sphinx at all**; -- **the pytest plugin** — `sphinx_test_reports.pytest_plugin`, which writes the XML - shape the extension reads, including per-case properties for traceability. - -So **Sphinx and sphinx-needs are an `[project.optional-dependencies]` extra, not -dependencies**: `pip install sphinx-test-reports` gets you `lxml`, `ub-project` and the -last two surfaces; `pip install "sphinx-test-reports[sphinx]"` gets you the extension. The -published wheel's `Requires-Dist` is those two alone. Two things in this repository exist -because of that — the `toolchain-free` CI job and this package's `compat-requirements.txt` -— and both are described below. +sphinx-test-reports is **the Sphinx extension, and only that**: the `test-file`, +`test-suite`, `test-case`, `test-report`, `test-results` and `test-env` directives, which read +JUnit / ctest / googletest XML and tox-envreport JSON and create sphinx-needs items from +them, plus the `tr_link` dynamic function. **Its dependencies are hard** — Sphinx, docutils, +sphinx-needs and `ub-test-reports`. The converter (`test-reports`), the pytest plugin, the +parsers, the result vocabulary, the deterministic IDs and the `[test_reports]` model are +**ub-test-reports** ([`packages/ub-test-reports/`](../ub-test-reports/AGENTS.md)), which +runs without Sphinx and which this package depends on; a change to any of those belongs +there. `[sphinx]` (empty) and `[pytest]` (a pass-through to `ub-test-reports[pytest]`) stay +as extras only so that 2.0.0's install lines keep working until 4.0. ## Package structure ```text -pyproject.toml # `[project]`, `[project.urls]`, `[project.scripts]` and the - # hatch build tables. NOT ruff, ty, pytest or dependency - # groups: those are the root's, and check (7) refuses them here -compat-requirements.txt # released deps the compat cell needs -- see "Releasing" below +pyproject.toml # `[project]`, `[project.urls]` and the hatch build tables. NOT + # ruff, ty, pytest or dependency groups: those are the root's, + # and check (7) refuses them here .readthedocs.yaml # this package's RTD project; its paths are REPOSITORY-root relative AUTHORS · LICENSE · README.rst design/ # import-commit-map.txt: old hash -> new hash for the 2026-09 import src/sphinxcontrib/test_reports/ # the pre-3.0 name: four warning aliases, removed in 4.0 src/sphinx_test_reports/ -├── __init__.py # `__version__`, and the lazy `setup` re-export -├── test_reports.py # the extension entry point: directives, config values, fields -├── cli.py # the `test-reports` converter command -├── pytest_plugin.py # the pytest plugin -├── junitparser.py · jsonparser.py · results.py · identity.py · fields.py -│ # the toolchain-free core: parsers, the result vocabulary, the -│ # deterministic case IDs, the one field table both writers share -├── projectconfig.py # the `[test_reports]` ubproject.toml model, read through ub-project -├── needs_export.py · remote.py · config.py · environment.py · exceptions.py · toolchain.py +├── __init__.py # `__version__` FIRST, then the eager `setup` import -- the order is +│ # load-bearing: `test_reports` imports `__version__` from here +├── test_reports.py # the extension entry point: directives, config values, the bridge +│ # that applies ub-test-reports' `[test_reports]` model +├── config.py · environment.py · exceptions.py ├── directives/ # one module per directive, all inheriting TestCommonDirective ├── functions/ # `tr_link`, a sphinx-needs dynamic function -├── css/ · schemas/JUnit.xsd +├── css/ └── directives/test_report_template.txt # the DEFAULT tr_report_template -- it SHIPS tests/ # `tests/__init__.py` is why this path is not in the root testpaths @@ -66,24 +53,27 @@ The package was `sphinxcontrib.test_reports` until 3.0. `src/sphinxcontrib/test_ keeps exactly four old names working until 4.0: the package as a Sphinx extension, which warns through Sphinx's logger (type `test_reports.deprecated`) and loads the real extension with `app.setup_extension`, and `pytest_plugin`, `junitparser` and `jsonparser`, one file -each, which put the REAL module into `sys.modules` under the old name with one -`FutureWarning` per process. **Do not add a finder or a catch-all**: every other old name is meant to -fail as a plain `ImportError`, and `tests/test_aliases.py` walks the real package -to hold that. **There is no `src/sphinxcontrib/__init__.py`, and there must never be +each, which put the REAL module -- `ub_test_reports.`, in the core -- into +`sys.modules` under the old name with one `FutureWarning` per process. **Do not add a finder +or a catch-all**: every other old name is meant to fail as a plain `ImportError`, and +`tests/test_aliases.py` walks BOTH real packages, this one and ub-test-reports, to hold that +-- so a module added to either is covered without anyone remembering the file. **There is no `src/sphinxcontrib/__init__.py`, and there must never be one**: `sphinxcontrib` is a PEP 420 namespace other distributions install into. ### hatchling, and the fence on the built artefacts This is the one member that builds with hatchling: its wheel ships two top-level packages, and flit ships one and drops the other without a word. An editable install reads `src/`, so -a build configuration that lost the aliases would leave every test green. The -`toolchain-free` job is therefore where the artefacts are checked. It builds in the -release's shape -- the sdist, then the wheel FROM the sdist, so the sdist's include list -bounds what the wheel ships -- and fails when either lacks a tracked file under `src/`, -when the sdist's files outside `src/` are not exactly its metadata files, or when the -wheel's top level is anything but the two packages and its dist-info, it ships anything -under `sphinxcontrib/` but `test_reports/`, or a licence file other than `LICENSE`; then it -runs its modules against that wheel. A new top-level package needs a line in +a build configuration that lost the aliases would leave every test green. CI's +`toolchain-free` job is therefore where the artefacts are checked -- the job is named for +ub-test-reports, whose suite it runs without Sphinx, and this fence lives there because it +only reads archives. It builds in the release's shape -- the sdist, then the wheel FROM the +sdist, so the sdist's include list bounds what the wheel ships -- and fails when either lacks +a tracked file under `src/`, when the sdist's files outside `src/` are not exactly its +metadata files, or when the wheel's top level is anything but the two packages and its +dist-info, it ships anything under `sphinxcontrib/` but `test_reports/`, or a licence file +other than `LICENSE`. Nothing installs that wheel there: it needs the toolchain, and the +release's compat cell is what walks and tests it. A new top-level package needs a line in `[tool.hatch.build.targets.wheel]` AND `[tool.hatch.build.targets.sdist]`. ### The suite needs no renderer; the DOCS need two @@ -119,43 +109,21 @@ The short name is **`reports`**, not `test-reports`: the naming rule takes the d name minus its `sphinx-` prefix, which here would collide with the task verb and give `test-test-reports`. -### One test drives an in-process pytest session, and the default environment breaks it - -`tests/test_pytest_plugin.py`'s `NESTED` fixture starts a `pytest.main()` *inside* a -`pytester` session. In a developer's default `.venv` that process has every installed plugin -loaded, and **pytest-playwright** — from the root `js` group, which the default `dev` group -includes — keeps a module-global soft-assertion scope, so the nested session dies with -*nested soft assertion scopes are not supported*. Every CI cell syncs -`--no-default-groups --group test --group sphinx-N`, where the plugin is absent. The fixture -therefore passes `-p no:playwright`, and **that line has to stay one line**: a sibling test -builds its own fixture from the same string by replacing the literal `str(inner)]) == 0`. - -This is the shape to remember: **run the suite in the default `.venv` AND in a cell.** Green -in one proves nothing about the other. - ### The old name still appears in the tree, on purpose -Three fixture files in `tests/doc_test/utils/` carry paths like -`file="sphinxcontrib/test_reports/junitparser.py"`: **test data** describing a historical -pytest run, not paths anything opens. The docs' `classname` examples match that data, and +Fixture files in `tests/doc_test/utils/` (and their copies under ub-test-reports' +`tests/fixtures/`) carry paths like `file="sphinxcontrib/test_reports/junitparser.py"`: +**test data** describing a historical pytest run, not paths anything opens. The docs' `classname` examples match that data, and the pytest plugin's reserved `user_properties` names (`sphinxcontrib.test_reports:file`, `:line`) are documented wire names. None of them is an import path, so none moved with the package; a rename `sed` over the tree would corrupt them silently. -### `ubproject.toml` is read through `ub-project` - -`projectconfig.py` finds, loads and anchors the file through `ub-project`, the workspace's -shared reader (a runtime dependency, standard library only): `find_project_config` is its -walk re-exported, and `load_toml` and `anchor` do the reading and the joining. What stays -here is this package's policy — the `[test_reports]` keys, their types, the normalisation, -unknown keys warned rather than fatal — and **`TomlConfigError`, which is still the only -exception either consumer catches**: `load_project_config` re-raises ub-project's -`ProjectConfigError` as it with the same message, and must never be made a subclass of it. -The walk stops at the first `ubproject.toml`, else at `.git`, and only where no `.git` -exists anywhere above, at `pyproject.toml` — so a member's `packages//pyproject.toml` -never ends it, and a repository-root `ubproject.toml` (there is none today) would be picked -up by every consumer under `packages/`. `packages/ub-project/design/reading-contract.md` is -the specification; a change the walk or the anchoring needs belongs there, not here. +### `ubproject.toml` is read by ub-test-reports + +The `[test_reports]` model -- keys, types, normalisation, `TomlConfigError` -- is +`ub_test_reports.projectconfig`, read through `ub-project`; the core's `AGENTS.md` has its +rules. What is here is the bridge in `test_reports.py` that applies it to the `tr_*` values +at `config-inited`, and `-D` precedence over it. ## Testing @@ -167,21 +135,20 @@ The suite spawns Sphinx builds in three places, and all three go through `tests/test_subprocess_fence.py` is what keeps that true; it reads SOURCE, because a site that spawns the bare word passes in every environment where `PATH` happens to be right. -**`toolchain-free` (a CI job in `ci.yaml`, not a cell) is the fence that keeps the converter -and the plugin importable with no documentation toolchain.** It cannot be a `uv sync` cell, -structurally: the root's `[project] dependencies` name every member, those are installed in -every environment, and sphinx-needs declares sphinx at runtime — so every environment this -root can produce has Sphinx in it. The job builds one outside the project with -`uv pip install --no-sources` of the wheel it has just built and checked, asserts `sphinx`, -`sphinx_needs` and `docutils` are all absent, and runs the toolchain-free modules with -`-m "not toolchain"`. The `toolchain` marker itself lives in the ROOT's `markers` list. +**This suite always has Sphinx**, and every test in it may use it: the converter's and the +plugin's tests are ub-test-reports' and run without the toolchain there. A test of a +Sphinx-free module does not belong here — not even split by a marker, which this package no +longer has. ## Releasing -`compat-requirements.txt` names `sphinx` and `sphinx-needs`, and unlike both siblings' it -covers the package's **own runtime** requirements rather than a test-only need — because they -are optional. The compat cell installs a bare wheel path with no extras, so without that file -the `import_check` walk fails `6 of 25 modules` on `sphinx_needs` before pytest even starts. +**ub-test-reports releases first.** This package's floor on the core is tight-tracked, so +the core's release pull request (`poe bump ub-test-reports …`) rewrites it, and every +release gate here resolves the core from PyPI: `poe import-check-reports`, the release +plan and the compat cell are red until the core version this tree names is published. The +core's documentation lives on this package's site, so that site's Read the Docs project +must build THIS repository (`packages/sphinx-test-reports/.readthedocs.yaml`) before the +core's first tag — until then its PyPI page links a site with no page about it. ## What the move into this workspace cost, deliberately @@ -193,8 +160,8 @@ Recorded here because none of it is visible in a diff: - **Five ruff rule families left**: `FURB`, `PERF`, `PGH`, `PIE`, `SLF`, which this package enabled and the root's shared set does not. All five are at zero violations today, which is exactly why the loss would otherwise be silent. -- **`plugin_floor`** — the plugin against the oldest pytest of each Python — has no - replacement yet. It returns with the release that makes the plugin a shipped surface. +- **`plugin_floor`** — the plugin against the oldest pytest of each Python — returned as + `ci.yaml`'s `plugin-floor` job when the plugin moved to ub-test-reports. - **mypy left for ty**, with the whole package checked rather than the 15-entry `exclude` the mypy configuration carried. - **Beyond those five families, the root `extend-ignore`s `B904`, `ICN001`, `ISC004` and diff --git a/packages/sphinx-test-reports/compat-requirements.txt b/packages/sphinx-test-reports/compat-requirements.txt deleted file mode 100644 index cfd435417..000000000 --- a/packages/sphinx-test-reports/compat-requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -# The RELEASED dependencies this package needs beyond what its wheel declares, for the -# compat cell in .github/workflows/release.yaml -- the one gate that runs a member's suite -# against everything as PUBLISHED rather than as checked out. It is installed BEFORE the -# wheel and then passed again as a `-c` constraint, so it cannot be downgraded by the -# wheel's own `Requires-Dist`. -# -# This file covers something neither sibling's does: sphinx-test-reports is the first member -# here whose RUNTIME dependencies are optional. Its 2.0.0 release moved Sphinx and -# sphinx-needs behind a `sphinx` extra -- so that the `test-reports` converter and the pytest -# plugin install without a documentation toolchain -- and the compat cell installs a bare -# wheel path with no extras. Measured on the built wheel: `Requires-Dist` is `lxml` and -# `ub-project`, full stop. -# -# Without these two lines the failure is LOUD rather than silent, which is the good case: -# `release.yaml` runs the `import_check` walk BEFORE pytest, and that walk fails -# `FAIL 6 of 25 modules failed to import` naming `sphinx_needs`, exit 1, under the step's -# default `bash -e`. Measured by running the cell by hand. With them, the walk imports 25 of -# 25 and the suite runs. -# -# Note that `sphinx` alone is already present in that environment -- the root `test` group -# carries it for the other suites -- and `sphinx-needs` is NOT, because it is a `[project]` -# dependency of the workspace ROOT and `uv export --only-group test` deliberately excludes -# it. It is named here anyway: this file states what THIS package needs, and a future change -# to the `test` group must not be able to take it away silently. -# -# The specifiers are the member's own `sphinx` extra, verbatim, so the cell tests the range -# the package actually claims. -sphinx>=7.4 -sphinx-needs>=8.5.0,<9 diff --git a/packages/sphinx-test-reports/docs/changelog.rst b/packages/sphinx-test-reports/docs/changelog.rst index c682e2dfc..8fb87c270 100644 --- a/packages/sphinx-test-reports/docs/changelog.rst +++ b/packages/sphinx-test-reports/docs/changelog.rst @@ -6,13 +6,76 @@ Changelog Unreleased ---------- +The Sphinx-free half is its own distribution, ub-test-reports +............................................................. + +- ‼️ The ``test-reports`` converter, the pytest plugin, the parsers, the result + vocabulary, the deterministic case IDs and the ``[test_reports]`` model of + ``ubproject.toml`` are now **ub-test-reports** 1.0.0, a distribution of their own with no + Sphinx in it, imported as ``ub_test_reports``. They moved unchanged:: + + pip install ub-test-reports # the test-reports command + pip install "ub-test-reports[pytest]" # and the pytest plugin + + sphinx-test-reports is the Sphinx extension alone, and depends on ub-test-reports, so + both still arrive with it. + +- ‼️ **pip install sphinx-test-reports brings Sphinx, Sphinx-Needs, docutils and + ub-test-reports again.** This reverses 2.0.0's install-footprint change: the extension's + dependencies are hard dependencies now, not the ``sphinx`` extra. A CI job or a Bazel + action that installed sphinx-test-reports only for the ``test-reports`` command or the + pytest plugin now gets the whole documentation toolchain -- and a resolver conflict + wherever that environment pins another Sphinx. Install ``ub-test-reports`` there instead: + it is the same command and the same plugin, without the toolchain. + +- ‼️ **The** ``test-reports`` **command belongs to ub-test-reports now**, so a tool installer + that takes the command from the package you name finds none in sphinx-test-reports: + ``pipx install sphinx-test-reports`` fails with "No apps associated with package + sphinx-test-reports", and ``uv tool install sphinx-test-reports`` with "No executables are + provided by package \`sphinx-test-reports\`". Name ``ub-test-reports`` there (``pipx install + ub-test-reports``, ``uv tool install ub-test-reports``), and in anything else that looks + the script up in the installing package's own metadata. ``pip install + sphinx-test-reports`` still puts ``test-reports`` on the path, through the dependency. + +- The ``sphinx`` extra is accepted and ignored until 4.0 -- ``pip install + "sphinx-test-reports[sphinx]"`` installs exactly what the bare line does -- and the + ``pytest`` extra passes through to ``ub-test-reports[pytest]`` until 4.0, so 2.0.0's + documented install lines keep working. + +- ‼️ **The pytest plugin is** ``-p ub_test_reports.pytest_plugin``. Its pluggy registration + name is now ``ub_test_reports.xml_shape`` and the configuration warnings it issues start + with ``ub_test_reports.pytest_plugin:`` (2.0.0: ``sphinxcontrib.test_reports.xml_shape`` + and ``sphinxcontrib.test_reports.pytest_plugin:``); a warning filter matching that text + needs the new prefix. The two ``user_properties`` wire names it reserves do not change. + +- ‼️ **The old module names need the extension.** ``sphinxcontrib.test_reports.junitparser``, + ``.jsonparser`` and ``.pytest_plugin`` (below) are shipped by sphinx-test-reports, so they + resolve only where it is installed -- which now means with Sphinx. On 2.0.0 a Sphinx-free + environment could ``pip install sphinx-test-reports`` and import them; on 3.0 such an + environment installs ``ub-test-reports`` and imports ``ub_test_reports.junitparser`` (and + so on). With only ub-test-reports installed the old names fail as a plain + ``ModuleNotFoundError: No module named 'sphinxcontrib'``. + +- 🔧 The load-time toolchain check is gone. It existed because the toolchain was an opt-in + extra pip never saw; as hard dependencies, pip resolves the floors itself. An environment + whose Sphinx-Needs is downgraded below the floor AFTER installing is no longer refused: + the extension does not check, so a below-floor Sphinx-Needs runs untested (it may work, it + may fail anywhere); ``pip check`` (or ``uv pip check``) names the conflict. The extension + itself still refuses a Sphinx older than 7.4 when it loads. With the check went + ``compat-requirements.txt``, which the release's compatibility cell needed only while the + toolchain was optional. + +- The modules that moved were ``sphinx_test_reports.`` only in this unreleased + version -- 2.0.0 shipped them as ``sphinxcontrib.test_reports.`` -- so no alias is + kept for the ``sphinx_test_reports`` spelling; the old names that do keep working are the + four below. + The import name moves ..................... -- ♻️ The package is imported as ``sphinx_test_reports`` now, not - ``sphinxcontrib.test_reports``: the extension is ``extensions = ["sphinx_test_reports"]`` - and the pytest plugin is ``-p sphinx_test_reports.pytest_plugin``. Nothing else about - either changes. +- ♻️ The extension is imported as ``sphinx_test_reports`` now, not + ``sphinxcontrib.test_reports``: ``extensions = ["sphinx_test_reports"]``. The pytest + plugin and the parsers moved further, to ``ub_test_reports`` (above). Four old names keep working until **4.0**, each with a warning that names its replacement. 4.0 turns all four into errors that say the same thing. Under pytest's @@ -28,7 +91,8 @@ The import name moves loads the extension once. - **The pytest plugin**, ``-p sphinxcontrib.test_reports.pytest_plugin`` (and the same name in ``addopts``, ``PYTEST_PLUGINS`` or a ``conftest.py``'s ``pytest_plugins``), - loads the real plugin and raises a ``FutureWarning`` at start-up -- once per process, so + loads the real plugin, ``ub_test_reports.pytest_plugin``, and raises a ``FutureWarning`` + at start-up -- once per process, so once more for each pytest-xdist worker. A filter written against the old name -- ``ignore::sphinxcontrib.test_reports.pytest_plugin.TestReportsConfigWarning`` -- still matches, because resolving it imports the alias. Change it together with the ``-p`` @@ -38,7 +102,8 @@ The import name moves ``pytest_plugins``, stops pytest with "Plugin already registered under a different name"; keep one. - **The parsers**, ``sphinxcontrib.test_reports.junitparser`` and - ``sphinxcontrib.test_reports.jsonparser``, are the real modules under the old name, + ``sphinxcontrib.test_reports.jsonparser``, are the real modules -- + ``ub_test_reports.junitparser`` and ``ub_test_reports.jsonparser`` -- under the old name, with a ``FutureWarning`` (once per process) that points at the ``import`` statement naming them -- at ``importlib`` itself when they are loaded with ``importlib.import_module``. Their classes, and a ``mock.patch`` target through the old @@ -58,8 +123,8 @@ The import name moves ordinary ``ImportError`` (``ModuleNotFoundError`` for an ``import`` statement): ``projectconfig``, ``identity``, ``results``, the directives and the rest were never documented as an API. The module is the same under - the new name -- ``sphinxcontrib.test_reports.projectconfig`` is - ``sphinx_test_reports.projectconfig``. + its new name -- ``sphinxcontrib.test_reports.projectconfig`` is + ``ub_test_reports.projectconfig``, and the directives are under ``sphinx_test_reports``. The two ``user_properties`` names the pytest plugin reserves for its location override, ``sphinxcontrib.test_reports:file`` and ``sphinxcontrib.test_reports:line``, are wire @@ -89,9 +154,10 @@ New and Improved - **Release tags** are prefixed: ``sphinx-test-reports-v2.0.0`` rather than ``2.0.0``. Six of the ten historical bare names collided with existing Sphinx-Needs releases, so the prefix is load-bearing rather than tidy. - - **The distribution does not change.** It is still ``sphinx-test-reports``, and the - ``test-reports`` command is still the same command. The import name does change, in - this same release: see *The import name moves* above. + - **The distribution keeps its name.** It is still ``sphinx-test-reports``, and the + ``test-reports`` command is still the same command, now shipped by ub-test-reports, + which it depends on. The import names do change, in this same release: see the two + sections above. - 🔧 The shipped default ``tr_report_template`` ends with a ``literalinclude`` of itself, by a path relative to the including document. **That is still broken for your project** @@ -119,16 +185,15 @@ New and Improved two lines below a slice of the missing value, which raised first. It now raises ``TestReportFileNotSetError`` like every other configuration mistake. -- ⬆️ The ``sphinx`` extra now requires docutils 0.21 or newer, the floor the whole +- ⬆️ The extension now requires docutils 0.21 or newer, as a dependency, the floor the whole Sphinx-Needs workspace declares and type-checks against (previously whatever Sphinx - accepted, which is 0.20 for Sphinx 7.4 through 9.0). The extension checks the extra's - floors when it loads, so it now refuses docutils 0.20 with the install line; the - ``test-reports`` command and the pytest plugin still install no docutils at all. + accepted, which is 0.20 for Sphinx 7.4 through 9.0). The ``test-reports`` command and the + pytest plugin, which are ub-test-reports, install no docutils at all. - ♻️ The ``ubproject.toml`` reader is now `ub-project `__, - the shared reader every useblocks tool uses for the file, and a new runtime dependency - (standard library only, so the ``test-reports`` command and the pytest plugin still run - without Sphinx). Nothing changes in behaviour: the walk up to the repository root, the + the shared reader every useblocks tool uses for the file, and a runtime dependency of + ub-test-reports, where the ``[test_reports]`` model now lives (standard library only, so + the ``test-reports`` command and the pytest plugin still run without Sphinx). Nothing changes in behaviour: the walk up to the repository root, the anchoring of relative paths at the file's directory and every message are as before. One failure that used to escape as a traceback is now reported like the others: a file that is not UTF-8 is a configuration error naming the file. @@ -140,16 +205,17 @@ What the move costs, stated rather than left to the CI diff suite against Sphinx-Needs 6.0.1, 6.3.0, 7.0.0, 8.0.0 and 8.5.0; in the workspace the suite runs against the sibling in the tree, across Sphinx 7.4, 8.2 and 9.1 instead. With it, **the declared floor narrows from** ``sphinx-needs>=6.0.1`` **to** - ``sphinx-needs>=8.5.0,<9`` in the ``sphinx`` extra (and from ``>=6`` in ``docs``). That is + ``sphinx-needs>=8.5.0,<9`` (and from ``>=6`` in ``docs``). That is the workspace's tight-tracking policy for a dependency on a sibling, and it is enforced; it means this release supports a narrower range of Sphinx-Needs than 2.0.0 did. - **Five ruff rule families are no longer enforced here** -- ``FURB``, ``PERF``, ``PGH``, ``PIE`` and ``SLF`` -- because the workspace has one shared rule set and they are not in it. All five were at zero violations, so nothing changed in the code; what changed is that a new violation would no longer be caught. -- **The** ``plugin_floor`` **lane is gone for now**: the pytest plugin is no longer tested - against the oldest pytest of each Python. The ``toolchain_free`` lane survives, as a CI - job that installs the package with no documentation toolchain at all and asserts it. +- **Both retired nox lanes have CI jobs**, now that the plugin is ub-test-reports': the + ``toolchain_free`` lane is a job that installs ub-test-reports with no documentation + toolchain at all, asserts it, and runs that package's whole suite; the ``plugin_floor`` + lane runs the same suite on pytest 7.0.1 (Python 3.11) and 7.3.2 (Python 3.12). - **mypy is replaced by ty**, which checks the whole package -- the mypy configuration excluded fifteen modules. - **Beyond those five families, four rules this package enabled are now ignored** diff --git a/packages/sphinx-test-reports/docs/cli.rst b/packages/sphinx-test-reports/docs/cli.rst index 97c1b9ba7..d3b752208 100644 --- a/packages/sphinx-test-reports/docs/cli.rst +++ b/packages/sphinx-test-reports/docs/cli.rst @@ -4,7 +4,7 @@ Command line interface ====================== .. versionadded:: 2.0.0 -``Sphinx-Test-Reports`` ships a ``test-reports`` command that converts +``ub-test-reports`` ships a ``test-reports`` command that converts test-result XML into a ``needs.json`` **without running Sphinx**. Why this exists: parsing test results inside a documentation build couples the @@ -13,10 +13,15 @@ be cached by a build system, and the data is unavailable to anything else. The CLI splits the computation out; the documentation build only imports the result. The command imports no Sphinx code at all, so it can run as a build action in an -environment that has no documentation toolchain installed. Installed as -``pip install sphinx-test-reports`` -- without the ``sphinx`` extra the -:doc:`extension needs ` -- the package brings a single dependency, -``lxml``. +environment that has no documentation toolchain installed. Install it as +``pip install ub-test-reports``, whose dependencies are ``lxml`` and ``ub-project`` +(see :doc:`/install`); sphinx-test-reports depends on it, so the command is also there +wherever the extension is. + +.. versionchanged:: 3.0.0 + The command is shipped by ub-test-reports, not sphinx-test-reports. On 2.0.0 the bare + ``pip install sphinx-test-reports`` was the way to get it without Sphinx; that line now + brings the documentation toolchain. Converting a report ------------------- diff --git a/packages/sphinx-test-reports/docs/conf.py b/packages/sphinx-test-reports/docs/conf.py index c16046eac..4ed7b0299 100644 --- a/packages/sphinx-test-reports/docs/conf.py +++ b/packages/sphinx-test-reports/docs/conf.py @@ -8,6 +8,7 @@ # -- Path setup -------------------------------------------------------------- import datetime +import importlib.metadata import os import shutil @@ -35,10 +36,13 @@ copyright = f"team useblocks, 2017-{now.year}" author = "team useblocks" +# The full version, including alpha/beta/rc tags: the EXTENSION's, read from the installed +# distribution so it cannot drift from the manifest. ub-test-reports, which this site +# documents too, has its own version line; the index page says so +release = importlib.metadata.version("sphinx-test-reports") # The short X.Y version -version = "2.0" -# The full version, including alpha/beta/rc tags -release = "2.0.0" +_release = Version(release) +version = f"{_release.major}.{_release.minor}" needs_id_regex = ".*" needs_css = "dark.css" diff --git a/packages/sphinx-test-reports/docs/configuration.rst b/packages/sphinx-test-reports/docs/configuration.rst index da24e88f7..45c32a31a 100644 --- a/packages/sphinx-test-reports/docs/configuration.rst +++ b/packages/sphinx-test-reports/docs/configuration.rst @@ -421,6 +421,11 @@ shared file other useblocks tooling (sphinx-needs, sphinx-codelinks, sphinx-mounts, ubCode) reads. It describes the project once, so every tool acting on it works from the same settings instead of each restating them. +The section's model -- its keys, their types, how they are normalised and which are +rejected -- belongs to ub-test-reports (``ub_test_reports.projectconfig``), so the +:ref:`build needs command ` and the extension read it identically; the extension +applies it to the ``tr_*`` values below when a build starts. + .. code-block:: toml [test_reports] diff --git a/packages/sphinx-test-reports/docs/index.rst b/packages/sphinx-test-reports/docs/index.rst index 4341bf59e..1d6011b78 100644 --- a/packages/sphinx-test-reports/docs/index.rst +++ b/packages/sphinx-test-reports/docs/index.rst @@ -69,18 +69,35 @@ As example, here is a shorten list of tests results from the Sphinx-pytest examp Content ------- +Test reports come in two distributions: **sphinx-test-reports**, the Sphinx extension, and +**ub-test-reports**, the converter, the pytest plugin and the parsers, which run without +Sphinx and which the extension depends on. The version this site shows is the +extension's; ub-test-reports has its own version and its own +`changelog `__. + .. toctree:: :maxdepth: 2 + :caption: The Sphinx extension install directives/index configuration - cli - pytest - parsers filter functions examples/index + +.. toctree:: + :maxdepth: 2 + :caption: Without Sphinx (ub-test-reports) + + cli + pytest + parsers + +.. toctree:: + :maxdepth: 2 + :caption: Support + support changelog diff --git a/packages/sphinx-test-reports/docs/install.rst b/packages/sphinx-test-reports/docs/install.rst index 68518772d..cd0af6c12 100644 --- a/packages/sphinx-test-reports/docs/install.rst +++ b/packages/sphinx-test-reports/docs/install.rst @@ -3,26 +3,29 @@ Installation ============ -The package has three consumers, and each installs a different part of it. +Test reports come in two distributions, and each consumer installs the one it needs. -A documentation project needs the Sphinx extension, and with it the -documentation toolchain -- Sphinx and -`Sphinx-Needs `_ -- which is -the ``sphinx`` extra of the package:: +A documentation project needs the Sphinx extension, **sphinx-test-reports**. It brings the +documentation toolchain -- Sphinx, docutils and +`Sphinx-Needs `_ -- and ub-test-reports +with it:: - pip install "sphinx-test-reports[sphinx]" + pip install sphinx-test-reports + +A build action that only runs the :ref:`test-reports command `, which turns test +results into a ``needs.json`` without a Sphinx build, installs **ub-test-reports**, which +has no Sphinx in it (its dependencies are ``lxml`` and ``ub-project``):: + + pip install ub-test-reports A test runner that should write the XML shape the extension reads installs the -:ref:`pytest plugin ` as the ``pytest`` extra, which adds pytest +:ref:`pytest plugin `, ub-test-reports' ``pytest`` extra, which adds pytest and nothing of the documentation toolchain:: - pip install "sphinx-test-reports[pytest]" + pip install "ub-test-reports[pytest]" -A build action that only runs the :ref:`test-reports command `, which -turns test results into a ``needs.json`` without a Sphinx build, installs the -bare package, whose single dependency is ``lxml``:: - - pip install sphinx-test-reports +ub-test-reports has its own version number and +`changelog `__. .. versionchanged:: 2.0.0 ``pip install sphinx-test-reports`` -- without an extra -- no longer @@ -30,12 +33,17 @@ bare package, whose single dependency is ``lxml``:: ``sphinx`` extra to its install line; a test runner or a build action that has no documentation toolchain no longer gets one. -The ``sphinx`` extra also states the supported versions: Sphinx 7.4 and -Sphinx-Needs 6.0.1 or later. An extra is opt-in, so a project that keeps -installing the bare package into an environment holding an older toolchain -would never be told by ``pip``; the extension therefore checks the installed -versions when Sphinx loads it and stops the build with a message naming the -install line above. +.. versionchanged:: 3.0.0 + The converter, the pytest plugin and the parsers moved to their own distribution, + ub-test-reports, and ``pip install sphinx-test-reports`` installs Sphinx, docutils and + Sphinx-Needs again: they are dependencies of the extension, not an extra. A test runner + or a build action that installed sphinx-test-reports for the command or the plugin + installs ub-test-reports instead. The ``sphinx`` extra is accepted and ignored, and the + ``pytest`` extra installs ``ub-test-reports[pytest]``, until 4.0. + +The extension supports Sphinx 7.4 or later, docutils 0.21 or later and Sphinx-Needs 8.5 +(``>=8.5.0,<9``); ``pip`` resolves those as it installs it, and the extension itself +refuses an older Sphinx when it loads. After that the extension must be added to the ``conf.py`` file:: diff --git a/packages/sphinx-test-reports/docs/pytest.rst b/packages/sphinx-test-reports/docs/pytest.rst index 3d7c24369..203418242 100644 --- a/packages/sphinx-test-reports/docs/pytest.rst +++ b/packages/sphinx-test-reports/docs/pytest.rst @@ -11,7 +11,7 @@ Under pytest's default ``junit_family = xunit2`` no ```` carries the of ``tr_deterministic_case_ids``). Under ``xunit1`` pytest writes them, but counts the line from 0, keeps Bazel's runfiles prefix, and has no way to point a case at the file that drove it. And nothing records the requirements a test -verifies. ``Sphinx-Test-Reports`` ships a small pytest plugin that takes care +verifies. ``ub-test-reports`` ships a small pytest plugin that takes care of both. The plugin is generic: which properties exist, what they are called in the XML @@ -21,20 +21,22 @@ is the worked example below. Enabling it ----------- -The plugin is the ``pytest`` extra of the package: ``pip install -"sphinx-test-reports[pytest]"`` installs it without the documentation toolchain -(see :doc:`/install`). Enable it in the pytest configuration: +The plugin is part of ub-test-reports, the Sphinx-free distribution, and its ``pytest`` +extra brings pytest: ``pip install "ub-test-reports[pytest]"`` installs it without the +documentation toolchain (see :doc:`/install`). Enable it in the pytest configuration: .. code-block:: ini # pytest.ini / pyproject.toml [tool.pytest.ini_options] - addopts = -p sphinx_test_reports.pytest_plugin + addopts = -p ub_test_reports.pytest_plugin junit_family = xunit1 .. versionchanged:: 3.0.0 - The module is ``sphinx_test_reports.pytest_plugin``. The pre-3.0 name, - ``sphinxcontrib.test_reports.pytest_plugin``, still loads the same plugin - until 4.0, with a ``FutureWarning`` at start-up asking for the new one. That + The module is ``ub_test_reports.pytest_plugin``, in the ub-test-reports distribution; + ``pip install "sphinx-test-reports[pytest]"`` still installs it, until 4.0. The pre-3.0 + name, ``sphinxcontrib.test_reports.pytest_plugin``, still loads the same plugin until + 4.0 wherever sphinx-test-reports is installed, with a ``FutureWarning`` at start-up + asking for the new one. That warning comes before pytest installs its own filters, so only Python's own options silence it: ``PYTHONWARNINGS=ignore::FutureWarning``, which pytest-xdist's workers inherit, or @@ -62,7 +64,7 @@ Nothing else changes for tests that do not use the decorator below. The start-up notice is a ``TestReportsConfigWarning``. A project that turns warnings into errors (``filterwarnings = error``, ``-W error``) gets it as a clean usage error instead; -``ignore::sphinx_test_reports.pytest_plugin.TestReportsConfigWarning`` +``ignore::ub_test_reports.pytest_plugin.TestReportsConfigWarning`` silences it. Declaring the properties @@ -108,7 +110,7 @@ plugin this one was ported from: # pytest.ini [pytest] - addopts = -p sphinx_test_reports.pytest_plugin + addopts = -p ub_test_reports.pytest_plugin junit_family = xunit1 test_reports_properties = partially_verifies = PartiallyVerifies, list @@ -120,7 +122,7 @@ plugin this one was ported from: # pyproject.toml [tool.pytest.ini_options] - addopts = "-p sphinx_test_reports.pytest_plugin" + addopts = "-p ub_test_reports.pytest_plugin" junit_family = "xunit1" test_reports_properties = [ "partially_verifies = PartiallyVerifies, list", @@ -160,7 +162,7 @@ With the S-CORE model declared: .. code-block:: python - from sphinx_test_reports.pytest_plugin import add_test_properties + from ub_test_reports.pytest_plugin import add_test_properties @add_test_properties( partially_verifies=["REQ_1", "REQ_2"], @@ -234,7 +236,7 @@ of at the test function: .. code-block:: python - from sphinx_test_reports.pytest_plugin import apply_test_metadata + from ub_test_reports.pytest_plugin import apply_test_metadata @pytest.mark.parametrize("spec", SPECS) def test_spec(spec, record_property): diff --git a/packages/sphinx-test-reports/pyproject.toml b/packages/sphinx-test-reports/pyproject.toml index d406d84be..d87babd51 100644 --- a/packages/sphinx-test-reports/pyproject.toml +++ b/packages/sphinx-test-reports/pyproject.toml @@ -26,30 +26,35 @@ keywords = ["sphinx", "documentation", "test-reports"] # and publishes; the gap shows up only as an install failure for a user on an interpreter # inside it. This package declared a bare `>=3.11` before the import. requires-python = ">=3.11,<4" -dependencies = ["lxml", "ub-project>=1.1.0,<2"] +# HARD dependencies: this package is the Sphinx extension and nothing else. The converter, +# the pytest plugin, the parsers and the `[test_reports]` model are `ub-test-reports`, which +# runs without the documentation toolchain and is the dependency that brings `lxml` and +# `ub-project` along. +# +# TIGHT tracking on both siblings, not a compatibility range: sphinx-needs and +# ub-test-reports are members of this workspace, and `check_workspace.py` check (4) +# requires `>=,<` on every intra-workspace +# edge. uv never validates a specifier it resolves through `workspace = true` (uv#9811), so +# the floor is the only statement of what was actually tested, and `propagate_floors.py` +# moves it at each sibling release -- ub-test-reports' `1.0.0.dev0` becomes `1.0.0` in the +# core's own release pull request. The published 2.0.0 said `sphinx-needs>=6.0.1`; +# narrowing it is what the import costs, and the changelog says so. +dependencies = [ + "sphinx>=7.4", + "docutils>=0.21", + "sphinx-needs>=8.5.0,<9", + "ub-test-reports>=1.0.0.dev0,<2", +] [project.optional-dependencies] -# The Sphinx extension. The `test-reports` command and the pytest plugin run -# without the documentation toolchain, so it is an extra rather than a -# dependency; the lazy -# `setup` in `__init__.py` enforces these floors when the extension is loaded, -# since an extra is opt-in and pip never sees them otherwise. -# -# TIGHT tracking on the sibling, not a compatibility range: sphinx-needs is a member of -# this workspace, and `check_workspace.py` check (4) requires -# `>=,<` on every intra-workspace edge -- -# including one declared in an extra. uv never validates a specifier it resolves through -# `workspace = true` (uv#9811), so the floor is the only statement of what was actually -# tested, and `propagate_floors.py` moves it at each sphinx-needs release. The published -# 2.0.0 said `>=6.0.1`; narrowing it is what the import costs, and the changelog says so. -sphinx = ["sphinx>=7.4", "docutils>=0.21", "sphinx-needs>=8.5.0,<9"] -# The pytest plugin (sphinx_test_reports.pytest_plugin); pytest is not -# a dependency of the extension or the converter. 7.0 is where everything the -# plugin uses exists (pytest.Config/Item/Mark, Config.stash, -# issue_config_time_warning). pytest before 7.3.2 does not run on Python 3.12 -# at all -- that floor is pytest's own, not this one's; the `toolchain-free` CI job runs -# the plugin's tests with no documentation toolchain installed at all. -pytest = ["pytest>=7.0"] +# Accepted and ignored until 4.0: 2.0.0 documented `pip install "sphinx-test-reports[sphinx]"` +# when the toolchain was an extra, and an install line that names a missing extra warns +# under both pip and uv. The toolchain is a dependency now +sphinx = [] +# A pass-through until 4.0: 2.0.0 documented `pip install "sphinx-test-reports[pytest]"` for +# the plugin, which is `ub-test-reports[pytest]` now. Kept so that line still installs +# pytest; check (4) wants the extra's specifier spelled with the same tight floor +pytest = ["ub-test-reports[pytest]>=1.0.0.dev0,<2"] # There is no `test` extra any more: the workspace's test tooling is the ROOT's shared # `test` dependency group, which every cell and the release compat cell install, and # `check_workspace.py` check (7) refuses a `[dependency-groups]` table here. `nox` and @@ -76,9 +81,6 @@ Repository = "https://github.com/useblocks/sphinx-needs" Issues = "https://github.com/useblocks/sphinx-needs/issues" Changelog = "https://github.com/useblocks/sphinx-needs/blob/master/packages/sphinx-test-reports/docs/changelog.rst" -[project.scripts] -test-reports = "sphinx_test_reports.cli:main" - # hatchling rather than flit, for this member alone. The wheel ships TWO top-level # packages: `sphinx_test_reports`, and `sphinxcontrib/test_reports/`, the aliases that keep # the pre-3.0 import name working until 4.0. flit ships exactly one module and silently @@ -86,7 +88,8 @@ test-reports = "sphinx_test_reports.cli:main" # extension would break on the first release. The `toolchain-free` CI job builds the sdist # and the wheel the way the release does (the wheel FROM the sdist) and fails when either # is missing a tracked file under `src/`, because an editable install reads `src/` and would -# keep every test green over a broken release. +# keep every test green over a broken release. (The job is named for ub-test-reports, whose +# suite it runs without Sphinx; this fence only reads archives, so it needs no toolchain.) [build-system] requires = ["hatchling >=1.27,<2"] build-backend = "hatchling.build" @@ -108,6 +111,5 @@ only-include = ["src/sphinx_test_reports", "src/sphinxcontrib"] # `check_workspace.Member.module`, `import_check.py` and `release.yaml` all derive it. # Everything else -- ruff, ty, pytest and the dependency groups -- is the ROOT's, and # `check_workspace.py` check (7) refuses those tables here: a member `[tool.ruff]` does not -# extend the root's configuration, it replaces it. The `toolchain` pytest marker this -# package declared moved to the root's `markers` list with the rest of the pytest -# configuration; `[tool.mypy]` went because the workspace type-checks with ty. +# extend the root's configuration, it replaces it. `[tool.mypy]` went because the workspace +# type-checks with ty. diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/__init__.py b/packages/sphinx-test-reports/src/sphinx_test_reports/__init__.py index b6fcb44f8..c3004ff79 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/__init__.py +++ b/packages/sphinx-test-reports/src/sphinx_test_reports/__init__.py @@ -1,18 +1,10 @@ -"""Sphinx-Test-Reports. +"""Sphinx-Test-Reports, the Sphinx extension. -``setup`` is resolved lazily (PEP 562) so that importing a submodule of this -package does not import Sphinx: the ``test-reports`` command and -:mod:`sphinx_test_reports.projectconfig` are used where the -documentation toolchain is not installed -- it is the ``sphinx`` extra of the -package, not a dependency -- and every import of a submodule runs this file -first. Sphinx still finds ``setup`` through normal attribute access when it -loads this package as an extension. - -Resolving ``setup`` is also where the toolchain is checked. An extra is opt-in, -so a project that installs the bare package into an environment already holding -an older Sphinx or sphinx-needs never shows pip the extra's version floors; -:mod:`sphinx_test_reports.toolchain` enforces them here instead, with -the install line in the message. +The converter, the pytest plugin, the parsers and the ``[test_reports]`` model are +``ub-test-reports`` (:mod:`ub_test_reports`), which this package depends on; what is here +is the extension that turns the same reports into needs inside a Sphinx build. Its +dependencies -- Sphinx, docutils, sphinx-needs and the core -- are hard requirements, so +``setup`` is imported eagerly. """ __all__ = ["__version__", "setup"] @@ -21,50 +13,6 @@ #: ``poe bump``; the extension's metadata reads it from here. __version__ = "2.0.0" - -def __getattr__(name: str) -> object: - if name != "setup": - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - from sphinx_test_reports.toolchain import INSTALL_HINT, unmet_requirements - - unmet = unmet_requirements() - if unmet: - # Before the import: an outdated sphinx-needs may well import and fail - # only later, inside a directive, with a traceback that does not say - # why. Sphinx fetches `setup` with getattr(), which only tolerates - # AttributeError, so the error reaches the user as it is raised here. - message = ( - f"Could not load extension {__name__}: {'; '.join(unmet)}. " - f"Install the Sphinx extension's dependencies with: {INSTALL_HINT}" - ) - try: - from sphinx.errors import ExtensionError - except ImportError: - raise ImportError(message) from None - raise ExtensionError(message) - - try: - from sphinx_test_reports.test_reports import setup - except ImportError as error: - # Sphinx wraps an ImportError from importing the *package* in a clean - # "Could not import extension" message, but fetches `setup` with - # getattr(), which only tolerates AttributeError. Resolving lazily - # would let a missing sphinx-needs escape as a raw traceback, so the - # message Sphinx would have produced is raised here instead -- when - # Sphinx is there to receive it -- naming the extra that installs the - # toolchain, the likely cause. The wrapped exception goes in the second - # argument only: ExtensionError.__str__ renders it as - # "(exception: ...)", so spelling it out in the message too would print - # it twice. - try: - from sphinx.errors import ExtensionError - except ImportError: - raise error from None - raise ExtensionError( - f"Could not import extension {__name__}; the Sphinx extension's " - f"dependencies are an extra, install them with: {INSTALL_HINT}", - error, - ) from error - - return setup +# The order is load-bearing: `test_reports` imports `__version__` from this package while +# this import runs, so the literal above has to be assigned before it. +from sphinx_test_reports.test_reports import setup diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_case.py b/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_case.py index 7bd4eff35..fd86db67d 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_case.py +++ b/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_case.py @@ -8,7 +8,7 @@ from sphinx_test_reports.config import DEFAULT_OPTIONS from sphinx_test_reports.directives.test_common import TestCommonDirective from sphinx_test_reports.exceptions import TestReportInvalidOptionError -from sphinx_test_reports.identity import split_case_name +from ub_test_reports.identity import split_case_name class TestCase(nodes.General, nodes.Element): diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_common.py b/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_common.py index a1dbc0e5e..f97130e06 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_common.py +++ b/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_common.py @@ -20,9 +20,9 @@ SphinxError, TestReportFileNotSetError, ) -from sphinx_test_reports.identity import deterministic_case_id -from sphinx_test_reports.jsonparser import JsonParser -from sphinx_test_reports.junitparser import JUnitParser +from ub_test_reports.identity import deterministic_case_id +from ub_test_reports.jsonparser import JsonParser +from ub_test_reports.junitparser import JUnitParser # fmt: on diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_results.py b/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_results.py index 8280a58ef..6ed5ee975 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_results.py +++ b/packages/sphinx-test-reports/src/sphinx_test_reports/directives/test_results.py @@ -3,7 +3,7 @@ from docutils import nodes from docutils.parsers.rst import Directive -from sphinx_test_reports.junitparser import JUnitParser +from ub_test_reports.junitparser import JUnitParser class TestResults(nodes.General, nodes.Element): diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/test_reports.py b/packages/sphinx-test-reports/src/sphinx_test_reports/test_reports.py index f9ea5bdf8..dc1f1410c 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/test_reports.py +++ b/packages/sphinx-test-reports/src/sphinx_test_reports/test_reports.py @@ -33,14 +33,14 @@ ) from sphinx_test_reports.environment import install_styles_static_files from sphinx_test_reports.exceptions import InvalidConfigurationError -from sphinx_test_reports.fields import ( +from sphinx_test_reports.functions import tr_link +from ub_test_reports.fields import ( FIELDS, RENAMEABLE_FIELDS, RESERVED_NAMES, declaration, ) -from sphinx_test_reports.functions import tr_link -from sphinx_test_reports.projectconfig import ( +from ub_test_reports.projectconfig import ( BRIDGE_KEYS, DEFAULT_FIELD_NAMES, DEFAULT_TOML_FILENAME, @@ -101,6 +101,7 @@ def setup(app: Sphinx) -> dict[str, object]: * test_env * test_report """ + app.require_sphinx((7, 4)) # Name of the need field carrying the path of the XML *report*. app.add_config_value("tr_file_option", DEFAULT_FIELD_NAMES["file_option"], "html") diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/toolchain.py b/packages/sphinx-test-reports/src/sphinx_test_reports/toolchain.py deleted file mode 100644 index e11f59baa..000000000 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/toolchain.py +++ /dev/null @@ -1,77 +0,0 @@ -"""The documentation toolchain the Sphinx extension needs, checked at load time. - -Sphinx and sphinx-needs are the ``sphinx`` extra of this package rather than -dependencies: the ``test-reports`` command runs where they are not installed. -An extra is opt-in, so nothing shows pip the extra's version floors when a -project installs the bare package into an environment that already holds an -older toolchain -- the old versions stay, and the extension would fail later, -inside a directive, with a traceback that does not say why. - -:func:`unmet_requirements` compares the installed toolchain against the floors -the extra declares, read from this package's own metadata so that they live in -``pyproject.toml`` alone. The lazy ``setup`` in the package ``__init__`` calls -it before the extension is imported and raises Sphinx's ``ExtensionError`` -with the install line. -""" - -from __future__ import annotations - -from importlib.metadata import PackageNotFoundError, requires, version -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from packaging.requirements import Requirement - -#: The distribution whose metadata declares the toolchain. -DISTRIBUTION = "sphinx-test-reports" -#: The extra holding the Sphinx extension's dependencies. -EXTRA = "sphinx" -#: The install line named by every message about a missing or outdated toolchain. -INSTALL_HINT = f'pip install "{DISTRIBUTION}[{EXTRA}]"' - - -def toolchain_requirements() -> list[Requirement]: - """The requirements the extra declares, read from the installed metadata. - - Empty without metadata (the package imported from a source tree on - ``sys.path``) and without ``packaging``, which parses them: it is not a - dependency of this package but arrives with Sphinx (and with pytest), so it - is there wherever the extension is loaded. - """ - try: - from packaging.requirements import Requirement - except ImportError: - return [] - try: - declared = requires(DISTRIBUTION) or [] - except PackageNotFoundError: - return [] - requirements = [Requirement(spec) for spec in declared] - return [ - requirement - for requirement in requirements - if requirement.marker is not None - and requirement.marker.evaluate({"extra": EXTRA}) - ] - - -def unmet_requirements() -> list[str]: - """One sentence per requirement of the extra the environment falls short of. - - A requirement whose distribution is not installed at all is not reported: - the import that fails on it states the problem more precisely, and a - toolchain importable from a source tree without metadata must not be - refused on the strength of the metadata alone. - """ - unmet: list[str] = [] - for requirement in toolchain_requirements(): - try: - installed = version(requirement.name) - except PackageNotFoundError: - continue - if not requirement.specifier.contains(installed, prereleases=True): - unmet.append( - f"{requirement.name} {installed} is installed, " - f"but {requirement.name}{requirement.specifier} is required" - ) - return unmet diff --git a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/__init__.py b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/__init__.py index 813a816a9..34f264df4 100644 --- a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/__init__.py +++ b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/__init__.py @@ -1,15 +1,17 @@ """The pre-3.0 name of sphinx-test-reports, kept working until 4.0. -The package is :mod:`sphinx_test_reports` now. Four old names still resolve, each -with a warning that names the new one: +The extension is :mod:`sphinx_test_reports` now, and the parsers and the pytest plugin are +:mod:`ub_test_reports`, the ``ub-test-reports`` distribution it depends on. Four old names +still resolve, each with a warning that names the new one: * this package, as a Sphinx extension (``extensions = ["sphinxcontrib.test_reports"]``), which warns through Sphinx's logger, type ``test_reports.deprecated`` -- the channel a documentation build shows, fails under ``-W`` and silences with ``suppress_warnings``; * ``pytest_plugin``, ``junitparser`` and ``jsonparser``, one file each next to this - one, which put the REAL module into :data:`sys.modules` under the old name and raise - one :class:`FutureWarning` per process (not a :class:`DeprecationWarning`, which - Python's default filters hide outside ``__main__``). It is attributed to the + one, which put the REAL module (``ub_test_reports.``) into :data:`sys.modules` + under the old name and raise one :class:`FutureWarning` per process (not a + :class:`DeprecationWarning`, which Python's default filters hide outside + ``__main__``). It is attributed to the ``import`` statement that names the module; ``importlib.import_module``, and pytest when it loads a ``-p`` or ``pytest_plugins`` name, are frames of their own and take the attribution instead. @@ -18,9 +20,9 @@ error: there is deliberately no finder here that would alias the rest. ``setup`` is resolved lazily (PEP 562), so importing one of the module aliases does -not import Sphinx: ``junitparser`` and ``pytest_plugin`` are used where the -documentation toolchain is not installed, and every import of a submodule runs this -file first. +not import Sphinx -- every import of a submodule runs this file first, and a test run +that loads ``-p sphinxcontrib.test_reports.pytest_plugin`` has no use for a Sphinx +import. """ from __future__ import annotations @@ -52,10 +54,10 @@ def _setup(app: Sphinx) -> dict[str, Any]: type="test_reports", subtype="deprecated", ) - # After the warning, so a toolchain error from the real `setup` has the deprecation - # line above it. Through Sphinx rather than by calling the real `setup`: a conf.py - # that lists both names then registers the extension once, and the real package's - # own lazy `setup` -- with its toolchain check -- is what runs. + # After the warning, so an error from the real `setup` has the deprecation line above + # it. Through Sphinx rather than by calling the real `setup`: a conf.py that lists both + # names then registers the extension once, and the real package's own `setup` is what + # runs. app.setup_extension(_NEW_NAME) extension = app.extensions[_NEW_NAME] # `Extension` pops these three out of the metadata it keeps; hand back all of it. diff --git a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/jsonparser.py b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/jsonparser.py index fde372f3f..c2ed4ebd4 100644 --- a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/jsonparser.py +++ b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/jsonparser.py @@ -1,4 +1,4 @@ -"""Deprecated: the old name of :mod:`sphinx_test_reports.jsonparser`, removed in 4.0. +"""Deprecated: the old name of :mod:`ub_test_reports.jsonparser`, removed in 4.0. Importing it gives the real module -- the same object, so classes and ``mock.patch`` targets written against the old path keep working, and warning filters naming it keep @@ -12,10 +12,10 @@ import sys import warnings -from sphinx_test_reports import jsonparser as _module +from ub_test_reports import jsonparser as _module warnings.warn( - "sphinxcontrib.test_reports.jsonparser has moved to sphinx_test_reports.jsonparser; " + "sphinxcontrib.test_reports.jsonparser has moved to ub_test_reports.jsonparser; " "import it from there. The old name stops working in sphinx-test-reports 4.0.", FutureWarning, # 2 is the frame that ran the import: `warnings` skips importlib's frozen bootstrap diff --git a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/junitparser.py b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/junitparser.py index 5ed5e4502..a9776e5fe 100644 --- a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/junitparser.py +++ b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/junitparser.py @@ -1,4 +1,4 @@ -"""Deprecated: the old name of :mod:`sphinx_test_reports.junitparser`, removed in 4.0. +"""Deprecated: the old name of :mod:`ub_test_reports.junitparser`, removed in 4.0. Importing it gives the real module -- the same object, so classes and ``mock.patch`` targets written against the old path keep working, and warning filters naming it keep @@ -12,10 +12,10 @@ import sys import warnings -from sphinx_test_reports import junitparser as _module +from ub_test_reports import junitparser as _module warnings.warn( - "sphinxcontrib.test_reports.junitparser has moved to sphinx_test_reports.junitparser; " + "sphinxcontrib.test_reports.junitparser has moved to ub_test_reports.junitparser; " "import it from there. The old name stops working in sphinx-test-reports 4.0.", FutureWarning, # 2 is the frame that ran the import: `warnings` skips importlib's frozen bootstrap diff --git a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/pytest_plugin.py b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/pytest_plugin.py index 2fe691d05..e1fdf1f68 100644 --- a/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/pytest_plugin.py +++ b/packages/sphinx-test-reports/src/sphinxcontrib/test_reports/pytest_plugin.py @@ -1,4 +1,4 @@ -"""Deprecated: the old name of :mod:`sphinx_test_reports.pytest_plugin`, removed in 4.0. +"""Deprecated: the old name of :mod:`ub_test_reports.pytest_plugin`, removed in 4.0. Importing it gives the real module -- the same object, so classes and ``mock.patch`` targets written against the old path keep working, and warning filters naming it keep @@ -12,10 +12,10 @@ import sys import warnings -from sphinx_test_reports import pytest_plugin as _module +from ub_test_reports import pytest_plugin as _module warnings.warn( - "sphinxcontrib.test_reports.pytest_plugin has moved to sphinx_test_reports.pytest_plugin; " + "sphinxcontrib.test_reports.pytest_plugin has moved to ub_test_reports.pytest_plugin; " "import it from there. The old name stops working in sphinx-test-reports 4.0.", FutureWarning, # 2 is the frame that ran the import: `warnings` skips importlib's frozen bootstrap diff --git a/packages/sphinx-test-reports/tests/conftest.py b/packages/sphinx-test-reports/tests/conftest.py index abb52087d..2bac6d86b 100644 --- a/packages/sphinx-test-reports/tests/conftest.py +++ b/packages/sphinx-test-reports/tests/conftest.py @@ -1,21 +1,15 @@ """Pytest conftest module containing common test configuration and fixtures.""" -import importlib.util import shutil from pathlib import Path from tempfile import mkdtemp import pytest -# The documentation toolchain is an extra of the package, and the converter's -# and the pytest plugin's tests run where it is not installed (the -# `toolchain_free` and `plugin_floor` nox sessions), so Sphinx's fixtures are -# loaded only where Sphinx is. Tests that need a build carry the `toolchain` -# mark and are deselected there. `pytester`, which the plugin's tests drive, -# ships with pytest itself. -pytest_plugins = ( - ["sphinx.testing.fixtures"] if importlib.util.find_spec("sphinx") else [] -) + ["pytester"] +# Sphinx is a dependency of this package, so its fixtures are always there. `pytester`, +# which the alias tests drive the old plugin name with, ships with pytest itself. (The +# converter's and the plugin's own suite is ub-test-reports', and runs without Sphinx.) +pytest_plugins = ["sphinx.testing.fixtures", "pytester"] def copy_srcdir_to_tmpdir(srcdir, tmp): diff --git a/packages/sphinx-test-reports/tests/doc_test/utils/xml_data_2.xml b/packages/sphinx-test-reports/tests/doc_test/utils/xml_data_2.xml deleted file mode 100644 index 30edbbc92..000000000 --- a/packages/sphinx-test-reports/tests/doc_test/utils/xml_data_2.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - key "name" could not be accessed in junit object - - - Passed on the last test run - - diff --git a/packages/sphinx-test-reports/tests/test_aliases.py b/packages/sphinx-test-reports/tests/test_aliases.py index a038f0e23..68c99412d 100644 --- a/packages/sphinx-test-reports/tests/test_aliases.py +++ b/packages/sphinx-test-reports/tests/test_aliases.py @@ -3,12 +3,12 @@ ``sphinxcontrib.test_reports`` keeps four names until 4.0: the package itself as a Sphinx extension, which warns through Sphinx's logger as ``[test_reports.deprecated]``, and the ``pytest_plugin``, ``junitparser`` and ``jsonparser`` modules, which ARE the real modules -and raise one ``FutureWarning`` at the importing line. Every other old module name fails -as an ordinary missing module. +-- in ``ub_test_reports``, the core this extension depends on -- and raise one +``FutureWarning`` at the importing line. Every other old module name, of either package, +fails as an ordinary missing module. -The module tests run where the documentation toolchain is not installed (the -``toolchain-free`` CI job runs this file against the BUILT wheel); the builds carry the -``toolchain`` mark. +Neither real package is imported at module level here: the extension's root imports Sphinx +eagerly, and the walk below needs only where the two packages are. """ from __future__ import annotations @@ -19,15 +19,17 @@ import sys import textwrap import warnings +from importlib.util import find_spec from pathlib import Path from unittest import mock import pytest -import sphinx_test_reports - OLD = "sphinxcontrib.test_reports" +#: the extension, which the old EXTENSION name loads NEW = "sphinx_test_reports" +#: the core, where the three old MODULE names point +CORE = "ub_test_reports" #: The aliased modules, each with one public name to check identity and patching by. ALIASES = { @@ -36,20 +38,33 @@ "jsonparser": "JsonParser", } -#: Every other module of the real package, walked rather than listed, so a module added -#: later is covered without anyone remembering this file. Read off the files rather than + +def _root(package: str) -> Path: + """The directory of *package*, found without importing it.""" + spec = find_spec(package) + assert spec is not None, package + assert spec.origin is not None, package + return Path(spec.origin).parent + + +#: Every other module of BOTH real packages -- the extension and the core -- walked rather +#: than listed, so a module added later to either is covered without anyone remembering +#: this file: under the old name each one must fail plainly. Read off the files rather than #: through `pkgutil`, which imports each subpackage -- and `directives` imports Sphinx. -_ROOT = Path(sphinx_test_reports.__file__).parent +_ROOTS = [_root(NEW), _root(CORE)] UNALIASED = sorted( - name - for name in ( - ".".join(path.relative_to(_ROOT).with_suffix("").parts).removesuffix( - ".__init__" + { + name + for root in _ROOTS + for name in ( + ".".join(path.relative_to(root).with_suffix("").parts).removesuffix( + ".__init__" + ) + for path in root.rglob("*.py") + if path.name != "__init__.py" or path.parent != root ) - for path in _ROOT.rglob("*.py") - if path.name != "__init__.py" or path.parent != _ROOT - ) - if name not in ALIASES + if name not in ALIASES + } ) @@ -84,16 +99,17 @@ def _restore_old_names(): def test_every_real_module_is_either_aliased_or_walked() -> None: - # the fence's fence: an empty walk would make the unaliased tests pass vacuously - assert "identity" in UNALIASED - assert "directives.test_case" in UNALIASED + # the fence's fence: an empty walk would make the unaliased tests pass vacuously, and + # a walk of one root would leave the other package's old names unchecked + assert "identity" in UNALIASED # the core's + assert "directives.test_case" in UNALIASED # the extension's assert set(ALIASES).isdisjoint(UNALIASED) @pytest.mark.parametrize("module", sorted(ALIASES)) def test_the_alias_is_the_real_module(module: str) -> None: imported, _ = _import_old(module) - real = importlib.import_module(f"{NEW}.{module}") + real = importlib.import_module(f"{CORE}.{module}") assert imported is real assert sys.modules[f"{OLD}.{module}"] is real assert getattr(sys.modules[OLD], module) is real @@ -108,7 +124,7 @@ def test_the_alias_warns_once_at_the_importing_line(module: str) -> None: (warning,) = caught assert (warning.filename, warning.lineno) == ("", 1) message = str(warning.message) - assert f"{OLD}.{module} has moved to {NEW}.{module}" in message + assert f"{OLD}.{module} has moved to {CORE}.{module}" in message assert "4.0" in message @@ -124,17 +140,17 @@ def test_a_second_import_does_not_warn_again(module: str) -> None: @pytest.mark.parametrize("module", sorted(ALIASES)) def test_the_real_module_keeps_its_own_spec(module: str) -> None: _import_old(module) - real = importlib.import_module(f"{NEW}.{module}") - assert real.__name__ == f"{NEW}.{module}" + real = importlib.import_module(f"{CORE}.{module}") + assert real.__name__ == f"{CORE}.{module}" assert real.__spec__ is not None - assert real.__spec__.name == f"{NEW}.{module}" + assert real.__spec__.name == f"{CORE}.{module}" @pytest.mark.parametrize("module", sorted(ALIASES)) def test_patching_through_the_old_path_reaches_the_real_module(module: str) -> None: _import_old(module) name = ALIASES[module] - real = importlib.import_module(f"{NEW}.{module}") + real = importlib.import_module(f"{CORE}.{module}") with warnings.catch_warnings(): warnings.simplefilter("ignore", FutureWarning) with mock.patch(f"{OLD}.{module}.{name}") as patched: @@ -152,6 +168,7 @@ def test_an_unaliased_old_name_fails_plainly(module: str) -> None: assert excinfo.value.name is not None assert excinfo.value.name.startswith(OLD) assert NEW not in str(excinfo.value) + assert CORE not in str(excinfo.value) def _run_user_code(tmp_path: Path, source: str) -> subprocess.CompletedProcess[str]: @@ -183,7 +200,10 @@ def test_the_warning_is_visible_under_default_filters(tmp_path, module: str) -> assert f"user_code.py:1: FutureWarning: {OLD}.{module} has moved" in result.stderr -def test_the_aliases_import_without_sphinx(tmp_path) -> None: +def test_the_aliases_import_without_loading_sphinx(tmp_path) -> None: + # This suite always has Sphinx installed, so what it proves is that the alias chain + # does not LOAD it: `-p sphinxcontrib.test_reports.pytest_plugin` must not pull the + # toolchain into every pytest run, which is what the lazy package root is for. result = _run_user_code( tmp_path, f"""\ @@ -191,6 +211,7 @@ def test_the_aliases_import_without_sphinx(tmp_path) -> None: import {OLD} import {OLD}.junitparser import {OLD}.jsonparser + import {OLD}.pytest_plugin loaded = sorted(m for m in ("sphinx", "sphinx_needs", "docutils") if m in sys.modules) assert not loaded, loaded """, @@ -211,7 +232,7 @@ class TestPytestPlugin: OLD_PLUGIN = f"{OLD}.pytest_plugin" SOURCE = """ -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @add_test_properties(test_type="unit") def test_decorated(): @@ -234,7 +255,7 @@ def test_the_old_name_loads_the_plugin_and_warns(self, pytester) -> None: result, report = self._run(pytester, self.OLD_PLUGIN) result.assert_outcomes(passed=1) result.stderr.fnmatch_lines( - [f"*FutureWarning: {self.OLD_PLUGIN} has moved to {NEW}.pytest_plugin*"] + [f"*FutureWarning: {self.OLD_PLUGIN} has moved to {CORE}.pytest_plugin*"] ) xml = report.read_text(encoding="utf-8") # the plugin's hooks ran: the property is written, and written once @@ -259,13 +280,13 @@ def test_the_old_filter_line_matches_the_real_class(self, pytester) -> None: assert result.ret == pytest.ExitCode.USAGE_ERROR def test_both_names_fail_loudly_naming_both(self, pytester) -> None: - result, report = self._run(pytester, f"{NEW}.pytest_plugin", self.OLD_PLUGIN) + result, report = self._run(pytester, f"{CORE}.pytest_plugin", self.OLD_PLUGIN) assert result.ret != 0 assert not report.exists() result.stderr.fnmatch_lines( [ "*Plugin already registered under a different name: " - f"{self.OLD_PLUGIN}= None: # fenced here and documented, not worked around. result, report = self._run( pytester, - f"{NEW}.pytest_plugin", + f"{CORE}.pytest_plugin", family="xunit2", ini=f"filterwarnings =\n error\n ignore::{self.OLD_PLUGIN}.TestReportsConfigWarning\n", ) @@ -286,7 +307,7 @@ def test_the_old_filter_line_must_move_with_the_p_line(self, pytester) -> None: result.stderr.fnmatch_lines( [ f"*ignore::{self.OLD_PLUGIN}.TestReportsConfigWarning*", - f"*FutureWarning: {self.OLD_PLUGIN} has moved to {NEW}.pytest_plugin*", + f"*FutureWarning: {self.OLD_PLUGIN} has moved to {CORE}.pytest_plugin*", ] ) @@ -323,7 +344,6 @@ def _needs(app) -> dict: return dict(SphinxNeedsData(app.env).get_needs_view()) -@pytest.mark.toolchain class TestExtensionAlias: """``extensions = ["sphinxcontrib.test_reports"]`` builds, and says so.""" @@ -340,7 +360,9 @@ def test_the_old_name_builds_with_the_deprecation_warning(self, test_app) -> Non # the real extension is registered, and its metadata is what the alias reports real = app.extensions[NEW] alias = app.extensions[OLD] - assert alias.version == real.version == sphinx_test_reports.__version__ + from sphinx_test_reports import __version__ + + assert alias.version == real.version == __version__ assert alias.parallel_read_safe == real.parallel_read_safe @pytest.mark.parametrize( @@ -408,29 +430,6 @@ def test_a_strict_build_fails_on_it_and_suppress_warnings_clears_it( assert passed.returncode == 0, passed.stderr -@pytest.mark.toolchain -def test_the_deprecation_comes_before_a_toolchain_error( - make_app, tmp_path, monkeypatch -) -> None: - """A project that names the old extension with an outdated toolchain sees the - deprecation first, then the real extension's toolchain error, which names the new one. - """ - import io - - from sphinx.errors import ExtensionError - - from sphinx_test_reports import toolchain - - monkeypatch.setattr( - toolchain, "unmet_requirements", lambda: ["sphinx-needs 5.1.0 < 8.5.0"] - ) - warning = io.StringIO() - with pytest.raises(ExtensionError, match=f"Could not load extension {NEW}"): - _make_old_project(make_app, tmp_path, {}, warning=warning) - # the error aborted the application, so the warning stream holds only what came first - _assert_the_deprecation_once(warning.getvalue()) - - def _make_old_project(make_app, tmp_path: Path, confoverrides: dict, **kwargs): import shutil diff --git a/packages/sphinx-test-reports/tests/test_cli_convert.py b/packages/sphinx-test-reports/tests/test_cli_convert.py index ba471dba1..09ccd645d 100644 --- a/packages/sphinx-test-reports/tests/test_cli_convert.py +++ b/packages/sphinx-test-reports/tests/test_cli_convert.py @@ -1,31 +1,22 @@ -"""Tests for the Sphinx-free ``test-reports build needs`` CLI (TR-A). +"""The ``test-reports build needs`` CLI's output, inside Sphinx (TR-A). -This is the keystone of the build-system story: a test-XML to needs.json -conversion that runs as a build action *outside* Sphinx, so the docs build only -imports the result. Two properties are load-bearing and therefore tested -explicitly rather than assumed: - -* the CLI must not import Sphinx -- otherwise a Bazel action pulls the whole - documentation toolchain into the test-result conversion; -* the output must be byte-stable, because it is a cached build artifact and - qualification evidence. +This is the half that needs Sphinx: the converter's output checked against sphinx-needs' +schema and imported into a build. The converter's own tests are ub-test-reports', +``packages/ub-test-reports/tests/test_cli_convert.py``. """ import json -import subprocess -import sys from pathlib import Path import pytest UTILS = Path(__file__).parent / "doc_test" / "utils" GTEST_XML = UTILS / "gtest_data.xml" -PYTEST_XML = UTILS / "pytest_data.xml" def _convert(tmp_path, *args, xml=GTEST_XML): """Run the converter and return (exit_code, parsed output or None).""" - from sphinx_test_reports.cli import main + from ub_test_reports.cli import main output = tmp_path / "needs.json" code = main(["build", "needs", str(xml), "--output", str(output), *args]) @@ -46,29 +37,7 @@ def _field_type(declaration): return next(iter(set(kind) - {"null"})) -class TestNoSphinxImport: - """The converter has to be usable without the documentation toolchain.""" - - def test_importing_the_cli_does_not_import_sphinx(self): - script = ( - "import sys;" - "import sphinx_test_reports.cli;" - "leaked = sorted(m for m in sys.modules" - " if m == 'sphinx' or m.startswith(('sphinx.', 'sphinx_needs')));" - "print(','.join(leaked))" - ) - result = subprocess.run( - [sys.executable, "-c", script], - capture_output=True, - text=True, - check=True, - ) - - assert result.stdout.strip() == "" - - class TestEnvelope: - @pytest.mark.toolchain def test_output_passes_the_sphinx_needs_schema(self, tmp_path): """The output must validate against sphinx-needs' own needs.json schema.""" from sphinx_needs import needsfile @@ -78,546 +47,7 @@ def test_output_passes_the_sphinx_needs_schema(self, tmp_path): assert code == 0 assert needsfile.check_needs_data(data).schema == [] - def test_envelope_carries_project_and_current_version(self, tmp_path): - _, data = _convert(tmp_path, "--project", "Score Docs-as-Code") - - assert data["project"] == "Score Docs-as-Code" - assert data["current_version"] in data["versions"] - - def test_needs_amount_matches_the_number_of_cases(self, tmp_path): - _, data = _convert(tmp_path) - - version = data["versions"][data["current_version"]] - assert version["needs_amount"] == len(version["needs"]) == 5 - - def test_every_written_field_is_declared(self, tmp_path): - """The file says what its fields are, without a Sphinx build to ask.""" - _, data = _convert(tmp_path) - - version = data["versions"][data["current_version"]] - declared = set(version["needs_schema"]["properties"]) - written = {key for need in version["needs"].values() for key in need} - - assert written - declared == set() - - def test_no_timestamp_is_written(self, tmp_path): - """A wall clock would defeat action caching and evidence diffs.""" - _, data = _convert(tmp_path) - - assert "created" not in data - assert "created" not in data["versions"][data["current_version"]] - - def test_output_is_byte_stable_across_runs(self, tmp_path): - from sphinx_test_reports.cli import main - - first = tmp_path / "first.json" - second = tmp_path / "second.json" - for output in (first, second): - assert ( - main(["build", "needs", str(GTEST_XML), "--output", str(output)]) == 0 - ) - - assert first.read_bytes() == second.read_bytes() - - -class TestNeedContent: - def test_ids_match_the_deterministic_scheme(self, tmp_path): - from sphinx_test_reports.identity import deterministic_case_id - - _, data = _convert(tmp_path) - - expected = deterministic_case_id( - classname="MathTest", name="Addition", file="src/math_test.cc" - ) - assert expected in _needs(data) - - def test_source_location_is_emitted_verbatim(self, tmp_path): - _, data = _convert(tmp_path) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["case_file"] == "src/math_test.cc" - assert need["case_line"] == "12" - # `file` is the report path, as in a locally created test-case need. - assert need["file"] == str(GTEST_XML) - - def test_type_and_title_match_the_directive_s(self, tmp_path): - # The directive titles a case need with the case name; a needtable - # title column must not tell an imported case from a built one. - _, data = _convert(tmp_path) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["type"] == "testcase" - assert need["title"] == "Addition" - - def test_result_vocabulary_includes_disabled(self, tmp_path): - _, data = _convert(tmp_path) - - need = _needs(data)["testcase__MathTest__DISABLED_Division_jnyzp"] - assert need["result"] == "disabled" - - def test_content_keeps_every_failure_part(self, tmp_path): - """R2: the debug output has to survive the conversion.""" - _, data = _convert(tmp_path) - - content = _needs(data)["testcase__MathTest__Subtraction_srmht"]["content"] - assert "Expected equality of these values" in content - assert "Actual: false" in content - assert "overflow guard hit" in content - - def test_result_text_is_the_first_failure_message(self, tmp_path): - _, data = _convert(tmp_path) - - need = _needs(data)["testcase__MathTest__Subtraction_srmht"] - assert need["result_text"].startswith("src/math_test.cc:22") - assert "\n" not in need["result_text"] - - def test_named_properties_become_fields(self, tmp_path, capsys): - # Only properties the build would accept (extra_options, or here the - # flag standing in for it) become fields; the rest is reported once. - code, data = _convert(tmp_path, "--no-config", "--extra-option", "TestType") - assert code == 0 - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["TestType"] == "requirements-based" - assert "PartiallyVerifies" not in need - message = capsys.readouterr().err - assert "properties not exported" in message - assert "PartiallyVerifies" in message - assert "extra_options" in message - - def test_unexported_properties_are_reported_once_for_all_names( - self, tmp_path, capsys - ): - # One line naming every left-out property -- not one line per name, - # and not one per case that carries it. - code, _ = _convert(tmp_path, "--no-config") - assert code == 0 - lines = [ - line - for line in capsys.readouterr().err.splitlines() - if "properties not exported" in line - ] - assert len(lines) == 1 - assert "PartiallyVerifies" in lines[0] and "TestType" in lines[0] - - def test_an_exported_property_is_present_on_every_case(self, tmp_path): - # Null where the case has no such property, as the build leaves a - # registered field a directive did not set -- so one schema can - # require the field of imported and locally created needs alike. - _, data = _convert(tmp_path, "--no-config", "--extra-option", "TestType") - needs = _needs(data) - assert needs["testcase__MathTest__Addition_hcuyy"]["TestType"] == ( - "requirements-based" - ) - assert needs["testcase__MathTest__Subtraction_srmht"]["TestType"] is None - assert all("TestType" in need for need in needs.values()) - - def test_unnamed_properties_are_left_out_quietly_when_none_exist( - self, tmp_path, capsys - ): - code, _ = _convert(tmp_path, "--no-config", xml=PYTEST_XML) - assert code == 0 - assert "properties not exported" not in capsys.readouterr().err - - def test_tags_are_configurable(self, tmp_path): - _, data = _convert(tmp_path, "--tags", "TEST") - - assert _needs(data)["testcase__MathTest__Addition_hcuyy"]["tags"] == ["TEST"] - - -class TestLinkProperties: - def test_a_property_can_be_promoted_to_a_link_field(self, tmp_path): - _, data = _convert( - tmp_path, "--link-property", "PartiallyVerifies=partially_verifies" - ) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["partially_verifies"] == ["REQ_1", "REQ_2"] - assert "PartiallyVerifies" not in need - - def test_link_fields_are_always_present_even_when_empty(self, tmp_path): - """A converter must emit its fields unconditionally, so schemas can require them.""" - _, data = _convert( - tmp_path, "--link-property", "PartiallyVerifies=partially_verifies" - ) - - need = _needs(data)["testcase__MathTest__DISABLED_Division_jnyzp"] - assert need["partially_verifies"] == [] - - def test_malformed_link_property_is_rejected(self, tmp_path): - from sphinx_test_reports.cli import main - - code = main( - [ - "build", - "needs", - str(GTEST_XML), - "--output", - str(tmp_path / "out.json"), - "--link-property", - "NoEqualsSign", - ] - ) - - assert code != 0 - - -class TestRemoteUrls: - def test_external_url_and_remote_url_are_synthesized(self, tmp_path): - _, data = _convert( - tmp_path, - "--remote-url", - "https://github.com/org/repo", - "--commit", - "abc123", - ) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - expected = "https://github.com/org/repo/blob/abc123/src/math_test.cc#L12" - assert need["external_url"] == expected - assert need["remote_url"] == expected - - def test_scp_style_remote_is_normalised(self, tmp_path): - _, data = _convert( - tmp_path, - "--remote-url", - "git@github.com:org/repo.git", - "--commit", - "abc123", - ) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["remote_url"].startswith("https://github.com/org/repo/blob/abc123/") - - def test_url_pattern_is_configurable(self, tmp_path): - _, data = _convert( - tmp_path, - "--remote-url", - "https://gitlab.com/org/repo", - "--commit", - "abc123", - "--url-pattern", - "{base}/-/blob/{commit}/{file}#L{line}", - ) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["remote_url"] == ( - "https://gitlab.com/org/repo/-/blob/abc123/src/math_test.cc#L12" - ) - - def test_credentials_in_the_remote_url_are_not_written(self, tmp_path): - # GitLab's CI_REPOSITORY_URL embeds the job token; the base lands in - # every need of a cached artifact and, imported, in published HTML. - from sphinx_test_reports.cli import main - - output = tmp_path / "needs.json" - code = main( - [ - "build", - "needs", - str(GTEST_XML), - "--output", - str(output), - "--no-config", - "--remote-url", - "https://gitlab-ci-token:glcbt-secret@gitlab.example.com/org/repo.git", - "--commit", - "abc123", - ] - ) - assert code == 0 - text = output.read_text(encoding="utf-8") - assert "glcbt-secret" not in text and "gitlab-ci-token" not in text - need = _needs(json.loads(text))["testcase__MathTest__Addition_hcuyy"] - assert need["remote_url"] == ( - "https://gitlab.example.com/org/repo/blob/abc123/src/math_test.cc#L12" - ) - - def test_without_repo_metadata_the_url_fields_are_empty(self, tmp_path): - """A hermetic sandbox has no git remote; that must not drop the need.""" - _, data = _convert(tmp_path) - - need = _needs(data)["testcase__MathTest__Addition_hcuyy"] - assert need["remote_url"] == "" - assert need["external_url"] == "" - - -class TestUrlPatternErrors: - """A bad template is a configuration error at the start, not a traceback.""" - - def test_an_unknown_placeholder_is_an_error(self, tmp_path, capsys): - code, data = _convert( - tmp_path, - "--no-config", - "--remote-url", - "https://github.com/o/r", - "--commit", - "abc", - "--url-pattern", - "{base}/blob/{ref}/{file}#L{line}", - ) - assert code == 2 - assert data is None - message = capsys.readouterr().err - assert "--url-pattern" in message - assert "{ref}" in message - - def test_an_unbalanced_brace_is_an_error(self, tmp_path, capsys): - code, _ = _convert( - tmp_path, "--no-config", "--url-pattern", "{base/blob/{commit}/{file}" - ) - assert code == 2 - assert "malformed" in capsys.readouterr().err - - def test_an_attribute_lookup_is_an_error_not_a_traceback(self, tmp_path, capsys): - # str.format resolves {base.__class__}; only KeyError was caught. - code, data = _convert( - tmp_path, - "--no-config", - "--remote-url", - "https://github.com/o/r", - "--commit", - "abc", - "--url-pattern", - "{base.__class__}/{file}", - ) - assert code == 2 - assert data is None - message = capsys.readouterr().err - assert "unknown placeholder {base.__class__}" in message - assert "Traceback" not in message - - def test_the_pattern_is_checked_before_any_report_is_read(self, tmp_path, capsys): - # A missing report and a bad pattern: the pattern error wins, because - # the template is checked before the first file is opened. - code, data = _convert( - tmp_path, - "--no-config", - "--url-pattern", - "{base}/blob/{ref}/{file}", - xml=tmp_path / "does-not-exist.xml", - ) - assert code == 2 - assert data is None - message = capsys.readouterr().err - assert "{ref}" in message - assert "no such file" not in message - - -class TestMultipleInputs: - def test_several_reports_are_merged_into_one_file(self, tmp_path): - from sphinx_test_reports.cli import main - - output = tmp_path / "needs.json" - code = main( - ["build", "needs", str(GTEST_XML), str(PYTEST_XML), "--output", str(output)] - ) - data = json.loads(output.read_text(encoding="utf-8")) - - assert code == 0 - assert len(_needs(data)) > 5 - - def test_the_same_report_given_twice_is_refused(self, tmp_path, capsys): - # Silently collapsing the repeats would produce a valid file that has - # lost half its evidence -- the worst outcome for a cached artifact. - from sphinx_test_reports.cli import main - - output = tmp_path / "needs.json" - code = main( - [ - "build", - "needs", - str(GTEST_XML), - str(GTEST_XML), - "--no-config", - "-o", - str(output), - ] - ) - assert code == 2 - assert not output.exists() - message = capsys.readouterr().err - assert "more than once" in message - assert "testcase__" in message - - def test_a_missing_input_file_exits_nonzero(self, tmp_path): - from sphinx_test_reports.cli import main - - code = main( - [ - "build", - "needs", - str(tmp_path / "nope.xml"), - "--output", - str(tmp_path / "out.json"), - ] - ) - - assert code != 0 - - -class TestDiagnostics: - def test_absent_line_attributes_warn_about_junit_family(self, tmp_path, capsys): - """pytest's default junit_family drops file/line; say so, don't guess.""" - from sphinx_test_reports.cli import main - - main( - [ - "build", - "needs", - str(PYTEST_XML), - "--output", - str(tmp_path / "out.json"), - ] - ) - - assert "junit_family" in capsys.readouterr().err - - def test_nested_suites_get_the_hint_too(self, tmp_path, capsys): - # The parser files the cases of a nested report under testsuite_nested; - # a hint that only looked at the top level went quiet on exactly the - # Ant/Maven-shaped reports that most often lack source locations. - code, data = _convert( - tmp_path, "--no-config", xml=UTILS / "pytest_nested_example.xml" - ) - assert code == 0 - assert all(need["case_line"] == "" for need in _needs(data).values()) - assert "junit_family" in capsys.readouterr().err - - def test_reports_with_line_attributes_do_not_warn(self, tmp_path, capsys): - from sphinx_test_reports.cli import main - - main(["build", "needs", str(GTEST_XML), "--output", str(tmp_path / "out.json")]) - - assert "junit_family" not in capsys.readouterr().err - - def test_a_report_without_test_cases_warns(self, tmp_path, capsys): - # A pom.xml parses as one empty suite: a valid, empty needs.json with - # exit 0 is the one outcome a cached build action must never get - # silently. - pom = tmp_path / "pom.xml" - pom.write_text( - "4.0.0\n", encoding="utf-8" - ) - code, data = _convert(tmp_path, "--no-config", xml=pom) - assert code == 0 - assert data["versions"][data["current_version"]]["needs_amount"] == 0 - message = capsys.readouterr().err - assert "pom.xml" in message and "no test cases" in message - - def test_a_report_with_test_cases_does_not_get_the_empty_warning( - self, tmp_path, capsys - ): - code, _ = _convert(tmp_path, "--no-config") - assert code == 0 - assert "no test cases" not in capsys.readouterr().err - - -def test_the_cli_is_runnable_as_a_module(): - result = subprocess.run( - [ - sys.executable, - "-m", - "sphinx_test_reports.cli", - "build", - "needs", - "--help", - ], - capture_output=True, - text=True, - ) - - assert result.returncode == 0 - assert "--output" in result.stdout - - -def test_console_script_is_installed(): - """The name that goes into a BUILD file has to be a real entry point.""" - script = Path(sys.executable).parent / "test-reports" - if not script.exists(): - pytest.skip("package not installed into this environment") - - result = subprocess.run( - [str(script), "build", "needs", "--help"], capture_output=True, text=True - ) - - assert result.returncode == 0 - assert "--output" in result.stdout - - -@pytest.mark.parametrize("flag", ["--remote-url", "--commit"]) -def test_url_synthesis_needs_both_parts(tmp_path, flag): - """Half the metadata cannot produce a URL; fail loudly instead of guessing.""" - from sphinx_test_reports.cli import main - - code = main( - [ - "build", - "needs", - str(GTEST_XML), - "--output", - str(tmp_path / "out.json"), - flag, - "value", - ] - ) - - assert code != 0 - - -class TestResultVocabulary: - """The export uses the parser's vocabulary, which is the build's. - - ``failed`` is a documented need field value and a CSS class - (``tr_failed``), and the shipped report template filters on it. A project - that mixes imported and locally created test-case needs filters both with - one expression only if the two writers spell the result alike. - """ - - def test_failure_is_exported_as_the_build_spells_it(self, tmp_path): - _, data = _convert(tmp_path) - - assert ( - _needs(data)["testcase__MathTest__Subtraction_srmht"]["result"] == "failed" - ) - - @pytest.mark.parametrize( - ("need_id", "expected"), - [ - ("testcase__MathTest__Addition_hcuyy", "passed"), - ("testcase__MathTest__DISABLED_Division_jnyzp", "disabled"), - ("testcase__ParamTest_0__Legacy_owuvz", "skipped"), - ], - ) - def test_other_results_are_unchanged(self, tmp_path, need_id, expected): - _, data = _convert(tmp_path) - - assert _needs(data)[need_id]["result"] == expected - - -class TestContentIsNotDuplicated: - """googletest repeats the failure text in the message attribute. - - Emitting both verbatim shows the same stack trace twice in the rendered - need; the message block is only worth its space when it says something the - body does not. - """ - - def test_a_message_contained_in_the_body_is_not_repeated(self, tmp_path): - _, data = _convert(tmp_path) - - content = _needs(data)["testcase__MathTest__Subtraction_srmht"]["content"] - assert content.count("Expected equality of these values") == 1 - assert "message" not in content - - def test_a_message_absent_from_the_body_is_kept(self, tmp_path): - _, data = _convert(tmp_path) - - content = _needs(data)["testcase__ParamTest_0__Legacy_owuvz"]["content"] - assert "Skipped via GTEST_SKIP" in content - assert "not applicable on this platform" in content - -@pytest.mark.toolchain class TestImportIntoABuild: """The documented consumption path: ``needimport`` of the produced file. diff --git a/packages/sphinx-test-reports/tests/test_json_parser.py b/packages/sphinx-test-reports/tests/test_json_parser.py index 56e05c746..9c90b8a3b 100644 --- a/packages/sphinx-test-reports/tests/test_json_parser.py +++ b/packages/sphinx-test-reports/tests/test_json_parser.py @@ -8,14 +8,14 @@ def test_init_json_parser(): - from sphinx_test_reports.jsonparser import JsonParser + from ub_test_reports.jsonparser import JsonParser parser = JsonParser(json_path) assert parser is not None def test_parse_json_data(): - from sphinx_test_reports.jsonparser import JsonParser + from ub_test_reports.jsonparser import JsonParser json_mapping = { "json_config": { diff --git a/packages/sphinx-test-reports/tests/test_project_config.py b/packages/sphinx-test-reports/tests/test_project_config.py index 690694042..0b393a296 100644 --- a/packages/sphinx-test-reports/tests/test_project_config.py +++ b/packages/sphinx-test-reports/tests/test_project_config.py @@ -4,31 +4,22 @@ its keys onto the ``tr_*`` config values. Both must agree on which file describes a project, or the build is configured by something other than what the project declares. + +This is the half that needs Sphinx: the bridge onto the ``tr_*`` values, its precedence +and Sphinx's confval type check. The loader's half is ub-test-reports', +``packages/ub-test-reports/tests/test_project_config.py``. """ -import os -import subprocess -import sys from io import StringIO from pathlib import Path from shutil import copytree import pytest -import ub_project -from sphinx_test_reports.projectconfig import ( - BRIDGE_KEYS, - BUILD_TABLE, - DEFAULT_FIELD_NAMES, +from ub_test_reports.projectconfig import ( DEFAULT_TOML_FILENAME, SECTION, - TomlConfigError, - field_names, - find_project_config, - load_project_config, - needs_settings, ) -from ub_project import ProjectConfigError def _write(tmp_path, toml_source, name=DEFAULT_TOML_FILENAME): @@ -37,448 +28,6 @@ def _write(tmp_path, toml_source, name=DEFAULT_TOML_FILENAME): return config -class TestLoader: - """The Sphinx-free loader: parsing, normalising, anchoring, rejecting.""" - - def test_missing_file_is_none(self, tmp_path): - assert load_project_config(tmp_path / DEFAULT_TOML_FILENAME) is None - - def test_missing_section_is_empty(self, tmp_path): - _write(tmp_path, '[project]\nname = "x"\n') - assert load_project_config(tmp_path / DEFAULT_TOML_FILENAME) == {} - - def test_full_section_round_trips(self, tmp_path): - _write( - tmp_path, - """ - [test_reports] - file_option = "report_file" - source_file_option = "file" - import_encoding = "latin1" - deterministic_case_ids = true - suite_id_length = 4 - extra_options = ["more_info"] - property_link_types = { request = "req" } - """, - ) - config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - assert config["file_option"] == "report_file" - assert config["source_file_option"] == "file" - assert config["import_encoding"] == "latin1" - assert config["deterministic_case_ids"] is True - assert config["suite_id_length"] == 4 - assert config["extra_options"] == ["more_info"] - assert config["property_link_types"] == {"request": "req"} - - def test_build_needs_table_is_validated_but_never_bridged(self, tmp_path): - # [test_reports.build.needs] belongs to the command line. The build - # validates it -- one file, one verdict -- but must not map it onto a - # tr_* value. - _write( - tmp_path, - """ - [test_reports] - file_option = "report_file" - - [test_reports.build.needs] - project = "demo" - tags = ["ci"] - """, - ) - reported = [] - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) - assert reported == [] - assert needs_settings(section) == {"project": "demo", "tags": ["ci"]} - assert BUILD_TABLE not in BRIDGE_KEYS - - @pytest.mark.parametrize( - ("key", "value"), - [ - ("project", "42"), - ("tags", '"ci, unit"'), # a bare string is not an array - ("tags", "[1]"), - ("link_properties", '["a"]'), - ("link_properties", '{ Verifies = ["verifies"] }'), # values too - ], - ) - def test_build_needs_wrong_types_are_rejected(self, tmp_path, key, value): - _write(tmp_path, f"[test_reports.build.needs]\n{key} = {value}\n") - with pytest.raises(TomlConfigError, match=f"build.needs.{key}"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_unknown_artifact_under_build_is_reported_but_not_fatal(self, tmp_path): - # `build` holds one table per artifact the command line produces. A - # newer command may produce one this version does not know, and a file - # naming it must not take the build down. - _write( - tmp_path, - """ - [test_reports.build.needs] - project = "p" - - [test_reports.build.graph] - format = "svg" - """, - ) - reported = [] - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) - assert needs_settings(section) == {"project": "p"} - assert section[BUILD_TABLE] == {"needs": {"project": "p"}} - assert len(reported) == 1 - assert "graph" in reported[0] - assert "[test_reports.build]" in reported[0] - - def test_a_non_table_needs_artifact_is_rejected(self, tmp_path): - _write(tmp_path, '[test_reports.build]\nneeds = "yes"\n') - with pytest.raises(TomlConfigError, match=r"build.needs"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_build_needs_unknown_key_is_reported_but_not_fatal(self, tmp_path): - _write(tmp_path, "[test_reports.build.needs]\nprojct = 'typo'\nproject = 'p'\n") - reported = [] - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) - assert needs_settings(section) == {"project": "p"} - assert len(reported) == 1 - assert "projct" in reported[0] - assert "[test_reports.build.needs]" in reported[0] - - def test_need_type_and_case_type_must_agree(self, tmp_path): - # The converter takes the need type (and the deterministic-ID prefix) - # from build.needs.need_type, the build from case's type. Disagreeing - # produces a needs.json the build neither registers nor cross-links. - _write( - tmp_path, - """ - [test_reports.build.needs] - need_type = "testcase" - - [test_reports.case] - directive = "test-case" - type = "check" - name = "Check" - prefix = "CH_" - color = "#999999" - style = "rectangle" - """, - ) - with pytest.raises(TomlConfigError, match="need_type"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_a_missing_side_is_compared_at_its_default(self, tmp_path): - # A customised case next to a convert table without need_type is a - # disagreement too: the converter would write the default type. - _write( - tmp_path, - """ - [test_reports.build.needs] - project = "p" - - [test_reports.case] - directive = "test-case" - type = "check" - name = "Check" - prefix = "CH_" - color = "#999999" - style = "rectangle" - """, - ) - with pytest.raises(TomlConfigError, match=r"'testcase'.*'check'"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - # ... and the mirror: need_type set, case left at its default. - _write(tmp_path, '[test_reports.build.needs]\nneed_type = "check"\n') - with pytest.raises(TomlConfigError, match=r"'check'.*'testcase'"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_without_a_convert_table_the_case_type_is_free(self, tmp_path): - # A project that only builds may name its case type as it likes. - _write( - tmp_path, - """ - [test_reports.case] - directive = "test-case" - type = "check" - name = "Check" - prefix = "CH_" - color = "#999999" - style = "rectangle" - """, - ) - assert ( - load_project_config(tmp_path / DEFAULT_TOML_FILENAME)["case"][1] == "check" - ) - - def test_empty_link_property_names_are_rejected_by_the_loader(self, tmp_path): - # One verdict for both consumers: the build refuses what the converter - # would refuse. - _write( - tmp_path, - '[test_reports.build.needs]\nlink_properties = { Verifies = "" }\n', - ) - with pytest.raises(TomlConfigError, match="link_properties"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_field_names_default_to_the_build_s(self, tmp_path): - _write(tmp_path, "[test_reports]\nsource_file_option = 'src'\n") - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - names = field_names(section) - assert names["source_file_option"] == "src" - assert names["file_option"] == DEFAULT_FIELD_NAMES["file_option"] == "file" - assert names["source_line_option"] == "case_line" - - def test_colliding_field_names_are_rejected(self, tmp_path): - # file (report path) and file (source path) cannot share a field. - _write(tmp_path, "[test_reports]\nsource_file_option = 'file'\n") - with pytest.raises(TomlConfigError, match="both name the need field 'file'"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - @pytest.mark.parametrize( - ("key", "name"), - [ - ("file_option", "case"), - ("source_file_option", "result"), - ("source_line_option", "id"), - ], - ) - def test_a_rename_onto_a_fixed_field_is_rejected(self, tmp_path, key, name): - # Every test-case need has these already: the directives would pass - # the keyword twice, the converter overwrite one value with the other. - _write(tmp_path, f"[test_reports]\n{key} = '{name}'\n") - with pytest.raises(TomlConfigError, match=f"{key} = '{name}'"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_need_type_and_case_type_agreeing_is_fine(self, tmp_path): - _write( - tmp_path, - """ - [test_reports.build.needs] - need_type = "check" - - [test_reports.case] - directive = "test-case" - type = "check" - name = "Check" - prefix = "CH_" - color = "#999999" - style = "rectangle" - """, - ) - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - assert section["case"][1] == needs_settings(section)["need_type"] == "check" - - def test_unknown_key_is_reported_but_not_fatal(self, tmp_path): - # ubproject.toml is shared with tools on independent release cadences, - # so a key this reader does not model must not take the build down -- - # but a typo has to be visible, and the key must not be passed on. - _write(tmp_path, "[test_reports]\ndeterministic_id = true\n") - reported = [] - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) - assert section == {} - assert len(reported) == 1 - assert "deterministic_id" in reported[0] - assert "deterministic_case_ids" in reported[0] # supported keys listed - - def test_unknown_key_needs_no_reporter(self, tmp_path): - _write(tmp_path, "[test_reports]\nnope = 1\nfile_option = 'f'\n") - assert load_project_config(tmp_path / DEFAULT_TOML_FILENAME) == { - "file_option": "f" - } - - @pytest.mark.parametrize( - ("key", "value"), - [ - ("property_link_types", '{ request = ["req"] }'), - ("property_link_types", "{ request = 3 }"), - ], - ) - def test_table_values_are_type_checked(self, tmp_path, key, value): - # Without this the value reaches the directives, which fail with a bare - # TypeError on an unhashable field name instead of a config error. - _write(tmp_path, f"[test_reports]\n{key} = {value}\n") - with pytest.raises(TomlConfigError, match=key): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_json_mapping_nesting_stays_free_form(self, tmp_path): - # It mirrors an arbitrary parser mapping, so only the outer table is - # checked -- validating deeper would reject valid configurations. - _write( - tmp_path, - "[test_reports.json_mapping.json_config.testsuite]\nname = 1\n", - ) - section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - assert section["json_mapping"] == {"json_config": {"testsuite": {"name": 1}}} - - @pytest.mark.skipif( - hasattr(os, "geteuid") and os.geteuid() == 0, - reason="root reads unreadable files", - ) - @pytest.mark.skipif( - sys.platform == "win32", - reason="os.chmod on Windows only sets the read-only attribute; the file stays readable", - ) - def test_unreadable_file_is_a_config_error(self, tmp_path): - # is_file() succeeding does not mean the open will; an unwrapped - # OSError would surface as a traceback instead of a config error. - config = _write(tmp_path, "[test_reports]\nfile_option = 'f'\n") - config.chmod(0o000) - try: - with pytest.raises(TomlConfigError, match="cannot be read"): - load_project_config(config) - finally: - config.chmod(0o644) - - @pytest.mark.parametrize( - ("key", "value"), - [ - ("file_option", "42"), - ("suite_id_length", '"four"'), # string for int - ("suite_id_length", "true"), # bool must not pass for int - ("deterministic_case_ids", '"yes"'), # string for bool - ("extra_options", '"more_info"'), # a bare string is not an array - ("property_link_types", '["a"]'), - ], - ) - def test_wrong_types_are_rejected(self, tmp_path, key, value): - _write(tmp_path, f"[test_reports]\n{key} = {value}\n") - with pytest.raises(TomlConfigError, match=key): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_invalid_toml_is_rejected(self, tmp_path): - _write(tmp_path, "[test-reports\n") - with pytest.raises(TomlConfigError, match="invalid TOML"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_section_must_be_a_table(self, tmp_path): - _write(tmp_path, "test_reports = 5\n") - with pytest.raises(TomlConfigError, match="must be a table"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_need_type_positional_list_still_works(self, tmp_path): - # The conf.py spelling, so existing projects can copy their lists over - # verbatim. - _write( - tmp_path, - '[test_reports]\ncase = ["test-case", "testcase", "Test-Case", "TC_", "#999999", "rectangle"]\n', - ) - config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - assert config["case"] == [ - "test-case", - "testcase", - "Test-Case", - "TC_", - "#999999", - "rectangle", - ] - - def test_need_type_named_table(self, tmp_path): - # Six bare strings cannot be told apart; the table spelling names them. - _write( - tmp_path, - """ - [test_reports.case] - directive = "test-case" - type = "testcase" - name = "Test-Case" - prefix = "TC_" - color = "#999999" - style = "rectangle" - """, - ) - config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - assert config["case"] == [ - "test-case", - "testcase", - "Test-Case", - "TC_", - "#999999", - "rectangle", - ] - - def test_need_type_table_rejects_partial_and_unknown(self, tmp_path): - _write(tmp_path, '[test_reports.case]\ndirective = "test-case"\n') - with pytest.raises(TomlConfigError, match="missing"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - _write( - tmp_path, - """ - [test_reports.case] - directive = "test-case" - type = "testcase" - name = "Test-Case" - prefix = "TC_" - color = "#999999" - style = "rectangle" - typo = true - """, - ) - with pytest.raises(TomlConfigError, match="unknown typo"): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - @pytest.mark.parametrize( - ("toml_source", "problem"), - [ - # A non-string value inside the named table. Without the check the - # value is silently stringified and reaches sphinx-needs. - ( - """ - [test_reports.case] - directive = "test-case" - type = "testcase" - name = "Test-Case" - prefix = "TC_" - color = 999999 - style = "rectangle" - """, - "non-string color", - ), - # A non-string element of the positional list. - ( - '[test_reports]\ncase = ["test-case", "testcase", "Test-Case", "TC_", 999999, "rectangle"]\n', - "exactly 6 strings", - ), - # Too few elements. - ('[test_reports]\ncase = ["test-case", "testcase"]\n', "exactly 6 strings"), - ], - ) - def test_need_type_values_must_be_strings(self, tmp_path, toml_source, problem): - _write(tmp_path, toml_source) - with pytest.raises(TomlConfigError, match=problem): - load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - - def test_relative_paths_anchor_to_the_toml_directory(self, tmp_path): - # The file is self-describing: moving it as a unit keeps its relative - # paths meaningful, and both consumers resolve them identically. - subdir = tmp_path / "config" - subdir.mkdir() - _write( - subdir, - '[test_reports]\nrootdir = "docs"\nreport_template = "templates/report.txt"\n', - name=subdir / DEFAULT_TOML_FILENAME, - ) - config = load_project_config(subdir / DEFAULT_TOML_FILENAME) - assert config["rootdir"] == str(subdir / "docs") - assert config["report_template"] == str(subdir / "templates" / "report.txt") - - @pytest.mark.parametrize( - "suffix", - [ - pytest.param("", id="plain"), - # the two forms ``Path`` normalises away: a round trip through it - # would return ``/a/b`` for ``/a/b/`` and ``/a/b`` for ``/a//b``, - # so these are the cases that tell "left as the string it was" - # from "anchored, and absolute already" - pytest.param(os.sep, id="trailing-separator"), - pytest.param(f"{os.sep}{os.sep}x", id="doubled-separator"), - ], - ) - def test_absolute_paths_stay_untouched(self, tmp_path, suffix): - # a TOML literal string: in a basic string a Windows path's backslashes - # are escape sequences ("\U" starts a unicode escape) and the file is invalid - value = f"{tmp_path}{suffix}" - _write(tmp_path, f"[test_reports]\nrootdir = '{value}'\n") - config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) - assert config["rootdir"] == value - - def _not_utf8(tmp_path): """A file saved in Latin-1: ``é`` is the lone byte 0xE9, which UTF-8 refuses.""" config = tmp_path / DEFAULT_TOML_FILENAME @@ -486,65 +35,6 @@ def _not_utf8(tmp_path): return config -class TestSharedReaderBoundary: - """ub-project reads the file; its exception never leaves this package. - - Both consumers catch :class:`TomlConfigError` and nothing else, so a - ``ProjectConfigError`` escaping the loader would reach the user as a - traceback. Each case asserts the exact type and that the message is the - shared reader's, word for word. - """ - - def _assert_re_raised(self, config): - with pytest.raises(TomlConfigError) as caught: - load_project_config(config) - assert type(caught.value) is TomlConfigError - cause = caught.value.__cause__ - assert isinstance(cause, ProjectConfigError) - assert str(caught.value) == str(cause) - return str(caught.value) - - def test_the_two_exceptions_are_unrelated(self): - # a subclass either way round would put ub-project's exception on this - # package's public surface - assert not issubclass(TomlConfigError, ProjectConfigError) - assert not issubclass(ProjectConfigError, TomlConfigError) - - def test_the_walk_is_the_shared_reader_s_own(self): - # re-exported, not copied: a local fork of the walk would pass every - # discovery test and drift from the reader the other members use - assert find_project_config is ub_project.find_project_config - - def test_invalid_toml(self, tmp_path): - message = self._assert_re_raised(_write(tmp_path, "[test-reports\n")) - assert message.startswith(f"{tmp_path / DEFAULT_TOML_FILENAME}: invalid TOML: ") - - @pytest.mark.skipif( - hasattr(os, "geteuid") and os.geteuid() == 0, - reason="root reads unreadable files", - ) - @pytest.mark.skipif( - sys.platform == "win32", - reason="os.chmod on Windows only sets the read-only attribute; the file stays readable", - ) - def test_unreadable_file(self, tmp_path): - config = _write(tmp_path, "[test_reports]\nfile_option = 'f'\n") - config.chmod(0o000) - try: - message = self._assert_re_raised(config) - finally: - config.chmod(0o644) - assert message.startswith(f"{config}: cannot be read: ") - - def test_a_file_that_is_not_utf8(self, tmp_path): - # New with ub-project: before it, the decode error escaped the loader - # as a bare UnicodeDecodeError. - config = _not_utf8(tmp_path) - message = self._assert_re_raised(config) - assert message.startswith(f"{config}: not valid UTF-8 TOML: ") - - -@pytest.mark.toolchain class TestSphinxBridge: """The build reads the same section and honours the same precedence.""" @@ -628,184 +118,6 @@ def test_bridge_rejects_a_file_that_is_not_utf8(self, tmp_path): ) -class TestDiscovery: - """The upward search that lets both consumers find the same file. - - The search is bounded by the repository root -- the directory holding - ``.git``: a ``pyproject.toml`` on the way up marks a Python distribution, - not the project, and must not end the search. Outside any repository there - is no such root, so the distribution root bounds it instead -- otherwise - the walk reaches the filesystem root and adopts a stranger's file. - """ - - def test_finds_the_file_in_the_starting_directory(self, tmp_path): - config = _write(tmp_path, "[test_reports]\n") - assert find_project_config(tmp_path) == config - - def test_walks_up_to_the_repository_root(self, tmp_path): - (tmp_path / ".git").mkdir() - config = _write(tmp_path, "[test_reports]\n") - deep = tmp_path / "docs" / "source" - deep.mkdir(parents=True) - assert find_project_config(deep) == config - - def test_a_pyproject_toml_beside_conf_py_does_not_end_the_search(self, tmp_path): - # docs/ carrying its own pyproject.toml (its own dependency set) still - # belongs to the project whose shared file sits at the repository root. - (tmp_path / ".git").mkdir() - config = _write(tmp_path, "[test_reports]\n") - docs = tmp_path / "docs" - docs.mkdir() - (docs / "pyproject.toml").write_text("", encoding="utf-8") - assert find_project_config(docs) == config - - def test_walks_past_a_workspace_member_pyproject_toml(self, tmp_path): - # A uv-workspace member: packages//pyproject.toml with the docs - # below it, and one ubproject.toml at the repository root describing - # the whole monorepo. - (tmp_path / ".git").mkdir() - config = _write(tmp_path, "[test_reports]\n") - member = tmp_path / "packages" / "dist" - docs = member / "docs" - docs.mkdir(parents=True) - (member / "pyproject.toml").write_text("", encoding="utf-8") - assert find_project_config(docs) == config - - def test_stops_at_a_nested_repository_without_the_file(self, tmp_path): - # A checkout nested inside another repository (a vendored tree, a - # submodule) must not adopt the outer repository's configuration. - _write(tmp_path, "[test_reports]\n") - inner = tmp_path / "vendor" / "inner" - docs = inner / "docs" - docs.mkdir(parents=True) - (inner / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") - assert find_project_config(docs) is None - - def test_stops_at_the_distribution_root_without_a_repository(self, tmp_path): - # An unpacked sdist, a CI artefact directory, an exported docs tree: - # no .git anywhere, so nothing above would end the walk and a - # stranger's file further up would be adopted. The distribution root - # bounds the search instead, so it is not. - _write(tmp_path, "[test_reports]\n") # a stranger's, two levels up - dist = tmp_path / "downloads" / "sphinx-test-reports-1.4.0" - docs = dist / "docs" - docs.mkdir(parents=True) - (dist / "pyproject.toml").write_text("", encoding="utf-8") - assert find_project_config(docs) is None - - def test_the_file_at_the_distribution_root_is_still_found(self, tmp_path): - # The distribution root bounds the search without hiding a file that - # sits on it: an sdist shipping its own ubproject.toml is configured - # by it. - dist = tmp_path / "sphinx-test-reports-1.4.0" - docs = dist / "docs" - docs.mkdir(parents=True) - (dist / "pyproject.toml").write_text("", encoding="utf-8") - config = _write(dist, "[test_reports]\n") - assert find_project_config(docs) == config - - def test_a_repository_marker_outranks_a_distribution_root(self, tmp_path): - # The distribution root is only the fallback boundary. Inside a - # repository the walk still passes a pyproject.toml on the way up -- - # the workspace-member layout above depends on it. - (tmp_path / ".git").mkdir() - config = _write(tmp_path, "[test_reports]\n") - member = tmp_path / "packages" / "dist" - docs = member / "docs" - docs.mkdir(parents=True) - (member / "pyproject.toml").write_text("", encoding="utf-8") - assert find_project_config(docs) == config - - def test_a_fruitless_search_reports_the_distribution_root(self, tmp_path): - dist = tmp_path / "sphinx-test-reports-1.4.0" - docs = dist / "docs" - docs.mkdir(parents=True) - (dist / "pyproject.toml").write_text("", encoding="utf-8") - reported = [] - assert find_project_config(docs, report=reported.append) is None - assert len(reported) == 1 - assert f"distribution root {dist}" in reported[0] - assert "pyproject.toml" in reported[0] - - def test_the_file_wins_over_the_marker_in_one_directory(self, tmp_path): - # The root marker only ends a *fruitless* step; a repository root - # holding the file is the canonical layout and must be found. - config = _write(tmp_path, "[test_reports]\n") - (tmp_path / ".git").mkdir() - assert find_project_config(tmp_path) == config - - def test_missing_file_is_none(self, tmp_path): - (tmp_path / ".git").mkdir() - assert find_project_config(tmp_path) is None - - def test_a_fruitless_search_reports_where_it_ended(self, tmp_path): - # "Not found" must not be silent: the report names the directory whose - # marker ended the search, so a misplaced file can be diagnosed. - (tmp_path / ".git").mkdir() - docs = tmp_path / "docs" - docs.mkdir() - reported = [] - assert find_project_config(docs, report=reported.append) is None - assert len(reported) == 1 - assert str(docs) in reported[0] - assert f"repository root {tmp_path}" in reported[0] - assert ".git" in reported[0] - - def test_a_successful_search_reports_nothing(self, tmp_path): - (tmp_path / ".git").mkdir() - _write(tmp_path, "[test_reports]\n") - reported = [] - find_project_config(tmp_path / "docs", report=reported.append) - assert reported == [] - - def test_a_relative_start_is_searched_from_the_working_directory( - self, tmp_path, monkeypatch - ): - # A converter started with a relative path must still see the parents. - (tmp_path / ".git").mkdir() - config = _write(tmp_path, "[test_reports]\n") - docs = tmp_path / "docs" - docs.mkdir() - monkeypatch.chdir(docs) - assert find_project_config(Path(".")) == config - - def test_a_symlinked_start_walks_the_link_s_parents(self, tmp_path): - # The start is made absolute WITHOUT resolving: a symlinked docs/ - # belongs to the repository it is linked into, not to the one its - # target lives in. The only case here that tells the two apart -- - # tmp_path is already resolved, so every other start is too. - repo = tmp_path / "repo" - (repo / ".git").mkdir(parents=True) - config = _write(repo, "[test_reports]\n") - elsewhere = tmp_path / "elsewhere" - (elsewhere / ".git").mkdir(parents=True) - target = elsewhere / "docs" - target.mkdir() - link = repo / "docs" - try: - link.symlink_to(target, target_is_directory=True) - except OSError: # Windows without the symlink privilege - pytest.skip("creating a symlink needs a privilege this account lacks") - assert find_project_config(link) == config - - -class TestPathAnchoring: - """Relative paths anchor at the TOML file's directory, as given.""" - - def test_anchoring_does_not_resolve_the_given_directory(self, tmp_path): - # The loader leaves the form of the directory it was handed alone -- a - # symlinked path stays symlinked. Whether to resolve it is the - # consumer's call (Sphinx resolves its confdir before the bridge runs), - # not something the loader decides behind its back. - real = tmp_path / "real" - real.mkdir() - link = tmp_path / "link" - link.symlink_to(real, target_is_directory=True) - config = _write(link, "[test_reports]\nrootdir = 'reports'\n") - section = load_project_config(config) - assert section["rootdir"] == str(link / "reports") - - def _build(srcdir, **kwargs): """Set up a Sphinx application, returning it with its warning output. @@ -842,7 +154,6 @@ def _basic_doc(tmp_path, toml=None, conf_extra=""): return docs -@pytest.mark.toolchain class TestBridgePrecedence: """``-D`` > TOML > conf.py, and the diagnostics for a file that is missing.""" @@ -950,7 +261,6 @@ def _documented_toml_example(): return "\n".join(block) + "\n" -@pytest.mark.toolchain class TestConfvalTypes: """The bridged values must pass Sphinx's own confval type check. @@ -986,54 +296,3 @@ def test_the_documented_example_applies_without_warnings(self, tmp_path): assert app.config.tr_file_option == "report_file" assert app.config.tr_rootdir == str(tmp_path / "docs") assert app.config.tr_case[0] == "test-case" - - -class TestSphinxFree: - """A consumer without the documentation toolchain can read the section.""" - - def test_projectconfig_imports_without_sphinx(self): - # Importing the module runs the package __init__, so the package must - # not import Sphinx eagerly either -- or a build action that turns - # reports into a needs.json dies with ModuleNotFoundError wherever - # Sphinx is not installed. Checked in a subprocess: this process has - # Sphinx imported already. - code = ( - "import sys\n" - "sys.modules['sphinx'] = None\n" # any `import sphinx...` now fails - "import sphinx_test_reports.projectconfig\n" - ) - result = subprocess.run( - [sys.executable, "-c", code], capture_output=True, text=True - ) - assert result.returncode == 0, result.stderr - - @pytest.mark.toolchain - def test_a_missing_sphinx_needs_is_an_extension_error(self): - # The lazy `setup` owns the message Sphinx would have produced for a - # broken extension import, because Sphinx fetches `setup` with - # getattr() and would otherwise show a raw traceback. Sphinx renders - # the wrapped exception itself, so the message must not carry it a - # second time -- but it names the extra that installs the toolchain, - # the likely cause since the toolchain stopped being a dependency. - # Checked in a subprocess: sphinx_needs is importable here. - code = ( - "import sys\n" - "sys.modules['sphinx_needs'] = None\n" # `from sphinx_needs...` fails - "import sphinx_test_reports as pkg\n" - "from sphinx.errors import ExtensionError\n" - "try:\n" - " pkg.setup\n" - "except ExtensionError as error:\n" - " print(error)\n" - "else:\n" - " raise AssertionError('no ExtensionError')\n" - ) - result = subprocess.run( - [sys.executable, "-c", code], capture_output=True, text=True - ) - assert result.returncode == 0, result.stderr - message = result.stdout.strip() - assert message.startswith("Could not import extension sphinx_test_reports") - assert message.count("(exception:") == 1 - assert "sphinx_needs" in message - assert 'pip install "sphinx-test-reports[sphinx]"' in message diff --git a/packages/sphinx-test-reports/tests/test_properties.py b/packages/sphinx-test-reports/tests/test_properties.py index 345e4d6a9..f43f741c3 100644 --- a/packages/sphinx-test-reports/tests/test_properties.py +++ b/packages/sphinx-test-reports/tests/test_properties.py @@ -25,7 +25,7 @@ class TestParserExtractsTestcaseProperties: """JUnitParser must extract from elements.""" def test_testcase_with_properties_returns_dict(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -38,7 +38,7 @@ def test_testcase_with_properties_returns_dict(self): assert tc["properties"]["priority"] == "high" def test_testcase_with_single_property(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -49,7 +49,7 @@ def test_testcase_with_single_property(self): assert tc["properties"] == {"verifies": "REQ_AUTH_003"} def test_testcase_without_properties_returns_empty_dict(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -60,7 +60,7 @@ def test_testcase_without_properties_returns_empty_dict(self): assert tc["properties"] == {} def test_testcase_skipped_without_properties_returns_empty_dict(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -72,7 +72,7 @@ def test_testcase_skipped_without_properties_returns_empty_dict(self): assert tc["properties"] == {} def test_multiple_suites_testcase_properties(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -84,7 +84,7 @@ def test_multiple_suites_testcase_properties(self): assert tc["properties"]["category"] == "integration" def test_testcase_without_any_properties_element(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -99,7 +99,7 @@ class TestParserExtractsTestsuiteProperties: """JUnitParser must extract from elements.""" def test_testsuite_with_properties(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -112,7 +112,7 @@ def test_testsuite_with_properties(self): assert suite["properties"]["build_id"] == "build-7742" def test_testsuite_without_properties(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_properties_path) results = parser.parse() @@ -127,7 +127,7 @@ class TestParserBackwardCompatibility: """Adding properties extraction must not break existing XML without properties.""" def test_existing_xml_still_parses(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_no_properties_path) results = parser.parse() @@ -141,7 +141,7 @@ def test_existing_xml_still_parses(self): assert tc["properties"] == {} def test_existing_testcase_fields_unchanged(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_no_properties_path) results = parser.parse() @@ -152,7 +152,7 @@ def test_existing_testcase_fields_unchanged(self): assert tc["result"] == "passed" def test_existing_failure_testcase_unchanged(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_no_properties_path) results = parser.parse() @@ -271,7 +271,7 @@ class TestParserHandlesEmptyProperties: """JUnitParser must not crash on empty elements.""" def test_empty_testsuite_properties_element(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_empty_properties_path) results = parser.parse() @@ -281,7 +281,7 @@ def test_empty_testsuite_properties_element(self): assert suite["properties"] == {} def test_empty_testcase_properties_element(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_empty_properties_path) results = parser.parse() @@ -291,7 +291,7 @@ def test_empty_testcase_properties_element(self): assert tc["properties"] == {} def test_testcase_without_properties_alongside_empty(self): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_empty_properties_path) results = parser.parse() diff --git a/packages/sphinx-test-reports/tests/test_result_vocabulary.py b/packages/sphinx-test-reports/tests/test_result_vocabulary.py index 518b552af..a89b93088 100644 --- a/packages/sphinx-test-reports/tests/test_result_vocabulary.py +++ b/packages/sphinx-test-reports/tests/test_result_vocabulary.py @@ -13,145 +13,14 @@ participles, and the mapping is the only place that decides. The part-level ``kind`` is deliberately *not* normalised -- it names the XML element the evidence came from, which is what the rendered evidence heading reports. -""" -import os +This is the half that needs Sphinx: the shipped report template rendering the vocabulary. +The parsers' half is ub-test-reports', ``packages/ub-test-reports/tests/test_result_vocabulary.py``. +""" import pytest -UTILS = os.path.join(os.path.dirname(__file__), "doc_test", "utils") -XML_PATH = os.path.join(UTILS, "xml_data.xml") -JSON_PATH = os.path.join(UTILS, "json_data.json") - -#: The mapping the JSON parser needs, as ``tr_json_mapping`` declares it. -JSON_MAPPING = { - "testsuite": { - "name": (["name"], "unknown"), - "tests": (["tests"], "unknown"), - "errors": (["errors"], "unknown"), - "failures": (["failures"], "unknown"), - "skips": (["skips"], "unknown"), - "passed": (["passed"], "unknown"), - "time": (["time"], "unknown"), - "testcases": (["testcase"], "unknown"), - }, - "testcase": { - "name": (["name"], "unknown"), - "classname": (["classname"], "unknown"), - "file": (["file"], "unknown"), - "line": (["line"], "unknown"), - "time": (["time"], "unknown"), - "result": (["result"], "unknown"), - "type": (["type"], "unknown"), - "text": (["text"], "unknown"), - "message": (["message"], "unknown"), - "system-out": (["system-out"], "unknown"), - }, -} - - -class TestNormalisation: - """One function decides the vocabulary, so both parsers cannot disagree.""" - - def test_the_junit_failure_element_name_becomes_failed(self): - from sphinx_test_reports.results import normalize_result - - assert normalize_result("failure") == "failed" - - def test_normalising_the_canonical_spelling_changes_nothing(self): - """Normalisation runs on already-canonical values too, so it must be - idempotent -- a JSON report may already spell the result ``failed``.""" - from sphinx_test_reports.results import normalize_result - - assert normalize_result("failed") == "failed" - - @pytest.mark.parametrize("result", ["passed", "skipped", "error", "disabled"]) - def test_the_other_states_are_already_canonical(self, result): - from sphinx_test_reports.results import normalize_result - - assert normalize_result(result) == result - - def test_a_vocabulary_this_extension_does_not_know_is_left_alone(self): - """``tr_json_mapping`` points at an arbitrary report, so a project may - feed in states of its own. Rewriting those would break its filters.""" - from sphinx_test_reports.results import normalize_result - - assert normalize_result("flaky") == "flaky" - - def test_the_canonical_states_are_the_documented_ones(self): - """Ordered, because the declared field description is built from it and - the converter's output has to be byte-stable.""" - from sphinx_test_reports.results import CANONICAL_RESULTS - - assert CANONICAL_RESULTS == ( - "passed", - "failed", - "error", - "skipped", - "disabled", - ) - - -class TestDeclaredSchema: - """The converter writes the field declarations into the ``needs.json`` it - produces, so that a consumer which never loads this extension -- a schema - check, a metamodel validator -- learns the fields from the file. Naming the - states in the ``result`` description tells it the field's domain too. - """ - - def test_the_result_declaration_names_every_state(self): - from sphinx_test_reports.fields import declaration - from sphinx_test_reports.results import CANONICAL_RESULTS - - _, description = declaration("result") - - assert all(state in description for state in CANONICAL_RESULTS), description - - -class TestJUnitParser: - """The JUnit dialect is where the old spelling came from.""" - - def test_a_failure_child_yields_the_failed_result(self): - from sphinx_test_reports.junitparser import JUnitParser - - suite = JUnitParser(XML_PATH).parse()[0] - - assert suite["testcases"][2]["result"] == "failed" - - def test_a_result_part_keeps_the_name_of_its_xml_element(self): - """``kind`` reports which element the evidence came from -- the - converter capitalises it into the evidence heading -- so it stays the - XML name even though ``result`` no longer is.""" - from sphinx_test_reports.junitparser import JUnitParser - - suite = JUnitParser(XML_PATH).parse()[0] - - assert suite["testcases"][2]["parts"][0]["kind"] == "failure" - - -class TestJsonParser: - """The JSON parser's API is documented as being in sync with the JUnit - parser's, so the same report content has to produce the same result.""" - - def test_the_failure_spelling_in_a_json_report_is_normalised(self): - from sphinx_test_reports.jsonparser import JsonParser - - parser = JsonParser(JSON_PATH, json_mapping=JSON_MAPPING) - suite = parser.parse()[0] - - assert suite["testcases"][0]["result"] == "failed" - - def test_the_results_needing_no_normalisation_are_untouched(self): - from sphinx_test_reports.jsonparser import JsonParser - - parser = JsonParser(JSON_PATH, json_mapping=JSON_MAPPING) - suite = parser.parse()[0] - - assert suite["testcases"][1]["result"] == "passed" - assert suite["testcases"][2]["result"] == "skipped" - -@pytest.mark.toolchain @pytest.mark.parametrize( "test_app", [{"buildername": "html", "srcdir": "doc_test/default_tr_template"}], diff --git a/packages/sphinx-test-reports/tests/test_toolchain.py b/packages/sphinx-test-reports/tests/test_toolchain.py deleted file mode 100644 index 34629236a..000000000 --- a/packages/sphinx-test-reports/tests/test_toolchain.py +++ /dev/null @@ -1,158 +0,0 @@ -"""The documentation toolchain is an extra; the extension checks it at load time. - -``pip install sphinx-test-reports`` installs the converter's dependency only. -The Sphinx extension needs Sphinx and sphinx-needs at the versions the -``sphinx`` extra declares -- and an extra is opt-in, so a project that installs -the bare package next to an older toolchain never shows pip those floors. The -lazy ``setup`` compares the installed versions against the extra's declarations -when Sphinx loads the extension, and names the install line in its message. -""" - -import sys -from importlib.metadata import PackageNotFoundError -from io import StringIO - -import pytest - -import sphinx_test_reports as package -from sphinx_test_reports import toolchain - -DECLARED = [ - "lxml", - 'sphinx>=7.4; extra == "sphinx"', - 'sphinx-needs>=6.0.1; extra == "sphinx"', - 'pytest>=7.0; extra == "pytest"', - 'pytest>=7.0; extra == "test"', -] - -AT_THE_FLOORS = {"sphinx": "7.4.7", "sphinx-needs": "6.0.1"} - -VIOLATION = "sphinx-needs 5.1.0 is installed, but sphinx-needs>=6.0.1 is required" - - -def _environment(monkeypatch, installed, declared=DECLARED): - """Fake the metadata: what this package declares and what is installed.""" - - def requires(name): - assert name == toolchain.DISTRIBUTION - return declared - - def version(name): - try: - return installed[name] - except KeyError: - raise PackageNotFoundError(name) from None - - monkeypatch.setattr(toolchain, "requires", requires) - monkeypatch.setattr(toolchain, "version", version) - - -class TestUnmetRequirements: - def test_a_toolchain_at_the_floors_is_accepted(self, monkeypatch): - _environment(monkeypatch, AT_THE_FLOORS) - assert toolchain.unmet_requirements() == [] - - def test_a_version_below_the_floor_is_named_with_the_floor(self, monkeypatch): - _environment(monkeypatch, {**AT_THE_FLOORS, "sphinx-needs": "5.1.0"}) - assert toolchain.unmet_requirements() == [VIOLATION] - - def test_every_violation_is_reported_in_declaration_order(self, monkeypatch): - _environment(monkeypatch, {"sphinx": "7.3.7", "sphinx-needs": "5.1.0"}) - assert toolchain.unmet_requirements() == [ - "sphinx 7.3.7 is installed, but sphinx>=7.4 is required", - VIOLATION, - ] - - def test_a_missing_distribution_is_left_to_the_import(self, monkeypatch): - # The import that fails on a missing package states the problem more - # precisely -- and a toolchain importable from a source tree without - # metadata must not be refused on the strength of the metadata alone. - _environment(monkeypatch, {"sphinx": "7.4.7"}) - assert toolchain.unmet_requirements() == [] - - def test_other_extras_and_the_core_dependency_are_not_checked(self, monkeypatch): - _environment(monkeypatch, {**AT_THE_FLOORS, "lxml": "0.1", "pytest": "1.0"}) - assert toolchain.unmet_requirements() == [] - - def test_a_prerelease_above_the_floor_is_accepted(self, monkeypatch): - _environment(monkeypatch, {**AT_THE_FLOORS, "sphinx-needs": "9.0.0rc1"}) - assert toolchain.unmet_requirements() == [] - - def test_without_our_own_metadata_there_is_nothing_to_check(self, monkeypatch): - # Run from a source tree on sys.path, the package has no metadata. - def requires(name): - raise PackageNotFoundError(name) - - monkeypatch.setattr(toolchain, "requires", requires) - assert toolchain.unmet_requirements() == [] - - def test_without_packaging_the_check_is_skipped(self, monkeypatch): - # `packaging` is not a dependency of this package: it arrives with - # Sphinx (and with pytest). Where neither is installed, nothing loads - # the extension either -- so the check stands down instead of failing - # on its own import. - _environment(monkeypatch, {"sphinx": "7.3.7", "sphinx-needs": "5.1.0"}) - monkeypatch.setitem(sys.modules, "packaging.requirements", None) - assert toolchain.unmet_requirements() == [] - - def test_the_installed_metadata_declares_the_toolchain_under_the_extra(self): - # Against the real metadata: the extra the check reads is the one - # pyproject.toml declares. Renaming it there would silently disarm the - # check, and this test. - names = sorted( - requirement.name for requirement in toolchain.toolchain_requirements() - ) - assert names == ["docutils", "sphinx", "sphinx-needs"] - - def test_this_environment_meets_the_floors(self): - assert toolchain.unmet_requirements() == [] - - -@pytest.mark.toolchain -class TestLazySetup: - """Sphinx resolves ``setup`` through getattr(); the check runs first.""" - - def test_a_sufficient_toolchain_resolves_setup(self): - from sphinx_test_reports.test_reports import setup - - assert package.setup is setup - - def test_a_toolchain_below_the_floors_is_an_extension_error(self, monkeypatch): - from sphinx.errors import ExtensionError - - monkeypatch.setattr(toolchain, "unmet_requirements", lambda: [VIOLATION]) - with pytest.raises(ExtensionError) as info: - _ = package.setup - message = str(info.value) - assert message.startswith("Could not load extension sphinx_test_reports") - assert VIOLATION in message - assert toolchain.INSTALL_HINT in message - # Nothing was imported, so there is no exception to wrap; the message - # must not end in an empty "(exception: ...)". - assert "(exception:" not in message - - def test_sphinx_surfaces_the_error_when_loading_the_extension( - self, tmp_path, monkeypatch - ): - # Sphinx fetches `setup` with getattr(module, "setup", None), which - # only swallows AttributeError. The error must reach the user as the - # extension error it is, install line included -- not as "extension - # has no setup() function". - from sphinx.application import Sphinx - from sphinx.errors import ExtensionError - - monkeypatch.setattr(toolchain, "unmet_requirements", lambda: [VIOLATION]) - (tmp_path / "conf.py").write_text( - 'extensions = ["sphinx_test_reports"]\n', encoding="utf-8" - ) - (tmp_path / "index.rst").write_text("Index\n=====\n", encoding="utf-8") - with pytest.raises(ExtensionError, match=r"sphinx-test-reports\[sphinx\]"): - Sphinx( - srcdir=tmp_path, - confdir=tmp_path, - outdir=tmp_path / "_build", - doctreedir=tmp_path / "_doctrees", - buildername="html", - status=None, - warning=StringIO(), - ) diff --git a/packages/ub-test-reports/AGENTS.md b/packages/ub-test-reports/AGENTS.md new file mode 100644 index 000000000..b646864bd --- /dev/null +++ b/packages/ub-test-reports/AGENTS.md @@ -0,0 +1,67 @@ +# AGENTS.md — packages/ub-test-reports + +The delta for this package. Everything repository-level — the workspace layout, the +commands, the lock, lint/format/type-check configuration, the release recipe, the pull +request requirements — is in the ROOT [`AGENTS.md`](../../AGENTS.md), and this file does +not repeat it. + +## What it is + +The Sphinx-free core of sphinx-test-reports: the report parsers, the result vocabulary, +the deterministic case IDs, the `[test_reports]` model of `ubproject.toml`, the +`test-reports` converter that writes a `needs.json`, and the pytest plugin. It is a +**tool, not a Sphinx extension** — nothing to add to `conf.py`, no `Framework :: Sphinx` +classifier. sphinx-test-reports, the extension, depends on it and turns the same reports +into needs inside a build; the dependency never runs the other way. Its documentation is +the "Without Sphinx" section of sphinx-test-reports' site; it has no site of its own. + +## Commands + +```bash +uv run poe test-ub-test-reports # the suite (no sphinx axis: it has no Sphinx) +uv run poe import-check-ub-test-reports # import every module from the built wheel +uv run poe build-ub-test-reports # sdist + wheel into dist/ub-test-reports +``` + +## Rules + +- **Nothing here may import Sphinx, sphinx-needs or docutils** — not at module level and + not in a function body, because the converter runs as a build action and the plugin + inside a test run, and neither has a documentation toolchain. `tests/test_imports.py` + refuses any import STATEMENT naming the toolchain (or the extension), at any depth and in + every environment, whether or not a test reaches the line; three subprocess tests also + check that importing the CLI's chain, the plugin and `projectconfig` loads no Sphinx. + CI's `toolchain-free` job is the fence for what a static walk cannot see — a dependency + that drags Sphinx in, an import spelled dynamically — on the lines a test reaches, and for + the plugin's subprocess runs: it installs the built wheel where the toolchain is absent and + runs the whole suite there. +- **`ub-project` is its `ubproject.toml` reader.** Finding, loading and anchoring the file + come from there (`packages/ub-project/design/reading-contract.md` is the specification); + what stays here is the `[test_reports]` policy -- keys, types, normalisation, unknown keys + warned rather than fatal -- and **`TomlConfigError`, the only exception either consumer + catches**: `load_project_config` re-raises ub-project's `ProjectConfigError` as it, with + the same message, and it must never be made a subclass of it. +- **The `[test_reports]` model is a parity surface**: ubCode reads the same table and is + held to the same behaviour. A behaviour change in it — keys, types, normalisation, + defaults — says so in the changelog, so ubCode can follow. +- **The pytest plugin is opt-in**: `-p ub_test_reports.pytest_plugin`, and no `pytest11` + entry point — an auto-loaded plugin would change every pytest run in any environment that + merely has this package installed, the extension's users included. The `pytest` extra's + floor is fenced by CI's `plugin-floor` job, which runs the suite on the oldest pytest of + each Python it names. +- **The wire names `sphinxcontrib.test_reports:file` and `sphinxcontrib.test_reports:line` + must not change.** They are the documented `record_property` names the plugin reserves, + not an import path, and reports written by older plugins carry them. +- **The test fixtures are this package's own copies** (`tests/fixtures/`), so that its + suite reads nothing from another member's tree and runs from the installed wheel; some of + them also exist under sphinx-test-reports' `tests/doc_test/utils/`, where its test + projects and docs read them. Neither copy ships in an sdist. +- **One plugin test drives an in-process pytest session, and the default `.venv` breaks + it.** `tests/test_pytest_plugin.py`'s `NESTED` source starts `pytest.main()` inside a + `pytester` session, where every installed plugin loads; pytest-playwright (the root `js` + group, in the default `dev` group) refuses the nested soft-assertion scope. So it passes + `-p no:playwright`, and that line has to stay ONE line: a sibling test builds its own + source from it by replacing the literal `str(inner)]) == 0`. Run the suite in the default + `.venv` AND in a cell — green in one proves nothing about the other. +- **Tests build paths with `Path`** and read and write text with an explicit `encoding`, so + that the suite holds on Windows too. diff --git a/packages/ub-test-reports/CLAUDE.md b/packages/ub-test-reports/CLAUDE.md new file mode 100644 index 000000000..43c994c2d --- /dev/null +++ b/packages/ub-test-reports/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/packages/ub-test-reports/LICENSE b/packages/ub-test-reports/LICENSE new file mode 100644 index 000000000..616fe15fc --- /dev/null +++ b/packages/ub-test-reports/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 useblocks + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/ub-test-reports/README.rst b/packages/ub-test-reports/README.rst new file mode 100644 index 000000000..a8dd79ab3 --- /dev/null +++ b/packages/ub-test-reports/README.rst @@ -0,0 +1,32 @@ +ub-test-reports +=============== + +A tool for the sphinx-needs family -- not a Sphinx extension, and nothing to add to +``conf.py``. + +It turns test results into needs without running Sphinx, and holds everything about test +reports that does not need a documentation build: + +- **the parsers** -- JUnit XML (pytest, googletest, ctest, nose and friends) and + tox-envreport style JSON; +- **the result vocabulary** -- one spelling for every outcome, whichever report it came + from; +- **deterministic case IDs** -- the same case gets the same need ID in every run; +- **the** ``[test_reports]`` **model** of ``ubproject.toml``, read through ``ub-project``; +- **the converter** -- the ``test-reports build needs`` command, which writes a + ``needs.json`` that sphinx-needs can import; +- **the pytest plugin** -- ``-p ub_test_reports.pytest_plugin``, which shapes the JUnit XML + pytest writes (source locations, per-case properties for traceability). + +Install the converter with:: + + pip install ub-test-reports + +and the pytest plugin with:: + + pip install "ub-test-reports[pytest]" + +To show the same reports inside a Sphinx documentation build, install the extension, +``sphinx-test-reports``, which depends on this package. + +Documentation: the "Without Sphinx" section of https://sphinx-test-reports.readthedocs.io/en/latest/ diff --git a/packages/ub-test-reports/docs/changelog.rst b/packages/ub-test-reports/docs/changelog.rst new file mode 100644 index 000000000..a0ec36852 --- /dev/null +++ b/packages/ub-test-reports/docs/changelog.rst @@ -0,0 +1,13 @@ +.. _changelog: + +Changelog +========= + +Unreleased +---------- + +- 1.0.0 is the first release: the converter, the pytest plugin, the parsers, the result + vocabulary, the deterministic IDs and the ``[test_reports]`` model, moved out of + sphinx-test-reports 2.0.0 unchanged; the plugin is ``-p ub_test_reports.pytest_plugin``, + and its warning prefix and pluggy registration name now say ``ub_test_reports`` (the + extension's changelog lists both); its wire names are unchanged. diff --git a/packages/ub-test-reports/pyproject.toml b/packages/ub-test-reports/pyproject.toml new file mode 100644 index 000000000..10f7a8fb4 --- /dev/null +++ b/packages/ub-test-reports/pyproject.toml @@ -0,0 +1,60 @@ +[project] +name = "ub-test-reports" +version = "1.0.0.dev0" +description = "Test results as needs without Sphinx: report parsers, the needs.json converter and the pytest plugin" +authors = [{ name = "team useblocks", email = "info@useblocks.com" }] +license = { file = "LICENSE" } +readme = "README.rst" +requires-python = ">=3.11,<4" +# NO Sphinx, docutils or sphinx-needs, and that is the package's whole contract: the +# converter runs as a build action and the plugin inside a test run, neither of which has a +# documentation toolchain. ub-project is the `ubproject.toml` reader, TIGHT-tracked like +# every intra-workspace edge (`check_workspace.py` check (4)); `propagate_floors.py` moves +# the floor at each ub-project release. The `toolchain-free` CI job installs the built wheel +# where Sphinx is absent and runs the whole suite there +dependencies = ["lxml", "ub-project>=1.1.0,<2"] +keywords = ["test-reports", "junit", "needs.json", "sphinx-needs", "pytest"] +# NOT `Framework :: Sphinx`: nothing here depends on Sphinx +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Framework :: Pytest", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Software Development :: Testing", + "Typing :: Typed", +] + +[project.optional-dependencies] +# The pytest plugin (`-p ub_test_reports.pytest_plugin`); pytest is not a dependency of the +# converter. 7.0 is where everything the plugin uses exists (pytest.Config/Item/Mark, +# Config.stash, issue_config_time_warning). pytest before 7.3.2 does not run on Python +# 3.12 at all -- that floor is pytest's own, not this one's. The `plugin-floor` CI job runs +# the whole suite on pytest 7.0.1 (Python 3.11) and 7.3.2 (Python 3.12). +pytest = ["pytest>=7.0"] + +[project.scripts] +test-reports = "ub_test_reports.cli:main" + +[project.urls] +Documentation = "https://sphinx-test-reports.readthedocs.io/en/latest/" +Repository = "https://github.com/useblocks/sphinx-needs" +Issues = "https://github.com/useblocks/sphinx-needs/issues" +Changelog = "https://github.com/useblocks/sphinx-needs/blob/master/packages/ub-test-reports/docs/changelog.rst" + +[build-system] +requires = ["flit_core >=3.4,<5"] +build-backend = "flit_core.buildapi" + +# Deliberately NO `[tool.flit.sdist]`: flit's default sdist is the module (with its +# `py.typed` marker and `schemas/JUnit.xsd`), README, LICENSE and this file, and nothing +# else -- so neither `tests/` (and its fixture copies) nor `docs/` ships. Everything else -- +# ruff, ty, pytest and the dependency groups -- is the ROOT's, and +# `tools/src/sn_tools/check_workspace.py` refuses those tables here. diff --git a/packages/ub-test-reports/src/ub_test_reports/__init__.py b/packages/ub-test-reports/src/ub_test_reports/__init__.py new file mode 100644 index 000000000..252e7bdaa --- /dev/null +++ b/packages/ub-test-reports/src/ub_test_reports/__init__.py @@ -0,0 +1,17 @@ +"""ub-test-reports: test results as needs, without Sphinx. + +The report parsers, the result vocabulary, the deterministic case IDs, the +``[test_reports]`` model of ``ubproject.toml``, the needs.json converter behind the +``test-reports`` command, and the pytest plugin (``-p ub_test_reports.pytest_plugin``). +The modules are the API; this file only carries the version. + +**Nothing in this package may import Sphinx, sphinx-needs or docutils** -- not at module +level and not in a function body. sphinx-test-reports, the Sphinx extension, depends on +this package, never the other way round. +""" + +__all__ = ["__version__"] + +#: Checked against ``[project] version`` by ``check_workspace.py`` and stamped by +#: ``poe bump``. +__version__ = "1.0.0.dev0" diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/cli.py b/packages/ub-test-reports/src/ub_test_reports/cli.py similarity index 99% rename from packages/sphinx-test-reports/src/sphinx_test_reports/cli.py rename to packages/ub-test-reports/src/ub_test_reports/cli.py index 0130b4cef..37c6ec8d8 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/cli.py +++ b/packages/ub-test-reports/src/ub_test_reports/cli.py @@ -19,15 +19,15 @@ from collections.abc import Mapping, Sequence from pathlib import Path -from sphinx_test_reports.junitparser import JUnitParser -from sphinx_test_reports.needs_export import ( +from ub_test_reports.junitparser import JUnitParser +from ub_test_reports.needs_export import ( DEFAULT_VERSION, Report, build_needs_file, iter_cases, optional, ) -from sphinx_test_reports.projectconfig import ( +from ub_test_reports.projectconfig import ( CONVERSION_KEYS, DEFAULT_NEED_TYPE, DEFAULT_TOML_FILENAME, @@ -40,7 +40,7 @@ load_project_config, needs_settings, ) -from sphinx_test_reports.remote import ( +from ub_test_reports.remote import ( DEFAULT_URL_PATTERN, check_url_pattern, normalise_remote_url, diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/fields.py b/packages/ub-test-reports/src/ub_test_reports/fields.py similarity index 98% rename from packages/sphinx-test-reports/src/sphinx_test_reports/fields.py rename to packages/ub-test-reports/src/ub_test_reports/fields.py index a823ccaa6..08bbb0d67 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/fields.py +++ b/packages/ub-test-reports/src/ub_test_reports/fields.py @@ -18,7 +18,7 @@ from collections.abc import Iterable, Mapping -from sphinx_test_reports.results import CANONICAL_RESULTS +from ub_test_reports.results import CANONICAL_RESULTS #: ``name -> (JSON type, description)`` for every field declared under a fixed #: name. The build registers all of them, on test-file, test-suite and @@ -56,7 +56,7 @@ #: ``role -> (JSON type, description)`` for the fields whose *name* the #: configuration chooses; the roles are the keys of -#: :data:`~sphinx_test_reports.projectconfig.DEFAULT_FIELD_NAMES`. +#: :data:`~ub_test_reports.projectconfig.DEFAULT_FIELD_NAMES`. #: Keying them by role rather than by name is what lets the description follow #: a renamed field. RENAMEABLE_FIELDS: dict[str, tuple[str, str]] = { diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/identity.py b/packages/ub-test-reports/src/ub_test_reports/identity.py similarity index 100% rename from packages/sphinx-test-reports/src/sphinx_test_reports/identity.py rename to packages/ub-test-reports/src/ub_test_reports/identity.py diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/jsonparser.py b/packages/ub-test-reports/src/ub_test_reports/jsonparser.py similarity index 98% rename from packages/sphinx-test-reports/src/sphinx_test_reports/jsonparser.py rename to packages/ub-test-reports/src/ub_test_reports/jsonparser.py index 09262c781..7e1dbbc05 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/jsonparser.py +++ b/packages/ub-test-reports/src/ub_test_reports/jsonparser.py @@ -12,7 +12,7 @@ from functools import reduce from typing import Any -from sphinx_test_reports.results import normalize_result +from ub_test_reports.results import normalize_result def dict_get(root, items, default=None): diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/junitparser.py b/packages/ub-test-reports/src/ub_test_reports/junitparser.py similarity index 98% rename from packages/sphinx-test-reports/src/sphinx_test_reports/junitparser.py rename to packages/ub-test-reports/src/ub_test_reports/junitparser.py index c42633df2..fa1ac668e 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/junitparser.py +++ b/packages/ub-test-reports/src/ub_test_reports/junitparser.py @@ -6,7 +6,7 @@ from lxml import etree, objectify # ty: ignore[unresolved-import] -from sphinx_test_reports.results import normalize_result +from ub_test_reports.results import normalize_result #: Attributes the JUnit/googletest dialects define themselves. Every *other* #: attribute is a ``RecordProperty`` value in attribute form: googletest wrote @@ -50,7 +50,7 @@ #: ```` children carrying a result, in the precedence order used to #: classify a case that has more than one kind of them. These are XML element #: names, not ``result`` values -- ```` is read as the result -#: ``failed`` (see :mod:`sphinx_test_reports.results`). +#: ``failed`` (see :mod:`ub_test_reports.results`). RESULT_PART_KINDS = ("skipped", "failure", "error") diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/needs_export.py b/packages/ub-test-reports/src/ub_test_reports/needs_export.py similarity index 97% rename from packages/sphinx-test-reports/src/sphinx_test_reports/needs_export.py rename to packages/ub-test-reports/src/ub_test_reports/needs_export.py index 64bad4ae2..ee1a17b2b 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/needs_export.py +++ b/packages/ub-test-reports/src/ub_test_reports/needs_export.py @@ -14,7 +14,7 @@ also writes into the files it produces itself, so the type of a field is readable from the artifact instead of only from a Sphinx build with this extension loaded. The declarations come from - :mod:`sphinx_test_reports.fields`, the same table the extension + :mod:`ub_test_reports.fields`, the same table the extension registers its fields from. * **Nothing depends on the wall clock or on dict ordering**, so the file is a cacheable build artifact and a diffable piece of evidence. @@ -24,14 +24,14 @@ import textwrap from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence -from sphinx_test_reports.fields import case_needs_schema -from sphinx_test_reports.identity import ( +from ub_test_reports.fields import case_needs_schema +from ub_test_reports.identity import ( UNKNOWN, deterministic_case_id, split_case_name, ) -from sphinx_test_reports.projectconfig import DEFAULT_FIELD_NAMES -from sphinx_test_reports.remote import DEFAULT_URL_PATTERN, source_url +from ub_test_reports.projectconfig import DEFAULT_FIELD_NAMES +from ub_test_reports.remote import DEFAULT_URL_PATTERN, source_url #: A need field value as it appears in needs.json. ``None`` is the value of an #: exported property the case does not carry, as the build leaves it. @@ -178,7 +178,7 @@ def build_need( :data:`DEFAULT_FIELD_NAMES`) -- and so are the values: the title is the case name and ``result`` keeps the parser's spelling (``failed``, which is also a documented field value and the ``tr_failed`` CSS class, see - :mod:`sphinx_test_reports.results`), so that a need imported from + :mod:`ub_test_reports.results`), so that a need imported from the produced ``needs.json`` and one created locally from the same report are indistinguishable to a schema, a filter or a ``needtable``. diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/projectconfig.py b/packages/ub-test-reports/src/ub_test_reports/projectconfig.py similarity index 99% rename from packages/sphinx-test-reports/src/sphinx_test_reports/projectconfig.py rename to packages/ub-test-reports/src/ub_test_reports/projectconfig.py index b455fd92b..f73ab2a71 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/projectconfig.py +++ b/packages/ub-test-reports/src/ub_test_reports/projectconfig.py @@ -47,12 +47,12 @@ from pathlib import Path from typing import NoReturn -from sphinx_test_reports.fields import RESERVED_NAMES from ub_project import DEFAULT_FILENAME, ProjectConfigError, anchor, load_toml # Re-exported: the Sphinx bridge, the converter and the tests import the walk # from here. It raises nothing of its own, so it needs no wrapper. from ub_project import find_project_config as find_project_config +from ub_test_reports.fields import RESERVED_NAMES #: Default file the configuration is read from. Looked up by walking up from #: the ``confdir`` (Sphinx) or the working directory (a converter); see diff --git a/packages/ub-test-reports/src/ub_test_reports/py.typed b/packages/ub-test-reports/src/ub_test_reports/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/pytest_plugin.py b/packages/ub-test-reports/src/ub_test_reports/pytest_plugin.py similarity index 98% rename from packages/sphinx-test-reports/src/sphinx_test_reports/pytest_plugin.py rename to packages/ub-test-reports/src/ub_test_reports/pytest_plugin.py index 2302793e1..ea432e33f 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/pytest_plugin.py +++ b/packages/ub-test-reports/src/ub_test_reports/pytest_plugin.py @@ -1,6 +1,6 @@ """pytest plugin: shape the JUnit XML the way this extension reads it. -Enable it with ``-p sphinx_test_reports.pytest_plugin`` (or in +Enable it with ``-p ub_test_reports.pytest_plugin`` (or in ``addopts``) and write the report with ``--junitxml`` under ``junit_family = xunit1``. Two things then happen to every ````: @@ -92,7 +92,7 @@ } #: Registration name of the per-session hook object, :class:`_XmlShape`. -_HOOKS = "sphinx_test_reports.xml_shape" +_HOOKS = "ub_test_reports.xml_shape" Recorder = Callable[[str, str], None] @@ -357,7 +357,7 @@ class TestReportsConfigWarning(pytest.PytestWarning): Issued once at start-up. Where the project turns warnings into errors (``filterwarnings = error``, ``-W error``) it becomes a clean usage error rather than a traceback; - ``ignore::sphinx_test_reports.pytest_plugin.TestReportsConfigWarning`` + ``ignore::ub_test_reports.pytest_plugin.TestReportsConfigWarning`` silences it. """ @@ -376,7 +376,7 @@ def _report_family(config: pytest.Config) -> str | None: def _notify(config: pytest.Config, message: str) -> None: """Issue *message* as :class:`TestReportsConfigWarning` at configure time.""" - warning = TestReportsConfigWarning(f"sphinx_test_reports.pytest_plugin: {message}") + warning = TestReportsConfigWarning(f"ub_test_reports.pytest_plugin: {message}") try: config.issue_config_time_warning(warning, stacklevel=3) except TestReportsConfigWarning as error: @@ -401,7 +401,7 @@ def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line( "markers", f"{MARKER}(properties): properties written to the JUnit XML of the " - "test; attached by sphinx_test_reports.pytest_plugin.add_test_properties", + "test; attached by ub_test_reports.pytest_plugin.add_test_properties", ) lines: Sequence[str] = config.getini(OPTION) try: diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/remote.py b/packages/ub-test-reports/src/ub_test_reports/remote.py similarity index 100% rename from packages/sphinx-test-reports/src/sphinx_test_reports/remote.py rename to packages/ub-test-reports/src/ub_test_reports/remote.py diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/results.py b/packages/ub-test-reports/src/ub_test_reports/results.py similarity index 95% rename from packages/sphinx-test-reports/src/sphinx_test_reports/results.py rename to packages/ub-test-reports/src/ub_test_reports/results.py index dd99cc2bd..6b64a18d9 100644 --- a/packages/sphinx-test-reports/src/sphinx_test_reports/results.py +++ b/packages/ub-test-reports/src/ub_test_reports/results.py @@ -10,7 +10,7 @@ for googletest's ``status="notrun"``). The JSON parser passed its report's own value through untouched. So the same outcome could arrive under two spellings, and a single test case was ``failure`` while the count of them on its suite was -``failed`` (:data:`~sphinx_test_reports.fields.FIELDS`). +``failed`` (:data:`~ub_test_reports.fields.FIELDS`). ``error`` stays a noun-state deliberately: it agrees with the ``errors`` count beside it and with pytest, which spells that outcome ``error`` too. The pair @@ -29,7 +29,7 @@ #: consumers that never load this extension. #: #: A tuple, in the order a reader wants them rather than alphabetically: -#: :func:`~sphinx_test_reports.fields.declaration` renders it into the +#: :func:`~ub_test_reports.fields.declaration` renders it into the #: declared description of the field, and the converter's output has to be #: byte-stable, which a set's iteration order is not. CANONICAL_RESULTS: tuple[str, ...] = ( diff --git a/packages/sphinx-test-reports/src/sphinx_test_reports/schemas/JUnit.xsd b/packages/ub-test-reports/src/ub_test_reports/schemas/JUnit.xsd similarity index 100% rename from packages/sphinx-test-reports/src/sphinx_test_reports/schemas/JUnit.xsd rename to packages/ub-test-reports/src/ub_test_reports/schemas/JUnit.xsd diff --git a/packages/ub-test-reports/tests/__init__.py b/packages/ub-test-reports/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/ub-test-reports/tests/conftest.py b/packages/ub-test-reports/tests/conftest.py new file mode 100644 index 000000000..077eba3a3 --- /dev/null +++ b/packages/ub-test-reports/tests/conftest.py @@ -0,0 +1,9 @@ +"""ub-test-reports' suite needs no Sphinx: only pytest's own ``pytester``. + +It runs in the default environment and, in CI's ``toolchain-free`` and ``plugin-floor`` +jobs, against the BUILT wheel in an environment where Sphinx, sphinx-needs and docutils are +absent -- so nothing here may import them, and the fixtures it reads are this package's own +copies under ``tests/fixtures/``. +""" + +pytest_plugins = ["pytester"] diff --git a/packages/ub-test-reports/tests/fixtures/ctest.xml b/packages/ub-test-reports/tests/fixtures/ctest.xml new file mode 100644 index 000000000..bf757979d --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/ctest.xml @@ -0,0 +1,37 @@ + + + + usage: test <argument> + If <argument> is 0, print SUCCESS. Otherwise print FAIL. + + + + SUCCESS + + + + FAIL + + + + + FAIL + + + + + Skipped feature due to @skip tag +0 features passed, 0 failed, 1 skipped +0 scenarios passed, 0 failed, 14 skipped +0 steps passed, 0 failed, 42 skipped, 0 undefined + + + diff --git a/packages/ub-test-reports/tests/fixtures/gtest_data.xml b/packages/ub-test-reports/tests/fixtures/gtest_data.xml new file mode 100644 index 000000000..0c6ac8daf --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/gtest_data.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/ub-test-reports/tests/fixtures/json_complex_data.json b/packages/ub-test-reports/tests/fixtures/json_complex_data.json new file mode 100644 index 000000000..938ec42d1 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/json_complex_data.json @@ -0,0 +1,53 @@ +[ + { + "internals": { + "name": "test suite 1" + }, + "tests": 3, + "errors": 0, + "failures": 1, + "skips": 1, + "passed": 1, + "time": "20230515-15:04:32", + "testcase": { + "nested": [ + { + "name": "test case 1", + "classname": "class name 1", + "file": "my/test/file_1", + "line": 123, + "time": "20230515-15:04:32", + "result": "failure", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + }, + { + "name": "test case 2", + "classname": "class name 2", + "file": "my/test/file_2", + "line": 123, + "time": "20230515-15:04:32", + "result": "passed", + "type": "test type", + "text": "test text", + "message": "all went great :) (message)", + "system-out": "system-out text" + }, + { + "name": "test case 3", + "classname": "class name 3", + "file": "my/test/file_3", + "line": 123, + "time": "20230515-15:04:32", + "result": "skipped", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + } + ] + } + } +] diff --git a/packages/ub-test-reports/tests/fixtures/json_custom_data.json b/packages/ub-test-reports/tests/fixtures/json_custom_data.json new file mode 100644 index 000000000..098564c35 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/json_custom_data.json @@ -0,0 +1,72 @@ +[ + { + "name": "test suite 1", + "tests": 3, + "errors": 0, + "failures": 1, + "skips": 1, + "passed": 1, + "time": "20230515-15:04:32", + "testcase": [ + { + "name": "test case 1", + "classname": "class name 1", + "file": "my/test/file_1", + "line": 123, + "time": "20230515-15:04:32", + "result": "failure", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text", + "id": "TEST_CASE_1", + "status": "open", + "tags": "a,b,c", + "priority": "1" + }, + { + "name": "test case 2", + "classname": "class name 2", + "file": "my/test/file_2", + "line": 123, + "time": "20230515-15:04:32", + "result": "passed", + "type": "test type", + "text": "test text", + "message": "all went great :) (message)", + "system-out": "system-out text", + "id": "TEST_CASE_2", + "status": "closed", + "tags": "a,b", + "priority": "2" + }, + { + "name": "test case 3", + "classname": "class name 3", + "file": "my/test/file_3", + "line": 123, + "time": "20230515-15:04:32", + "result": "skipped", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text", + "id": "TEST_CASE_3", + "status": "", + "tags": "a" + }, + { + "name": "test case 4", + "classname": "class name 4", + "file": "my/test/file_3", + "line": 123, + "time": "20230515-15:04:32", + "result": "skipped", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + } + ] + } +] diff --git a/packages/ub-test-reports/tests/fixtures/json_data.json b/packages/ub-test-reports/tests/fixtures/json_data.json new file mode 100644 index 000000000..2ef48b783 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/json_data.json @@ -0,0 +1,49 @@ +[ + { + "name": "test suite 1", + "tests": 3, + "errors": 0, + "failures": 1, + "skips": 1, + "passed": 1, + "time": "20230515-15:04:32", + "testcase": [ + { + "name": "test case 1", + "classname": "class name 1", + "file": "my/test/file_1", + "line": 123, + "time": "20230515-15:04:32", + "result": "failure", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + }, + { + "name": "test case 2", + "classname": "class name 2", + "file": "my/test/file_2", + "line": 123, + "time": "20230515-15:04:32", + "result": "passed", + "type": "test type", + "text": "test text", + "message": "all went great :) (message)", + "system-out": "system-out text" + }, + { + "name": "test case 3", + "classname": "class name 3", + "file": "my/test/file_3", + "line": 123, + "time": "20230515-15:04:32", + "result": "skipped", + "type": "test type", + "text": "test text", + "message": "all went wrong :( (message)", + "system-out": "system-out text" + } + ] + } +] diff --git a/packages/ub-test-reports/tests/fixtures/nose_data.xml b/packages/ub-test-reports/tests/fixtures/nose_data.xml new file mode 100644 index 000000000..1117f63b6 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/nose_data.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/packages/ub-test-reports/tests/fixtures/pytest_data.xml b/packages/ub-test-reports/tests/fixtures/pytest_data.xml new file mode 100644 index 000000000..e7a1495f1 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/pytest_data.xml @@ -0,0 +1,52 @@ + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a8a0e950> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a8a0ea10> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a8497450> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a8497c10> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84a2350> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84a2a50> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84a2cd0> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84ae250> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84ae950> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84bf090> + + + diff --git a/packages/ub-test-reports/tests/fixtures/pytest_data_5_1.xml b/packages/ub-test-reports/tests/fixtures/pytest_data_5_1.xml new file mode 100644 index 000000000..1a429b924 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/pytest_data_5_1.xml @@ -0,0 +1,93 @@ + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a8a0e950> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a8a0ea10> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a8497450> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a8497c10> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a84a2350> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a84a2a50> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a84a2cd0> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a84ae250> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a84ae950> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: + <py._xmlgen.raw object at 0x7fd5a84bf090> + + + + diff --git a/packages/ub-test-reports/tests/fixtures/pytest_data_6_2.xml b/packages/ub-test-reports/tests/fixtures/pytest_data_6_2.xml new file mode 100644 index 000000000..37bbc8f2f --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/pytest_data_6_2.xml @@ -0,0 +1,29 @@ + + + + + + def test_fail (): +> assert False +E assert False + +test_example.py:11: AssertionError + + + ~/test_example.py:13: An example reason + + + ~/test_example.py:17: An example reason + + + + + + def test_error(): +> raise Exception("Bang!") +E Exception: Bang! + +test_example.py:31: Exception + + + diff --git a/packages/ub-test-reports/tests/fixtures/pytest_nested_example.xml b/packages/ub-test-reports/tests/fixtures/pytest_nested_example.xml new file mode 100644 index 000000000..b240964ac --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/pytest_nested_example.xml @@ -0,0 +1,40 @@ + + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a8a0e950> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a8a0ea10> + + + + + + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84a2cd0> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84ae250> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84ae950> + + + + + /home/daniel/workspace/sphinx/sphinx-test-reports/.tox/py27-sphinx15/local/lib/python2.7/site-packages/pytest_flake8.py:106: <py._xmlgen.raw object at 0x7fd5a84bf090> + + + + diff --git a/packages/ub-test-reports/tests/fixtures/runner_error_data.xml b/packages/ub-test-reports/tests/fixtures/runner_error_data.xml new file mode 100644 index 000000000..0bcf5a654 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/runner_error_data.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + diff --git a/packages/ub-test-reports/tests/fixtures/xml_data.xml b/packages/ub-test-reports/tests/fixtures/xml_data.xml new file mode 100644 index 000000000..135ca7003 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/xml_data.xml @@ -0,0 +1,7 @@ + + + + + details about failure + + diff --git a/packages/ub-test-reports/tests/fixtures/xml_data_error.xml b/packages/ub-test-reports/tests/fixtures/xml_data_error.xml new file mode 100644 index 000000000..1475938a2 --- /dev/null +++ b/packages/ub-test-reports/tests/fixtures/xml_data_error.xml @@ -0,0 +1,12 @@ + + + + some failure details + + + stack trace here + + + + + diff --git a/packages/sphinx-test-reports/tests/test_cli_config.py b/packages/ub-test-reports/tests/test_cli_config.py similarity index 99% rename from packages/sphinx-test-reports/tests/test_cli_config.py rename to packages/ub-test-reports/tests/test_cli_config.py index 13f2ba725..ebfd864ff 100644 --- a/packages/sphinx-test-reports/tests/test_cli_config.py +++ b/packages/ub-test-reports/tests/test_cli_config.py @@ -11,13 +11,13 @@ import pytest -from sphinx_test_reports.cli import _DEFAULTS, main -from sphinx_test_reports.projectconfig import ( +from ub_test_reports.cli import _DEFAULTS, main +from ub_test_reports.projectconfig import ( CONVERSION_KEYS, DEFAULT_TOML_FILENAME, ) -UTILS = Path(__file__).parent / "doc_test" / "utils" +UTILS = Path(__file__).parent / "fixtures" PYTEST_XML = str(UTILS / "pytest_data.xml") GTEST_XML = str(UTILS / "gtest_data.xml") # carries elements diff --git a/packages/ub-test-reports/tests/test_cli_convert.py b/packages/ub-test-reports/tests/test_cli_convert.py new file mode 100644 index 000000000..8a942a950 --- /dev/null +++ b/packages/ub-test-reports/tests/test_cli_convert.py @@ -0,0 +1,599 @@ +"""Tests for the Sphinx-free ``test-reports build needs`` CLI (TR-A). + +This is the keystone of the build-system story: a test-XML to needs.json +conversion that runs as a build action *outside* Sphinx, so the docs build only +imports the result. Two properties are load-bearing and therefore tested +explicitly rather than assumed: + +* the CLI must not import Sphinx -- otherwise a Bazel action pulls the whole + documentation toolchain into the test-result conversion; +* the output must be byte-stable, because it is a cached build artifact and + qualification evidence. +""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +UTILS = Path(__file__).parent / "fixtures" +GTEST_XML = UTILS / "gtest_data.xml" +PYTEST_XML = UTILS / "pytest_data.xml" + + +def _convert(tmp_path, *args, xml=GTEST_XML): + """Run the converter and return (exit_code, parsed output or None).""" + from ub_test_reports.cli import main + + output = tmp_path / "needs.json" + code = main(["build", "needs", str(xml), "--output", str(output), *args]) + data = json.loads(output.read_text(encoding="utf-8")) if output.exists() else None + return code, data + + +def _needs(data): + version = data["current_version"] + return data["versions"][version]["needs"] + + +class TestNoSphinxImport: + """The converter has to be usable without the documentation toolchain.""" + + def test_importing_the_cli_does_not_import_sphinx(self): + script = ( + "import sys;" + "import ub_test_reports.cli;" + "leaked = sorted(m for m in sys.modules" + " if m == 'sphinx' or m.startswith(('sphinx.', 'sphinx_needs')));" + "print(','.join(leaked))" + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=True, + ) + + assert result.stdout.strip() == "" + + +class TestEnvelope: + def test_envelope_carries_project_and_current_version(self, tmp_path): + _, data = _convert(tmp_path, "--project", "Score Docs-as-Code") + + assert data["project"] == "Score Docs-as-Code" + assert data["current_version"] in data["versions"] + + def test_needs_amount_matches_the_number_of_cases(self, tmp_path): + _, data = _convert(tmp_path) + + version = data["versions"][data["current_version"]] + assert version["needs_amount"] == len(version["needs"]) == 5 + + def test_every_written_field_is_declared(self, tmp_path): + """The file says what its fields are, without a Sphinx build to ask.""" + _, data = _convert(tmp_path) + + version = data["versions"][data["current_version"]] + declared = set(version["needs_schema"]["properties"]) + written = {key for need in version["needs"].values() for key in need} + + assert written - declared == set() + + def test_no_timestamp_is_written(self, tmp_path): + """A wall clock would defeat action caching and evidence diffs.""" + _, data = _convert(tmp_path) + + assert "created" not in data + assert "created" not in data["versions"][data["current_version"]] + + def test_output_is_byte_stable_across_runs(self, tmp_path): + from ub_test_reports.cli import main + + first = tmp_path / "first.json" + second = tmp_path / "second.json" + for output in (first, second): + assert ( + main(["build", "needs", str(GTEST_XML), "--output", str(output)]) == 0 + ) + + assert first.read_bytes() == second.read_bytes() + + +class TestNeedContent: + def test_ids_match_the_deterministic_scheme(self, tmp_path): + from ub_test_reports.identity import deterministic_case_id + + _, data = _convert(tmp_path) + + expected = deterministic_case_id( + classname="MathTest", name="Addition", file="src/math_test.cc" + ) + assert expected in _needs(data) + + def test_source_location_is_emitted_verbatim(self, tmp_path): + _, data = _convert(tmp_path) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["case_file"] == "src/math_test.cc" + assert need["case_line"] == "12" + # `file` is the report path, as in a locally created test-case need. + assert need["file"] == str(GTEST_XML) + + def test_type_and_title_match_the_directive_s(self, tmp_path): + # The directive titles a case need with the case name; a needtable + # title column must not tell an imported case from a built one. + _, data = _convert(tmp_path) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["type"] == "testcase" + assert need["title"] == "Addition" + + def test_result_vocabulary_includes_disabled(self, tmp_path): + _, data = _convert(tmp_path) + + need = _needs(data)["testcase__MathTest__DISABLED_Division_jnyzp"] + assert need["result"] == "disabled" + + def test_content_keeps_every_failure_part(self, tmp_path): + """R2: the debug output has to survive the conversion.""" + _, data = _convert(tmp_path) + + content = _needs(data)["testcase__MathTest__Subtraction_srmht"]["content"] + assert "Expected equality of these values" in content + assert "Actual: false" in content + assert "overflow guard hit" in content + + def test_result_text_is_the_first_failure_message(self, tmp_path): + _, data = _convert(tmp_path) + + need = _needs(data)["testcase__MathTest__Subtraction_srmht"] + assert need["result_text"].startswith("src/math_test.cc:22") + assert "\n" not in need["result_text"] + + def test_named_properties_become_fields(self, tmp_path, capsys): + # Only properties the build would accept (extra_options, or here the + # flag standing in for it) become fields; the rest is reported once. + code, data = _convert(tmp_path, "--no-config", "--extra-option", "TestType") + assert code == 0 + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["TestType"] == "requirements-based" + assert "PartiallyVerifies" not in need + message = capsys.readouterr().err + assert "properties not exported" in message + assert "PartiallyVerifies" in message + assert "extra_options" in message + + def test_unexported_properties_are_reported_once_for_all_names( + self, tmp_path, capsys + ): + # One line naming every left-out property -- not one line per name, + # and not one per case that carries it. + code, _ = _convert(tmp_path, "--no-config") + assert code == 0 + lines = [ + line + for line in capsys.readouterr().err.splitlines() + if "properties not exported" in line + ] + assert len(lines) == 1 + assert "PartiallyVerifies" in lines[0] and "TestType" in lines[0] + + def test_an_exported_property_is_present_on_every_case(self, tmp_path): + # Null where the case has no such property, as the build leaves a + # registered field a directive did not set -- so one schema can + # require the field of imported and locally created needs alike. + _, data = _convert(tmp_path, "--no-config", "--extra-option", "TestType") + needs = _needs(data) + assert needs["testcase__MathTest__Addition_hcuyy"]["TestType"] == ( + "requirements-based" + ) + assert needs["testcase__MathTest__Subtraction_srmht"]["TestType"] is None + assert all("TestType" in need for need in needs.values()) + + def test_unnamed_properties_are_left_out_quietly_when_none_exist( + self, tmp_path, capsys + ): + code, _ = _convert(tmp_path, "--no-config", xml=PYTEST_XML) + assert code == 0 + assert "properties not exported" not in capsys.readouterr().err + + def test_tags_are_configurable(self, tmp_path): + _, data = _convert(tmp_path, "--tags", "TEST") + + assert _needs(data)["testcase__MathTest__Addition_hcuyy"]["tags"] == ["TEST"] + + +class TestLinkProperties: + def test_a_property_can_be_promoted_to_a_link_field(self, tmp_path): + _, data = _convert( + tmp_path, "--link-property", "PartiallyVerifies=partially_verifies" + ) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["partially_verifies"] == ["REQ_1", "REQ_2"] + assert "PartiallyVerifies" not in need + + def test_link_fields_are_always_present_even_when_empty(self, tmp_path): + """A converter must emit its fields unconditionally, so schemas can require them.""" + _, data = _convert( + tmp_path, "--link-property", "PartiallyVerifies=partially_verifies" + ) + + need = _needs(data)["testcase__MathTest__DISABLED_Division_jnyzp"] + assert need["partially_verifies"] == [] + + def test_malformed_link_property_is_rejected(self, tmp_path): + from ub_test_reports.cli import main + + code = main( + [ + "build", + "needs", + str(GTEST_XML), + "--output", + str(tmp_path / "out.json"), + "--link-property", + "NoEqualsSign", + ] + ) + + assert code != 0 + + +class TestRemoteUrls: + def test_external_url_and_remote_url_are_synthesized(self, tmp_path): + _, data = _convert( + tmp_path, + "--remote-url", + "https://github.com/org/repo", + "--commit", + "abc123", + ) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + expected = "https://github.com/org/repo/blob/abc123/src/math_test.cc#L12" + assert need["external_url"] == expected + assert need["remote_url"] == expected + + def test_scp_style_remote_is_normalised(self, tmp_path): + _, data = _convert( + tmp_path, + "--remote-url", + "git@github.com:org/repo.git", + "--commit", + "abc123", + ) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["remote_url"].startswith("https://github.com/org/repo/blob/abc123/") + + def test_url_pattern_is_configurable(self, tmp_path): + _, data = _convert( + tmp_path, + "--remote-url", + "https://gitlab.com/org/repo", + "--commit", + "abc123", + "--url-pattern", + "{base}/-/blob/{commit}/{file}#L{line}", + ) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["remote_url"] == ( + "https://gitlab.com/org/repo/-/blob/abc123/src/math_test.cc#L12" + ) + + def test_credentials_in_the_remote_url_are_not_written(self, tmp_path): + # GitLab's CI_REPOSITORY_URL embeds the job token; the base lands in + # every need of a cached artifact and, imported, in published HTML. + from ub_test_reports.cli import main + + output = tmp_path / "needs.json" + code = main( + [ + "build", + "needs", + str(GTEST_XML), + "--output", + str(output), + "--no-config", + "--remote-url", + "https://gitlab-ci-token:glcbt-secret@gitlab.example.com/org/repo.git", + "--commit", + "abc123", + ] + ) + assert code == 0 + text = output.read_text(encoding="utf-8") + assert "glcbt-secret" not in text and "gitlab-ci-token" not in text + need = _needs(json.loads(text))["testcase__MathTest__Addition_hcuyy"] + assert need["remote_url"] == ( + "https://gitlab.example.com/org/repo/blob/abc123/src/math_test.cc#L12" + ) + + def test_without_repo_metadata_the_url_fields_are_empty(self, tmp_path): + """A hermetic sandbox has no git remote; that must not drop the need.""" + _, data = _convert(tmp_path) + + need = _needs(data)["testcase__MathTest__Addition_hcuyy"] + assert need["remote_url"] == "" + assert need["external_url"] == "" + + +class TestUrlPatternErrors: + """A bad template is a configuration error at the start, not a traceback.""" + + def test_an_unknown_placeholder_is_an_error(self, tmp_path, capsys): + code, data = _convert( + tmp_path, + "--no-config", + "--remote-url", + "https://github.com/o/r", + "--commit", + "abc", + "--url-pattern", + "{base}/blob/{ref}/{file}#L{line}", + ) + assert code == 2 + assert data is None + message = capsys.readouterr().err + assert "--url-pattern" in message + assert "{ref}" in message + + def test_an_unbalanced_brace_is_an_error(self, tmp_path, capsys): + code, _ = _convert( + tmp_path, "--no-config", "--url-pattern", "{base/blob/{commit}/{file}" + ) + assert code == 2 + assert "malformed" in capsys.readouterr().err + + def test_an_attribute_lookup_is_an_error_not_a_traceback(self, tmp_path, capsys): + # str.format resolves {base.__class__}; only KeyError was caught. + code, data = _convert( + tmp_path, + "--no-config", + "--remote-url", + "https://github.com/o/r", + "--commit", + "abc", + "--url-pattern", + "{base.__class__}/{file}", + ) + assert code == 2 + assert data is None + message = capsys.readouterr().err + assert "unknown placeholder {base.__class__}" in message + assert "Traceback" not in message + + def test_the_pattern_is_checked_before_any_report_is_read(self, tmp_path, capsys): + # A missing report and a bad pattern: the pattern error wins, because + # the template is checked before the first file is opened. + code, data = _convert( + tmp_path, + "--no-config", + "--url-pattern", + "{base}/blob/{ref}/{file}", + xml=tmp_path / "does-not-exist.xml", + ) + assert code == 2 + assert data is None + message = capsys.readouterr().err + assert "{ref}" in message + assert "no such file" not in message + + +class TestMultipleInputs: + def test_several_reports_are_merged_into_one_file(self, tmp_path): + from ub_test_reports.cli import main + + output = tmp_path / "needs.json" + code = main( + ["build", "needs", str(GTEST_XML), str(PYTEST_XML), "--output", str(output)] + ) + data = json.loads(output.read_text(encoding="utf-8")) + + assert code == 0 + assert len(_needs(data)) > 5 + + def test_the_same_report_given_twice_is_refused(self, tmp_path, capsys): + # Silently collapsing the repeats would produce a valid file that has + # lost half its evidence -- the worst outcome for a cached artifact. + from ub_test_reports.cli import main + + output = tmp_path / "needs.json" + code = main( + [ + "build", + "needs", + str(GTEST_XML), + str(GTEST_XML), + "--no-config", + "-o", + str(output), + ] + ) + assert code == 2 + assert not output.exists() + message = capsys.readouterr().err + assert "more than once" in message + assert "testcase__" in message + + def test_a_missing_input_file_exits_nonzero(self, tmp_path): + from ub_test_reports.cli import main + + code = main( + [ + "build", + "needs", + str(tmp_path / "nope.xml"), + "--output", + str(tmp_path / "out.json"), + ] + ) + + assert code != 0 + + +class TestDiagnostics: + def test_absent_line_attributes_warn_about_junit_family(self, tmp_path, capsys): + """pytest's default junit_family drops file/line; say so, don't guess.""" + from ub_test_reports.cli import main + + main( + [ + "build", + "needs", + str(PYTEST_XML), + "--output", + str(tmp_path / "out.json"), + ] + ) + + assert "junit_family" in capsys.readouterr().err + + def test_nested_suites_get_the_hint_too(self, tmp_path, capsys): + # The parser files the cases of a nested report under testsuite_nested; + # a hint that only looked at the top level went quiet on exactly the + # Ant/Maven-shaped reports that most often lack source locations. + code, data = _convert( + tmp_path, "--no-config", xml=UTILS / "pytest_nested_example.xml" + ) + assert code == 0 + assert all(need["case_line"] == "" for need in _needs(data).values()) + assert "junit_family" in capsys.readouterr().err + + def test_reports_with_line_attributes_do_not_warn(self, tmp_path, capsys): + from ub_test_reports.cli import main + + main(["build", "needs", str(GTEST_XML), "--output", str(tmp_path / "out.json")]) + + assert "junit_family" not in capsys.readouterr().err + + def test_a_report_without_test_cases_warns(self, tmp_path, capsys): + # A pom.xml parses as one empty suite: a valid, empty needs.json with + # exit 0 is the one outcome a cached build action must never get + # silently. + pom = tmp_path / "pom.xml" + pom.write_text( + "4.0.0\n", encoding="utf-8" + ) + code, data = _convert(tmp_path, "--no-config", xml=pom) + assert code == 0 + assert data["versions"][data["current_version"]]["needs_amount"] == 0 + message = capsys.readouterr().err + assert "pom.xml" in message and "no test cases" in message + + def test_a_report_with_test_cases_does_not_get_the_empty_warning( + self, tmp_path, capsys + ): + code, _ = _convert(tmp_path, "--no-config") + assert code == 0 + assert "no test cases" not in capsys.readouterr().err + + +def test_the_cli_is_runnable_as_a_module(): + result = subprocess.run( + [ + sys.executable, + "-m", + "ub_test_reports.cli", + "build", + "needs", + "--help", + ], + capture_output=True, + text=True, + ) + + assert result.returncode == 0 + assert "--output" in result.stdout + + +def test_console_script_is_installed(): + """The name that goes into a BUILD file has to be a real entry point.""" + script = Path(sys.executable).parent / "test-reports" + if not script.exists(): + pytest.skip("package not installed into this environment") + + result = subprocess.run( + [str(script), "build", "needs", "--help"], capture_output=True, text=True + ) + + assert result.returncode == 0 + assert "--output" in result.stdout + + +@pytest.mark.parametrize("flag", ["--remote-url", "--commit"]) +def test_url_synthesis_needs_both_parts(tmp_path, flag): + """Half the metadata cannot produce a URL; fail loudly instead of guessing.""" + from ub_test_reports.cli import main + + code = main( + [ + "build", + "needs", + str(GTEST_XML), + "--output", + str(tmp_path / "out.json"), + flag, + "value", + ] + ) + + assert code != 0 + + +class TestResultVocabulary: + """The export uses the parser's vocabulary, which is the build's. + + ``failed`` is a documented need field value and a CSS class + (``tr_failed``), and the shipped report template filters on it. A project + that mixes imported and locally created test-case needs filters both with + one expression only if the two writers spell the result alike. + """ + + def test_failure_is_exported_as_the_build_spells_it(self, tmp_path): + _, data = _convert(tmp_path) + + assert ( + _needs(data)["testcase__MathTest__Subtraction_srmht"]["result"] == "failed" + ) + + @pytest.mark.parametrize( + ("need_id", "expected"), + [ + ("testcase__MathTest__Addition_hcuyy", "passed"), + ("testcase__MathTest__DISABLED_Division_jnyzp", "disabled"), + ("testcase__ParamTest_0__Legacy_owuvz", "skipped"), + ], + ) + def test_other_results_are_unchanged(self, tmp_path, need_id, expected): + _, data = _convert(tmp_path) + + assert _needs(data)[need_id]["result"] == expected + + +class TestContentIsNotDuplicated: + """googletest repeats the failure text in the message attribute. + + Emitting both verbatim shows the same stack trace twice in the rendered + need; the message block is only worth its space when it says something the + body does not. + """ + + def test_a_message_contained_in_the_body_is_not_repeated(self, tmp_path): + _, data = _convert(tmp_path) + + content = _needs(data)["testcase__MathTest__Subtraction_srmht"]["content"] + assert content.count("Expected equality of these values") == 1 + assert "message" not in content + + def test_a_message_absent_from_the_body_is_kept(self, tmp_path): + _, data = _convert(tmp_path) + + content = _needs(data)["testcase__ParamTest_0__Legacy_owuvz"]["content"] + assert "Skipped via GTEST_SKIP" in content + assert "not applicable on this platform" in content diff --git a/packages/sphinx-test-reports/tests/test_identity.py b/packages/ub-test-reports/tests/test_identity.py similarity index 99% rename from packages/sphinx-test-reports/tests/test_identity.py rename to packages/ub-test-reports/tests/test_identity.py index d75d05815..a1f73f659 100644 --- a/packages/sphinx-test-reports/tests/test_identity.py +++ b/packages/ub-test-reports/tests/test_identity.py @@ -17,7 +17,7 @@ import pytest -from sphinx_test_reports.identity import ( +from ub_test_reports.identity import ( PLACEHOLDER_FILE, case_display_name, deterministic_case_id, diff --git a/packages/ub-test-reports/tests/test_imports.py b/packages/ub-test-reports/tests/test_imports.py new file mode 100644 index 000000000..a0f8796e3 --- /dev/null +++ b/packages/ub-test-reports/tests/test_imports.py @@ -0,0 +1,66 @@ +"""No module of ub-test-reports names the documentation toolchain, at any depth. + +A static walk, not an import: it reads every module's source, so it refuses an import +statement -- at module level, in a function body, in a ``try`` arm or under +``TYPE_CHECKING`` -- whether or not any test reaches that line, and in every environment, +the default one (which has Sphinx installed) included. ``importlib.import_module("...")`` +and ``__import__("...")`` calls are read too, when the module name is a positional string +literal (a keyword argument, a variable, an f-string or the ``package`` argument are not). +What a static walk cannot see -- a dependency that drags Sphinx in, an import spelled some +other way -- is what CI's ``toolchain-free`` job is for, on the lines a test reaches. +""" + +import ast +from pathlib import Path + +import pytest + +import ub_test_reports + +#: Top-level names this package must never import: the documentation toolchain, and the +#: extension, which depends on this package and not the other way round. +FORBIDDEN = { + "sphinx", + "sphinx_needs", + "docutils", + "sphinx_test_reports", + "sphinxcontrib", +} + +ROOT = Path(ub_test_reports.__file__).parent +MODULES = sorted(ROOT.rglob("*.py")) + + +def _imported_names(node: ast.AST) -> list[str]: + """The absolute module names *node* imports, if it is an import of any spelling.""" + if isinstance(node, ast.Import): + return [alias.name for alias in node.names] + if isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + return [node.module] + if ( + isinstance(node, ast.Call) + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + and ast.unparse(node.func).endswith(("import_module", "__import__")) + ): + return [node.args[0].value] + return [] + + +def test_the_walk_sees_the_whole_package() -> None: + # the fence's fence: a walk of the wrong directory would pass every module vacuously + names = {path.relative_to(ROOT).as_posix() for path in MODULES} + assert {"__init__.py", "cli.py", "jsonparser.py", "pytest_plugin.py"} <= names + + +@pytest.mark.parametrize("path", MODULES, ids=lambda p: p.relative_to(ROOT).as_posix()) +def test_no_module_imports_the_toolchain(path: Path) -> None: + tree = ast.parse(path.read_text(encoding="utf-8")) + leaks = [ + f"{path.name}:{node.lineno}: {name}" + for node in ast.walk(tree) + for name in _imported_names(node) + if name.split(".")[0] in FORBIDDEN + ] + assert leaks == [] diff --git a/packages/ub-test-reports/tests/test_json_parser.py b/packages/ub-test-reports/tests/test_json_parser.py new file mode 100644 index 000000000..56b362e3f --- /dev/null +++ b/packages/ub-test-reports/tests/test_json_parser.py @@ -0,0 +1,143 @@ +"""The JSON parser, called directly: parse, the mapping, result normalisation, errors. + +There is no standard JSON test report, so the parser reads whatever ``tr_json_mapping`` +(or the converter's equivalent) declares: each output key maps to a path into the report +and a default. These tests drive it over the three JSON fixtures -- flat, nested paths, and +custom fields -- without Sphinx; the extension's own suite covers the same reports through a +build. +""" + +from pathlib import Path + +import pytest + +from ub_test_reports.jsonparser import JsonFileMissing, JsonParser, dict_get + +FIXTURES = Path(__file__).parent / "fixtures" + +#: A case's fields as the flat fixture spells them, each read from the key of its name. +CASE_KEYS = [ + "name", + "classname", + "file", + "line", + "time", + "result", + "type", + "text", + "message", + "system-out", +] + + +def _mapping(suite_name=("name",), testcases=("testcase",), extra=None): + """A ``tr_json_mapping`` entry: ``key -> (path into the report, default)``.""" + testcase = {key: ([key], "unknown") for key in CASE_KEYS} + testcase.update(extra or {}) + return { + "testsuite": { + "name": (list(suite_name), "unknown"), + "tests": (["tests"], "unknown"), + "errors": (["errors"], "unknown"), + "failures": (["failures"], "unknown"), + "skips": (["skips"], "unknown"), + "passed": (["passed"], "unknown"), + "time": (["time"], "unknown"), + "testcases": (list(testcases), "unknown"), + }, + "testcase": testcase, + } + + +def _parse(name, mapping): + parser = JsonParser(FIXTURES / name, json_mapping=mapping) + assert parser.validate() is True + return parser.parse() + + +class TestParse: + def test_a_flat_report_gives_one_suite_with_its_cases(self): + (suite,) = _parse("json_data.json", _mapping()) + assert suite["name"] == "test suite 1" + assert (suite["tests"], suite["failures"], suite["skips"]) == (3, 1, 1) + assert suite["testsuite_nested"] == [] + assert [case["name"] for case in suite["testcases"]] == [ + "test case 1", + "test case 2", + "test case 3", + ] + first = suite["testcases"][0] + assert first["classname"] == "class name 1" + assert first["line"] == 123 + assert first["message"] == "all went wrong :( (message)" + + def test_nested_paths_reach_into_the_report(self): + (suite,) = _parse( + "json_complex_data.json", + _mapping( + suite_name=("internals", "name"), testcases=("testcase", "nested") + ), + ) + assert suite["name"] == "test suite 1" + assert [case["classname"] for case in suite["testcases"]] == [ + "class name 1", + "class name 2", + "class name 3", + ] + + def test_custom_fields_take_their_default_where_a_case_lacks_them(self): + extra = { + "id": (["id"], None), + "status": (["status"], "unknown"), + "tags": (["tags"], "unknown"), + } + (suite,) = _parse("json_custom_data.json", _mapping(extra=extra)) + cases = suite["testcases"] + assert [case["id"] for case in cases] == [ + "TEST_CASE_1", + "TEST_CASE_2", + "TEST_CASE_3", + None, + ] + assert cases[0]["tags"] == "a,b,c" + # present but empty is the report's value, not the default + assert cases[2]["status"] == "" + assert cases[3]["status"] == "unknown" + + +class TestResultNormalisation: + """The JSON parser's ``result`` is the JUnit parser's vocabulary.""" + + def test_each_case_result_is_normalised(self): + (suite,) = _parse("json_data.json", _mapping()) + # the report spells the first `failure`, after the JUnit element name + assert [case["result"] for case in suite["testcases"]] == [ + "failed", + "passed", + "skipped", + ] + + def test_a_missing_result_keeps_its_default(self): + mapping = _mapping(extra={"result": (["no_such_key"], None)}) + (suite,) = _parse("json_data.json", mapping) + assert [case["result"] for case in suite["testcases"]] == [None, None, None] + + +class TestErrors: + def test_a_missing_file_is_named(self, tmp_path): + missing = tmp_path / "missing.json" + with pytest.raises(JsonFileMissing, match=r"missing\.json"): + JsonParser(missing, json_mapping=_mapping()) + + @pytest.mark.parametrize( + ("items", "expected"), + [ + (["nested", "a_list", 0, "finally"], "target_data"), + (["nested", "no_such_key"], "default"), # KeyError + (["nested", "a_list", 5], "default"), # IndexError + (["nested", "a_list", "finally"], "default"), # TypeError: list["finally"] + ], + ) + def test_dict_get_falls_back_to_the_default(self, items, expected): + data = {"nested": {"a_list": [{"finally": "target_data"}]}} + assert dict_get(data, items, "default") == expected diff --git a/packages/sphinx-test-reports/tests/test_junit_parser.py b/packages/ub-test-reports/tests/test_junit_parser.py similarity index 70% rename from packages/sphinx-test-reports/tests/test_junit_parser.py rename to packages/ub-test-reports/tests/test_junit_parser.py index 99f3371b2..94fee78ba 100644 --- a/packages/sphinx-test-reports/tests/test_junit_parser.py +++ b/packages/ub-test-reports/tests/test_junit_parser.py @@ -1,28 +1,24 @@ import os -xml_path = os.path.join(os.path.dirname(__file__), "doc_test/utils", "xml_data.xml") -xml_pytest_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "pytest_data.xml" -) +xml_path = os.path.join(os.path.dirname(__file__), "fixtures", "xml_data.xml") +xml_pytest_path = os.path.join(os.path.dirname(__file__), "fixtures", "pytest_data.xml") xml_pytest51_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "pytest_data_5_1.xml" + os.path.dirname(__file__), "fixtures", "pytest_data_5_1.xml" ) xml_pytest62_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "pytest_data_6_2.xml" + os.path.dirname(__file__), "fixtures", "pytest_data_6_2.xml" ) -xml_nose_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "nose_data.xml" -) +xml_nose_path = os.path.join(os.path.dirname(__file__), "fixtures", "nose_data.xml") -xml_ctest_path = os.path.join(os.path.dirname(__file__), "doc_test/utils", "ctest.xml") +xml_ctest_path = os.path.join(os.path.dirname(__file__), "fixtures", "ctest.xml") xml_error_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "xml_data_error.xml" + os.path.dirname(__file__), "fixtures", "xml_data_error.xml" ) def test_init_parser(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_path) @@ -30,7 +26,7 @@ def test_init_parser(): def test_xml_object(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_path) obj = parser.junit_xml_object @@ -40,7 +36,7 @@ def test_xml_object(): def test_parse_easy_xml(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_path) assert hasattr(parser, "parse") @@ -54,7 +50,7 @@ def test_parse_easy_xml(): def test_parse_nosetest_xml(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_nose_path) assert hasattr(parser, "parse") @@ -73,7 +69,7 @@ def test_parse_nosetest_xml(): def test_parse_pytest_xml(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_pytest_path) assert hasattr(parser, "parse") @@ -94,7 +90,7 @@ def test_parse_pytest_xml(): def test_parse_pytest_51_xml(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_pytest51_path) assert hasattr(parser, "parse") @@ -105,7 +101,7 @@ def test_parse_pytest_51_xml(): def test_parse_pytest_61_gets_test_suite_attributes(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_pytest62_path) test_suites = parser.parse() @@ -123,7 +119,7 @@ def test_parse_pytest_61_gets_test_suite_attributes(): def test_parse_ctest_xml(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_ctest_path) test_suites = parser.parse() @@ -154,7 +150,7 @@ def test_parse_ctest_xml(): def test_parse_error_xml(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser parser = JUnitParser(xml_error_path) test_suites = parser.parse() @@ -184,12 +180,12 @@ def test_parse_error_xml(): xml_runner_error_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "runner_error_data.xml" + os.path.dirname(__file__), "fixtures", "runner_error_data.xml" ) def _runner_error_case(name): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser suite = JUnitParser(xml_runner_error_path).parse()[0] return next(case for case in suite["testcases"] if case["name"] == name) @@ -237,10 +233,55 @@ def test_a_passing_testcase_next_to_errors_is_still_passed(): def test_error_counts_are_taken_from_the_testsuite(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser suite = JUnitParser(xml_runner_error_path).parse()[0] assert suite["errors"] == 2 assert suite["failures"] == 0 assert suite["passed"] == 1 + + +#: The smallest report the shipped Apache Ant JUnit schema accepts: every required +#: attribute of `` and ``, and the four child elements in order. +CONFORMING_REPORT = """\ + + + + + + + +""" + + +class TestSchemaValidation: + """`validate()` reads `schemas/JUnit.xsd` from the installed package. + + The schema is package data, so it is what an artefact check must not lose: these + tests are the ones that fail when a built wheel ships without it. + """ + + def test_the_shipped_schema_accepts_a_conforming_report(self, tmp_path): + from pathlib import Path + + from ub_test_reports.junitparser import JUnitParser + + report = tmp_path / "conforming.xml" + report.write_text(CONFORMING_REPORT, encoding="utf-8") + parser = JUnitParser(str(report)) + + assert Path(parser.junit_xsd_path).is_file() + assert parser.validate() is True + + def test_the_shipped_schema_rejects_a_report_without_its_required_attributes( + self, + ): + from ub_test_reports.junitparser import JUnitParser + + # pytest's report has no `hostname` or `timestamp` on its `` + parser = JUnitParser(xml_pytest_path) + + assert parser.validate() is False + assert len(parser.xmlschema.error_log) > 0 diff --git a/packages/sphinx-test-reports/tests/test_junit_parser_gtest.py b/packages/ub-test-reports/tests/test_junit_parser_gtest.py similarity index 96% rename from packages/sphinx-test-reports/tests/test_junit_parser_gtest.py rename to packages/ub-test-reports/tests/test_junit_parser_gtest.py index f6cd0f7fc..928ac076b 100644 --- a/packages/sphinx-test-reports/tests/test_junit_parser_gtest.py +++ b/packages/ub-test-reports/tests/test_junit_parser_gtest.py @@ -17,13 +17,11 @@ import os -xml_gtest_path = os.path.join( - os.path.dirname(__file__), "doc_test/utils", "gtest_data.xml" -) +xml_gtest_path = os.path.join(os.path.dirname(__file__), "fixtures", "gtest_data.xml") def _suites(): - from sphinx_test_reports.junitparser import JUnitParser + from ub_test_reports.junitparser import JUnitParser return JUnitParser(xml_gtest_path).parse() diff --git a/packages/sphinx-test-reports/tests/test_needs_export.py b/packages/ub-test-reports/tests/test_needs_export.py similarity index 99% rename from packages/sphinx-test-reports/tests/test_needs_export.py rename to packages/ub-test-reports/tests/test_needs_export.py index 5ec48e1cb..568f422db 100644 --- a/packages/sphinx-test-reports/tests/test_needs_export.py +++ b/packages/ub-test-reports/tests/test_needs_export.py @@ -7,12 +7,12 @@ import pytest -from sphinx_test_reports.needs_export import ( +from ub_test_reports.needs_export import ( build_content, build_need, build_needs_file, ) -from sphinx_test_reports.remote import ( +from ub_test_reports.remote import ( check_url_pattern, normalise_remote_url, source_url, diff --git a/packages/ub-test-reports/tests/test_project_config.py b/packages/ub-test-reports/tests/test_project_config.py new file mode 100644 index 000000000..b47e3c5e4 --- /dev/null +++ b/packages/ub-test-reports/tests/test_project_config.py @@ -0,0 +1,739 @@ +"""Tests for the declarative configuration (``ubproject.toml``, ``[test_reports]``). + +The loader validates and normalises the section, and the Sphinx build bridges +its keys onto the ``tr_*`` config values. Both must agree on which file +describes a project, or the build is configured by something other than what +the project declares. +""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +import ub_project +from ub_project import ProjectConfigError +from ub_test_reports.projectconfig import ( + BRIDGE_KEYS, + BUILD_TABLE, + DEFAULT_FIELD_NAMES, + DEFAULT_TOML_FILENAME, + TomlConfigError, + field_names, + find_project_config, + load_project_config, + needs_settings, +) + + +def _write(tmp_path, toml_source, name=DEFAULT_TOML_FILENAME): + config = tmp_path / name + config.write_text(toml_source, encoding="utf-8") + return config + + +class TestLoader: + """The Sphinx-free loader: parsing, normalising, anchoring, rejecting.""" + + def test_missing_file_is_none(self, tmp_path): + assert load_project_config(tmp_path / DEFAULT_TOML_FILENAME) is None + + def test_missing_section_is_empty(self, tmp_path): + _write(tmp_path, '[project]\nname = "x"\n') + assert load_project_config(tmp_path / DEFAULT_TOML_FILENAME) == {} + + def test_full_section_round_trips(self, tmp_path): + _write( + tmp_path, + """ + [test_reports] + file_option = "report_file" + source_file_option = "file" + import_encoding = "latin1" + deterministic_case_ids = true + suite_id_length = 4 + extra_options = ["more_info"] + property_link_types = { request = "req" } + """, + ) + config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + assert config["file_option"] == "report_file" + assert config["source_file_option"] == "file" + assert config["import_encoding"] == "latin1" + assert config["deterministic_case_ids"] is True + assert config["suite_id_length"] == 4 + assert config["extra_options"] == ["more_info"] + assert config["property_link_types"] == {"request": "req"} + + def test_build_needs_table_is_validated_but_never_bridged(self, tmp_path): + # [test_reports.build.needs] belongs to the command line. The build + # validates it -- one file, one verdict -- but must not map it onto a + # tr_* value. + _write( + tmp_path, + """ + [test_reports] + file_option = "report_file" + + [test_reports.build.needs] + project = "demo" + tags = ["ci"] + """, + ) + reported = [] + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) + assert reported == [] + assert needs_settings(section) == {"project": "demo", "tags": ["ci"]} + assert BUILD_TABLE not in BRIDGE_KEYS + + @pytest.mark.parametrize( + ("key", "value"), + [ + ("project", "42"), + ("tags", '"ci, unit"'), # a bare string is not an array + ("tags", "[1]"), + ("link_properties", '["a"]'), + ("link_properties", '{ Verifies = ["verifies"] }'), # values too + ], + ) + def test_build_needs_wrong_types_are_rejected(self, tmp_path, key, value): + _write(tmp_path, f"[test_reports.build.needs]\n{key} = {value}\n") + with pytest.raises(TomlConfigError, match=f"build.needs.{key}"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_unknown_artifact_under_build_is_reported_but_not_fatal(self, tmp_path): + # `build` holds one table per artifact the command line produces. A + # newer command may produce one this version does not know, and a file + # naming it must not take the build down. + _write( + tmp_path, + """ + [test_reports.build.needs] + project = "p" + + [test_reports.build.graph] + format = "svg" + """, + ) + reported = [] + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) + assert needs_settings(section) == {"project": "p"} + assert section[BUILD_TABLE] == {"needs": {"project": "p"}} + assert len(reported) == 1 + assert "graph" in reported[0] + assert "[test_reports.build]" in reported[0] + + def test_a_non_table_needs_artifact_is_rejected(self, tmp_path): + _write(tmp_path, '[test_reports.build]\nneeds = "yes"\n') + with pytest.raises(TomlConfigError, match=r"build.needs"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_build_needs_unknown_key_is_reported_but_not_fatal(self, tmp_path): + _write(tmp_path, "[test_reports.build.needs]\nprojct = 'typo'\nproject = 'p'\n") + reported = [] + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) + assert needs_settings(section) == {"project": "p"} + assert len(reported) == 1 + assert "projct" in reported[0] + assert "[test_reports.build.needs]" in reported[0] + + def test_need_type_and_case_type_must_agree(self, tmp_path): + # The converter takes the need type (and the deterministic-ID prefix) + # from build.needs.need_type, the build from case's type. Disagreeing + # produces a needs.json the build neither registers nor cross-links. + _write( + tmp_path, + """ + [test_reports.build.needs] + need_type = "testcase" + + [test_reports.case] + directive = "test-case" + type = "check" + name = "Check" + prefix = "CH_" + color = "#999999" + style = "rectangle" + """, + ) + with pytest.raises(TomlConfigError, match="need_type"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_a_missing_side_is_compared_at_its_default(self, tmp_path): + # A customised case next to a convert table without need_type is a + # disagreement too: the converter would write the default type. + _write( + tmp_path, + """ + [test_reports.build.needs] + project = "p" + + [test_reports.case] + directive = "test-case" + type = "check" + name = "Check" + prefix = "CH_" + color = "#999999" + style = "rectangle" + """, + ) + with pytest.raises(TomlConfigError, match=r"'testcase'.*'check'"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + # ... and the mirror: need_type set, case left at its default. + _write(tmp_path, '[test_reports.build.needs]\nneed_type = "check"\n') + with pytest.raises(TomlConfigError, match=r"'check'.*'testcase'"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_without_a_convert_table_the_case_type_is_free(self, tmp_path): + # A project that only builds may name its case type as it likes. + _write( + tmp_path, + """ + [test_reports.case] + directive = "test-case" + type = "check" + name = "Check" + prefix = "CH_" + color = "#999999" + style = "rectangle" + """, + ) + assert ( + load_project_config(tmp_path / DEFAULT_TOML_FILENAME)["case"][1] == "check" + ) + + def test_empty_link_property_names_are_rejected_by_the_loader(self, tmp_path): + # One verdict for both consumers: the build refuses what the converter + # would refuse. + _write( + tmp_path, + '[test_reports.build.needs]\nlink_properties = { Verifies = "" }\n', + ) + with pytest.raises(TomlConfigError, match="link_properties"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_field_names_default_to_the_build_s(self, tmp_path): + _write(tmp_path, "[test_reports]\nsource_file_option = 'src'\n") + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + names = field_names(section) + assert names["source_file_option"] == "src" + assert names["file_option"] == DEFAULT_FIELD_NAMES["file_option"] == "file" + assert names["source_line_option"] == "case_line" + + def test_colliding_field_names_are_rejected(self, tmp_path): + # file (report path) and file (source path) cannot share a field. + _write(tmp_path, "[test_reports]\nsource_file_option = 'file'\n") + with pytest.raises(TomlConfigError, match="both name the need field 'file'"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + @pytest.mark.parametrize( + ("key", "name"), + [ + ("file_option", "case"), + ("source_file_option", "result"), + ("source_line_option", "id"), + ], + ) + def test_a_rename_onto_a_fixed_field_is_rejected(self, tmp_path, key, name): + # Every test-case need has these already: the directives would pass + # the keyword twice, the converter overwrite one value with the other. + _write(tmp_path, f"[test_reports]\n{key} = '{name}'\n") + with pytest.raises(TomlConfigError, match=f"{key} = '{name}'"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_need_type_and_case_type_agreeing_is_fine(self, tmp_path): + _write( + tmp_path, + """ + [test_reports.build.needs] + need_type = "check" + + [test_reports.case] + directive = "test-case" + type = "check" + name = "Check" + prefix = "CH_" + color = "#999999" + style = "rectangle" + """, + ) + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + assert section["case"][1] == needs_settings(section)["need_type"] == "check" + + def test_unknown_key_is_reported_but_not_fatal(self, tmp_path): + # ubproject.toml is shared with tools on independent release cadences, + # so a key this reader does not model must not take the build down -- + # but a typo has to be visible, and the key must not be passed on. + _write(tmp_path, "[test_reports]\ndeterministic_id = true\n") + reported = [] + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME, reported.append) + assert section == {} + assert len(reported) == 1 + assert "deterministic_id" in reported[0] + assert "deterministic_case_ids" in reported[0] # supported keys listed + + def test_unknown_key_needs_no_reporter(self, tmp_path): + _write(tmp_path, "[test_reports]\nnope = 1\nfile_option = 'f'\n") + assert load_project_config(tmp_path / DEFAULT_TOML_FILENAME) == { + "file_option": "f" + } + + @pytest.mark.parametrize( + ("key", "value"), + [ + ("property_link_types", '{ request = ["req"] }'), + ("property_link_types", "{ request = 3 }"), + ], + ) + def test_table_values_are_type_checked(self, tmp_path, key, value): + # Without this the value reaches the directives, which fail with a bare + # TypeError on an unhashable field name instead of a config error. + _write(tmp_path, f"[test_reports]\n{key} = {value}\n") + with pytest.raises(TomlConfigError, match=key): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_json_mapping_nesting_stays_free_form(self, tmp_path): + # It mirrors an arbitrary parser mapping, so only the outer table is + # checked -- validating deeper would reject valid configurations. + _write( + tmp_path, + "[test_reports.json_mapping.json_config.testsuite]\nname = 1\n", + ) + section = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + assert section["json_mapping"] == {"json_config": {"testsuite": {"name": 1}}} + + @pytest.mark.skipif( + hasattr(os, "geteuid") and os.geteuid() == 0, + reason="root reads unreadable files", + ) + @pytest.mark.skipif( + sys.platform == "win32", + reason="os.chmod on Windows only sets the read-only attribute; the file stays readable", + ) + def test_unreadable_file_is_a_config_error(self, tmp_path): + # is_file() succeeding does not mean the open will; an unwrapped + # OSError would surface as a traceback instead of a config error. + config = _write(tmp_path, "[test_reports]\nfile_option = 'f'\n") + config.chmod(0o000) + try: + with pytest.raises(TomlConfigError, match="cannot be read"): + load_project_config(config) + finally: + config.chmod(0o644) + + @pytest.mark.parametrize( + ("key", "value"), + [ + ("file_option", "42"), + ("suite_id_length", '"four"'), # string for int + ("suite_id_length", "true"), # bool must not pass for int + ("deterministic_case_ids", '"yes"'), # string for bool + ("extra_options", '"more_info"'), # a bare string is not an array + ("property_link_types", '["a"]'), + ], + ) + def test_wrong_types_are_rejected(self, tmp_path, key, value): + _write(tmp_path, f"[test_reports]\n{key} = {value}\n") + with pytest.raises(TomlConfigError, match=key): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_invalid_toml_is_rejected(self, tmp_path): + _write(tmp_path, "[test-reports\n") + with pytest.raises(TomlConfigError, match="invalid TOML"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_section_must_be_a_table(self, tmp_path): + _write(tmp_path, "test_reports = 5\n") + with pytest.raises(TomlConfigError, match="must be a table"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_need_type_positional_list_still_works(self, tmp_path): + # The conf.py spelling, so existing projects can copy their lists over + # verbatim. + _write( + tmp_path, + '[test_reports]\ncase = ["test-case", "testcase", "Test-Case", "TC_", "#999999", "rectangle"]\n', + ) + config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + assert config["case"] == [ + "test-case", + "testcase", + "Test-Case", + "TC_", + "#999999", + "rectangle", + ] + + def test_need_type_named_table(self, tmp_path): + # Six bare strings cannot be told apart; the table spelling names them. + _write( + tmp_path, + """ + [test_reports.case] + directive = "test-case" + type = "testcase" + name = "Test-Case" + prefix = "TC_" + color = "#999999" + style = "rectangle" + """, + ) + config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + assert config["case"] == [ + "test-case", + "testcase", + "Test-Case", + "TC_", + "#999999", + "rectangle", + ] + + def test_need_type_table_rejects_partial_and_unknown(self, tmp_path): + _write(tmp_path, '[test_reports.case]\ndirective = "test-case"\n') + with pytest.raises(TomlConfigError, match="missing"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + _write( + tmp_path, + """ + [test_reports.case] + directive = "test-case" + type = "testcase" + name = "Test-Case" + prefix = "TC_" + color = "#999999" + style = "rectangle" + typo = true + """, + ) + with pytest.raises(TomlConfigError, match="unknown typo"): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + @pytest.mark.parametrize( + ("toml_source", "problem"), + [ + # A non-string value inside the named table. Without the check the + # value is silently stringified and reaches sphinx-needs. + ( + """ + [test_reports.case] + directive = "test-case" + type = "testcase" + name = "Test-Case" + prefix = "TC_" + color = 999999 + style = "rectangle" + """, + "non-string color", + ), + # A non-string element of the positional list. + ( + '[test_reports]\ncase = ["test-case", "testcase", "Test-Case", "TC_", 999999, "rectangle"]\n', + "exactly 6 strings", + ), + # Too few elements. + ('[test_reports]\ncase = ["test-case", "testcase"]\n', "exactly 6 strings"), + ], + ) + def test_need_type_values_must_be_strings(self, tmp_path, toml_source, problem): + _write(tmp_path, toml_source) + with pytest.raises(TomlConfigError, match=problem): + load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + + def test_relative_paths_anchor_to_the_toml_directory(self, tmp_path): + # The file is self-describing: moving it as a unit keeps its relative + # paths meaningful, and both consumers resolve them identically. + subdir = tmp_path / "config" + subdir.mkdir() + _write( + subdir, + '[test_reports]\nrootdir = "docs"\nreport_template = "templates/report.txt"\n', + name=subdir / DEFAULT_TOML_FILENAME, + ) + config = load_project_config(subdir / DEFAULT_TOML_FILENAME) + assert config["rootdir"] == str(subdir / "docs") + assert config["report_template"] == str(subdir / "templates" / "report.txt") + + @pytest.mark.parametrize( + "suffix", + [ + pytest.param("", id="plain"), + # the two forms ``Path`` normalises away: a round trip through it + # would return ``/a/b`` for ``/a/b/`` and ``/a/b`` for ``/a//b``, + # so these are the cases that tell "left as the string it was" + # from "anchored, and absolute already" + pytest.param(os.sep, id="trailing-separator"), + pytest.param(f"{os.sep}{os.sep}x", id="doubled-separator"), + ], + ) + def test_absolute_paths_stay_untouched(self, tmp_path, suffix): + # a TOML literal string: in a basic string a Windows path's backslashes + # are escape sequences ("\U" starts a unicode escape) and the file is invalid + value = f"{tmp_path}{suffix}" + _write(tmp_path, f"[test_reports]\nrootdir = '{value}'\n") + config = load_project_config(tmp_path / DEFAULT_TOML_FILENAME) + assert config["rootdir"] == value + + +def _not_utf8(tmp_path): + """A file saved in Latin-1: ``é`` is the lone byte 0xE9, which UTF-8 refuses.""" + config = tmp_path / DEFAULT_TOML_FILENAME + config.write_bytes("[test_reports]\nfile_option = 'café'\n".encode("latin-1")) + return config + + +class TestSharedReaderBoundary: + """ub-project reads the file; its exception never leaves this package. + + Both consumers catch :class:`TomlConfigError` and nothing else, so a + ``ProjectConfigError`` escaping the loader would reach the user as a + traceback. Each case asserts the exact type and that the message is the + shared reader's, word for word. + """ + + def _assert_re_raised(self, config): + with pytest.raises(TomlConfigError) as caught: + load_project_config(config) + assert type(caught.value) is TomlConfigError + cause = caught.value.__cause__ + assert isinstance(cause, ProjectConfigError) + assert str(caught.value) == str(cause) + return str(caught.value) + + def test_the_two_exceptions_are_unrelated(self): + # a subclass either way round would put ub-project's exception on this + # package's public surface + assert not issubclass(TomlConfigError, ProjectConfigError) + assert not issubclass(ProjectConfigError, TomlConfigError) + + def test_the_walk_is_the_shared_reader_s_own(self): + # re-exported, not copied: a local fork of the walk would pass every + # discovery test and drift from the reader the other members use + assert find_project_config is ub_project.find_project_config + + def test_invalid_toml(self, tmp_path): + message = self._assert_re_raised(_write(tmp_path, "[test-reports\n")) + assert message.startswith(f"{tmp_path / DEFAULT_TOML_FILENAME}: invalid TOML: ") + + @pytest.mark.skipif( + hasattr(os, "geteuid") and os.geteuid() == 0, + reason="root reads unreadable files", + ) + @pytest.mark.skipif( + sys.platform == "win32", + reason="os.chmod on Windows only sets the read-only attribute; the file stays readable", + ) + def test_unreadable_file(self, tmp_path): + config = _write(tmp_path, "[test_reports]\nfile_option = 'f'\n") + config.chmod(0o000) + try: + message = self._assert_re_raised(config) + finally: + config.chmod(0o644) + assert message.startswith(f"{config}: cannot be read: ") + + def test_a_file_that_is_not_utf8(self, tmp_path): + # New with ub-project: before it, the decode error escaped the loader + # as a bare UnicodeDecodeError. + config = _not_utf8(tmp_path) + message = self._assert_re_raised(config) + assert message.startswith(f"{config}: not valid UTF-8 TOML: ") + + +class TestDiscovery: + """The upward search that lets both consumers find the same file. + + The search is bounded by the repository root -- the directory holding + ``.git``: a ``pyproject.toml`` on the way up marks a Python distribution, + not the project, and must not end the search. Outside any repository there + is no such root, so the distribution root bounds it instead -- otherwise + the walk reaches the filesystem root and adopts a stranger's file. + """ + + def test_finds_the_file_in_the_starting_directory(self, tmp_path): + config = _write(tmp_path, "[test_reports]\n") + assert find_project_config(tmp_path) == config + + def test_walks_up_to_the_repository_root(self, tmp_path): + (tmp_path / ".git").mkdir() + config = _write(tmp_path, "[test_reports]\n") + deep = tmp_path / "docs" / "source" + deep.mkdir(parents=True) + assert find_project_config(deep) == config + + def test_a_pyproject_toml_beside_conf_py_does_not_end_the_search(self, tmp_path): + # docs/ carrying its own pyproject.toml (its own dependency set) still + # belongs to the project whose shared file sits at the repository root. + (tmp_path / ".git").mkdir() + config = _write(tmp_path, "[test_reports]\n") + docs = tmp_path / "docs" + docs.mkdir() + (docs / "pyproject.toml").write_text("", encoding="utf-8") + assert find_project_config(docs) == config + + def test_walks_past_a_workspace_member_pyproject_toml(self, tmp_path): + # A uv-workspace member: packages//pyproject.toml with the docs + # below it, and one ubproject.toml at the repository root describing + # the whole monorepo. + (tmp_path / ".git").mkdir() + config = _write(tmp_path, "[test_reports]\n") + member = tmp_path / "packages" / "dist" + docs = member / "docs" + docs.mkdir(parents=True) + (member / "pyproject.toml").write_text("", encoding="utf-8") + assert find_project_config(docs) == config + + def test_stops_at_a_nested_repository_without_the_file(self, tmp_path): + # A checkout nested inside another repository (a vendored tree, a + # submodule) must not adopt the outer repository's configuration. + _write(tmp_path, "[test_reports]\n") + inner = tmp_path / "vendor" / "inner" + docs = inner / "docs" + docs.mkdir(parents=True) + (inner / ".git").write_text("gitdir: elsewhere\n", encoding="utf-8") + assert find_project_config(docs) is None + + def test_stops_at_the_distribution_root_without_a_repository(self, tmp_path): + # An unpacked sdist, a CI artefact directory, an exported docs tree: + # no .git anywhere, so nothing above would end the walk and a + # stranger's file further up would be adopted. The distribution root + # bounds the search instead, so it is not. + _write(tmp_path, "[test_reports]\n") # a stranger's, two levels up + dist = tmp_path / "downloads" / "sphinx-test-reports-1.4.0" + docs = dist / "docs" + docs.mkdir(parents=True) + (dist / "pyproject.toml").write_text("", encoding="utf-8") + assert find_project_config(docs) is None + + def test_the_file_at_the_distribution_root_is_still_found(self, tmp_path): + # The distribution root bounds the search without hiding a file that + # sits on it: an sdist shipping its own ubproject.toml is configured + # by it. + dist = tmp_path / "sphinx-test-reports-1.4.0" + docs = dist / "docs" + docs.mkdir(parents=True) + (dist / "pyproject.toml").write_text("", encoding="utf-8") + config = _write(dist, "[test_reports]\n") + assert find_project_config(docs) == config + + def test_a_repository_marker_outranks_a_distribution_root(self, tmp_path): + # The distribution root is only the fallback boundary. Inside a + # repository the walk still passes a pyproject.toml on the way up -- + # the workspace-member layout above depends on it. + (tmp_path / ".git").mkdir() + config = _write(tmp_path, "[test_reports]\n") + member = tmp_path / "packages" / "dist" + docs = member / "docs" + docs.mkdir(parents=True) + (member / "pyproject.toml").write_text("", encoding="utf-8") + assert find_project_config(docs) == config + + def test_a_fruitless_search_reports_the_distribution_root(self, tmp_path): + dist = tmp_path / "sphinx-test-reports-1.4.0" + docs = dist / "docs" + docs.mkdir(parents=True) + (dist / "pyproject.toml").write_text("", encoding="utf-8") + reported = [] + assert find_project_config(docs, report=reported.append) is None + assert len(reported) == 1 + assert f"distribution root {dist}" in reported[0] + assert "pyproject.toml" in reported[0] + + def test_the_file_wins_over_the_marker_in_one_directory(self, tmp_path): + # The root marker only ends a *fruitless* step; a repository root + # holding the file is the canonical layout and must be found. + config = _write(tmp_path, "[test_reports]\n") + (tmp_path / ".git").mkdir() + assert find_project_config(tmp_path) == config + + def test_missing_file_is_none(self, tmp_path): + (tmp_path / ".git").mkdir() + assert find_project_config(tmp_path) is None + + def test_a_fruitless_search_reports_where_it_ended(self, tmp_path): + # "Not found" must not be silent: the report names the directory whose + # marker ended the search, so a misplaced file can be diagnosed. + (tmp_path / ".git").mkdir() + docs = tmp_path / "docs" + docs.mkdir() + reported = [] + assert find_project_config(docs, report=reported.append) is None + assert len(reported) == 1 + assert str(docs) in reported[0] + assert f"repository root {tmp_path}" in reported[0] + assert ".git" in reported[0] + + def test_a_successful_search_reports_nothing(self, tmp_path): + (tmp_path / ".git").mkdir() + _write(tmp_path, "[test_reports]\n") + reported = [] + find_project_config(tmp_path / "docs", report=reported.append) + assert reported == [] + + def test_a_relative_start_is_searched_from_the_working_directory( + self, tmp_path, monkeypatch + ): + # A converter started with a relative path must still see the parents. + (tmp_path / ".git").mkdir() + config = _write(tmp_path, "[test_reports]\n") + docs = tmp_path / "docs" + docs.mkdir() + monkeypatch.chdir(docs) + assert find_project_config(Path(".")) == config + + def test_a_symlinked_start_walks_the_link_s_parents(self, tmp_path): + # The start is made absolute WITHOUT resolving: a symlinked docs/ + # belongs to the repository it is linked into, not to the one its + # target lives in. The only case here that tells the two apart -- + # tmp_path is already resolved, so every other start is too. + repo = tmp_path / "repo" + (repo / ".git").mkdir(parents=True) + config = _write(repo, "[test_reports]\n") + elsewhere = tmp_path / "elsewhere" + (elsewhere / ".git").mkdir(parents=True) + target = elsewhere / "docs" + target.mkdir() + link = repo / "docs" + try: + link.symlink_to(target, target_is_directory=True) + except OSError: # Windows without the symlink privilege + pytest.skip("creating a symlink needs a privilege this account lacks") + assert find_project_config(link) == config + + +class TestPathAnchoring: + """Relative paths anchor at the TOML file's directory, as given.""" + + def test_anchoring_does_not_resolve_the_given_directory(self, tmp_path): + # The loader leaves the form of the directory it was handed alone -- a + # symlinked path stays symlinked. Whether to resolve it is the + # consumer's call (Sphinx resolves its confdir before the bridge runs), + # not something the loader decides behind its back. + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real, target_is_directory=True) + config = _write(link, "[test_reports]\nrootdir = 'reports'\n") + section = load_project_config(config) + assert section["rootdir"] == str(link / "reports") + + +class TestSphinxFree: + """A consumer without the documentation toolchain can read the section.""" + + def test_projectconfig_imports_without_sphinx(self): + # Importing the module runs the package __init__, so the package must + # not import Sphinx eagerly either -- or a build action that turns + # reports into a needs.json dies with ModuleNotFoundError wherever + # Sphinx is not installed. Checked in a subprocess: this process has + # Sphinx imported already. + code = ( + "import sys\n" + "sys.modules['sphinx'] = None\n" # any `import sphinx...` now fails + "import ub_test_reports.projectconfig\n" + ) + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True + ) + assert result.returncode == 0, result.stderr diff --git a/packages/sphinx-test-reports/tests/test_pytest_plugin.py b/packages/ub-test-reports/tests/test_pytest_plugin.py similarity index 90% rename from packages/sphinx-test-reports/tests/test_pytest_plugin.py rename to packages/ub-test-reports/tests/test_pytest_plugin.py index e05d500e5..c6d32ed74 100644 --- a/packages/sphinx-test-reports/tests/test_pytest_plugin.py +++ b/packages/ub-test-reports/tests/test_pytest_plugin.py @@ -5,14 +5,15 @@ ``test_reports_properties`` ini option; S-CORE's is the profile most tests use. """ +import re import subprocess import sys import xml.etree.ElementTree as ET import pytest -from sphinx_test_reports import pytest_plugin -from sphinx_test_reports.pytest_plugin import ( +from ub_test_reports import pytest_plugin +from ub_test_reports.pytest_plugin import ( Property, apply_test_metadata, clean_source_path, @@ -20,7 +21,7 @@ properties_mapping, ) -PLUGIN = "sphinx_test_reports.pytest_plugin" +PLUGIN = "ub_test_reports.pytest_plugin" #: S-CORE's model, as the docs show it: the profile most of these tests run with. SCORE_PROFILE = """\ @@ -32,7 +33,7 @@ """ DECORATED = """ -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @add_test_properties( partially_verifies=["REQ_1", "REQ_2"], @@ -50,7 +51,7 @@ def test_plain(): RUNTIME = """ import pytest -from sphinx_test_reports.pytest_plugin import apply_test_metadata +from ub_test_reports.pytest_plugin import apply_test_metadata @pytest.mark.parametrize("spec", ["a.rst", "b.rst"]) def test_driven_by_a_file(spec, record_property): @@ -64,7 +65,7 @@ def test_driven_by_a_file(spec, record_property): """ RUNTIME_COMPAT = """ -from sphinx_test_reports.pytest_plugin import apply_test_metadata +from ub_test_reports.pytest_plugin import apply_test_metadata def test_score_style(record_property, record_xml_attribute): apply_test_metadata( @@ -78,7 +79,7 @@ def test_score_style(record_property, record_xml_attribute): SKIPPED = """ import pytest -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @pytest.fixture(scope="module") @@ -118,7 +119,7 @@ def test_runs(): # derives its own fixture from this string by replacing `str(inner)]) == 0`. NESTED = """ import pytest -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @add_test_properties(partially_verifies=["REQ_1"]) @@ -154,7 +155,7 @@ def test_strict(): """ STACKED = """ -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @add_test_properties(test_type="requirements-based", Owner="team-a") @@ -175,7 +176,7 @@ def test_stacked(): """ CUSTOM = """ -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @add_test_properties(satisfies=["REQ_1", "REQ_2"], reviewers=["ann", "bob"], Owner="x") def test_custom(): @@ -337,6 +338,27 @@ def test_xunit2_is_warned_about_at_configure_time(self, pytester): result.stdout.fnmatch_lines(["*junit_family is 'xunit2'*xunit1*"]) assert _cases(root)["test_plain"].get("file") is None + def test_the_names_it_prints_are_ub_test_reports(self, pytester): + # Both are documented in the changelog as changed from 2.0.0's + # `sphinxcontrib.test_reports...`: the prefix of the warnings it issues (what a + # user's warning filter matches) and the name its hook object is registered under. + source = ( + "def test_registered(request):\n" + " assert request.config.pluginmanager.has_plugin('ub_test_reports.xml_shape')\n" + ) + result, _ = _run(pytester, source, family="xunit2") + result.assert_outcomes(passed=1) + messages = [ + line.split("TestReportsConfigWarning: ", 1)[1] + for line in result.stdout.lines + if "TestReportsConfigWarning: " in line + ] + assert messages + assert all( + re.match(r"^ub_test_reports\.pytest_plugin: ", message) + for message in messages + ), messages + class TestPropertyModel: """The model is pytest configuration; the plugin ships no names of its own.""" @@ -412,7 +434,7 @@ def test_a_decorator_may_run_before_configuration_is_read(self, pytester): # nor loses the properties, which are written at setup. pytester.makepyfile( helpers=( - "from sphinx_test_reports.pytest_plugin import add_test_properties\n" + "from ub_test_reports.pytest_plugin import add_test_properties\n" "\n" '@add_test_properties(partially_verifies=["REQ_1"], test_type="interface-test")\n' "def test_from_helper():\n" @@ -558,10 +580,37 @@ def test_the_location_is_applied_without_metadata(self, pytester): assert _properties(case) == {} assert (case.get("file"), case.get("line")) == ("specs/a.rst", "7") + def test_the_location_travels_under_the_documented_wire_names(self, score_model): + # The two names are a documented wire format, not the import path: they did not + # move when the plugin did, and must not change. + recorded = [] + apply_test_metadata( + record_property=lambda name, value: recorded.append((name, value)), + metadata={}, + file="specs/a.rst", + line=7, + ) + assert recorded == [ + ("sphinxcontrib.test_reports:file", "specs/a.rst"), + ("sphinxcontrib.test_reports:line", "7"), + ] + + def test_a_property_under_a_wire_name_is_read_as_the_location(self, pytester): + source = ( + "def test_direct(record_property):\n" + " record_property('sphinxcontrib.test_reports:file', 'specs/b.rst')\n" + " record_property('sphinxcontrib.test_reports:line', '3')\n" + ) + result, root = _run(pytester, source) + result.assert_outcomes(passed=1) + case = _cases(root)["test_direct"] + assert _properties(case) == {} + assert (case.get("file"), case.get("line")) == ("specs/b.rst", "3") + BAD_SHAPE_AND_BROKEN_FIXTURE = """ import pytest -from sphinx_test_reports.pytest_plugin import add_test_properties +from ub_test_reports.pytest_plugin import add_test_properties @pytest.fixture @@ -579,7 +628,7 @@ class TestBadShape: def test_a_bad_shape_errors_the_case_at_setup(self, pytester): result, _root = _run( pytester, - "from sphinx_test_reports.pytest_plugin import add_test_properties\n\n@add_test_properties(test_type=['a', 'b'])\ndef test_shape():\n assert True\n", + "from ub_test_reports.pytest_plugin import add_test_properties\n\n@add_test_properties(test_type=['a', 'b'])\ndef test_shape():\n assert True\n", ) result.assert_outcomes(errors=1) result.stdout.fnmatch_lines(["*TypeError*'test_type' takes a single value*"]) diff --git a/packages/ub-test-reports/tests/test_result_vocabulary.py b/packages/ub-test-reports/tests/test_result_vocabulary.py new file mode 100644 index 000000000..cc0ae09b5 --- /dev/null +++ b/packages/ub-test-reports/tests/test_result_vocabulary.py @@ -0,0 +1,151 @@ +"""The ``result`` field's vocabulary, and the one place it is decided. + +A test case's ``result`` used to be whatever the input spelled it: the JUnit +parser passed the name of the ```` child element through verbatim +(``failure``), the JSON parser passed the report's own value through, and the +two states with no element of their own -- ``passed`` and ``disabled`` -- were +spelled as participles because nothing forced a choice. So one product said +``failure`` for a single case and ``failed`` for the count of them +(``fields.FIELDS``), and the documented value and the documented example +disagreed. + +These tests pin the vocabulary down: every parser maps its input onto the same +participles, and the mapping is the only place that decides. The part-level +``kind`` is deliberately *not* normalised -- it names the XML element the +evidence came from, which is what the rendered evidence heading reports. +""" + +import os + +import pytest + +UTILS = os.path.join(os.path.dirname(__file__), "fixtures") +XML_PATH = os.path.join(UTILS, "xml_data.xml") +JSON_PATH = os.path.join(UTILS, "json_data.json") + +#: The mapping the JSON parser needs, as ``tr_json_mapping`` declares it. +JSON_MAPPING = { + "testsuite": { + "name": (["name"], "unknown"), + "tests": (["tests"], "unknown"), + "errors": (["errors"], "unknown"), + "failures": (["failures"], "unknown"), + "skips": (["skips"], "unknown"), + "passed": (["passed"], "unknown"), + "time": (["time"], "unknown"), + "testcases": (["testcase"], "unknown"), + }, + "testcase": { + "name": (["name"], "unknown"), + "classname": (["classname"], "unknown"), + "file": (["file"], "unknown"), + "line": (["line"], "unknown"), + "time": (["time"], "unknown"), + "result": (["result"], "unknown"), + "type": (["type"], "unknown"), + "text": (["text"], "unknown"), + "message": (["message"], "unknown"), + "system-out": (["system-out"], "unknown"), + }, +} + + +class TestNormalisation: + """One function decides the vocabulary, so both parsers cannot disagree.""" + + def test_the_junit_failure_element_name_becomes_failed(self): + from ub_test_reports.results import normalize_result + + assert normalize_result("failure") == "failed" + + def test_normalising_the_canonical_spelling_changes_nothing(self): + """Normalisation runs on already-canonical values too, so it must be + idempotent -- a JSON report may already spell the result ``failed``.""" + from ub_test_reports.results import normalize_result + + assert normalize_result("failed") == "failed" + + @pytest.mark.parametrize("result", ["passed", "skipped", "error", "disabled"]) + def test_the_other_states_are_already_canonical(self, result): + from ub_test_reports.results import normalize_result + + assert normalize_result(result) == result + + def test_a_vocabulary_this_extension_does_not_know_is_left_alone(self): + """``tr_json_mapping`` points at an arbitrary report, so a project may + feed in states of its own. Rewriting those would break its filters.""" + from ub_test_reports.results import normalize_result + + assert normalize_result("flaky") == "flaky" + + def test_the_canonical_states_are_the_documented_ones(self): + """Ordered, because the declared field description is built from it and + the converter's output has to be byte-stable.""" + from ub_test_reports.results import CANONICAL_RESULTS + + assert CANONICAL_RESULTS == ( + "passed", + "failed", + "error", + "skipped", + "disabled", + ) + + +class TestDeclaredSchema: + """The converter writes the field declarations into the ``needs.json`` it + produces, so that a consumer which never loads this extension -- a schema + check, a metamodel validator -- learns the fields from the file. Naming the + states in the ``result`` description tells it the field's domain too. + """ + + def test_the_result_declaration_names_every_state(self): + from ub_test_reports.fields import declaration + from ub_test_reports.results import CANONICAL_RESULTS + + _, description = declaration("result") + + assert all(state in description for state in CANONICAL_RESULTS), description + + +class TestJUnitParser: + """The JUnit dialect is where the old spelling came from.""" + + def test_a_failure_child_yields_the_failed_result(self): + from ub_test_reports.junitparser import JUnitParser + + suite = JUnitParser(XML_PATH).parse()[0] + + assert suite["testcases"][2]["result"] == "failed" + + def test_a_result_part_keeps_the_name_of_its_xml_element(self): + """``kind`` reports which element the evidence came from -- the + converter capitalises it into the evidence heading -- so it stays the + XML name even though ``result`` no longer is.""" + from ub_test_reports.junitparser import JUnitParser + + suite = JUnitParser(XML_PATH).parse()[0] + + assert suite["testcases"][2]["parts"][0]["kind"] == "failure" + + +class TestJsonParser: + """The JSON parser's API is documented as being in sync with the JUnit + parser's, so the same report content has to produce the same result.""" + + def test_the_failure_spelling_in_a_json_report_is_normalised(self): + from ub_test_reports.jsonparser import JsonParser + + parser = JsonParser(JSON_PATH, json_mapping=JSON_MAPPING) + suite = parser.parse()[0] + + assert suite["testcases"][0]["result"] == "failed" + + def test_the_results_needing_no_normalisation_are_untouched(self): + from ub_test_reports.jsonparser import JsonParser + + parser = JsonParser(JSON_PATH, json_mapping=JSON_MAPPING) + suite = parser.parse()[0] + + assert suite["testcases"][1]["result"] == "passed" + assert suite["testcases"][2]["result"] == "skipped" diff --git a/pyproject.toml b/pyproject.toml index 89fcd2068..2f77cddf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "sphinx-mounts", "sphinx-codelinks", "sphinx-test-reports", + "ub-test-reports", "ub-project", "sphinx-needs-workspace-tools", ] @@ -68,6 +69,7 @@ sphinx-needs = { workspace = true } sphinx-mounts = { workspace = true } sphinx-codelinks = { workspace = true } sphinx-test-reports = { workspace = true } +ub-test-reports = { workspace = true } ub-project = { workspace = true } sphinx-needs-workspace-tools = { workspace = true } sphinx-needs-testkit = { workspace = true } @@ -257,11 +259,6 @@ markers = [ "jstest: marks tests as JavaScript test (deselect with '-m \"not jstest\"')", "benchmark: marks tests as expensive benchmark test (deselect with '-m \"not benchmark\"')", "bazel: tests that require a Bazel installation on PATH (deselect with '-m \"not bazel\"')", - # sphinx-test-reports', relocated from that member's own `[tool.pytest.ini_options]`, - # which check (7) refuses here: 8 sites / 25 tests carry it, and without the entry every - # one of them raises `PytestUnknownMarkWarning`. The `toolchain-free` CI job is what - # deselects them, in an environment with no Sphinx at all - "toolchain: needs the documentation toolchain (Sphinx, sphinx-needs) installed (deselect with '-m \"not toolchain\"')", ] filterwarnings = [ "ignore:.*removed in Python 3.14.*:DeprecationWarning", @@ -284,6 +281,8 @@ src = [ "packages/sphinx-codelinks/src", "packages/sphinx-test-reports", "packages/sphinx-test-reports/src", + "packages/ub-test-reports", + "packages/ub-test-reports/src", "packages/sphinx-needs-testkit", "packages/sphinx-needs-testkit/src", "packages/ub-project", @@ -354,6 +353,7 @@ include = [ "packages/sphinx-mounts/src", "packages/sphinx-codelinks/src", "packages/sphinx-test-reports/src", + "packages/ub-test-reports/src", "packages/ub-project/src", "tools/src", ".github/scripts", @@ -382,7 +382,7 @@ allowed-unresolved-imports = [ # sphinx-codelinks' optional libclang engine: `clang.cindex` ships no py.typed and sits # behind the `libclang` extra at runtime, so the typing floor deliberately lacks it "clang.**", - # sphinx-test-reports' pytest plugin. The `typing` environment deliberately has no + # ub-test-reports' pytest plugin. The `typing` environment deliberately has no # pytest -- it pins the oldest supported sphinx and docutils and nothing else it does # not need -- so the plugin's own imports cannot resolve there. The plugin IS type-checked # otherwise; what is unresolved is the runner it plugs into, which is exactly the shape @@ -712,6 +712,18 @@ help = "Run the ub-project suite; trailing args go to pytest" cmd = "pytest" cwd = "packages/ub-project" +# ub-test-reports, the Sphinx-free core of sphinx-test-reports. Like ub-project's, the +# short name keeps the whole distribution name (the naming rule drops only a `sphinx-` +# prefix), and there are no `-sphinxN` cells: the package has no Sphinx, and the environment +# that matters is the one WITHOUT it -- ci.yaml's `toolchain-free` job, which runs this +# suite there from the built wheel. No path is named: nothing under its `src/` is called +# `test_*.py`, unlike the extension's, so `poe test-ub-test-reports tests/test_cli_config.py` +# narrows the run rather than adding to it +[tool.poe.tasks.test-ub-test-reports] +help = "Run the ub-test-reports suite; trailing args go to pytest" +cmd = "pytest" +cwd = "packages/ub-test-reports" + # BARE, not `typecheck-needs`: the naming rule suffixes a task that acts on ONE package, # and this one checks the whole repository -- both packages plus the tooling -- out of a # single `[tool.ty]` configuration and a single environment. Splitting it per package would @@ -723,7 +735,7 @@ help = "Type-check every package with ty against the oldest supported sphinx (th # but it is NOT a fence on `src.include` above: ty intersects the two, so a stale `include` # makes this check zero files and exit 0. The canary step in ci.yaml's Lint job is what # guards that. -cmd = "ty check packages/sphinx-needs/src/sphinx_needs packages/sphinx-mounts/src/sphinx_mounts packages/sphinx-codelinks/src/sphinx_codelinks packages/sphinx-test-reports/src packages/ub-project/src/ub_project tools/src .github/scripts scripts --python .venvs/typing" +cmd = "ty check packages/sphinx-needs/src/sphinx_needs packages/sphinx-mounts/src/sphinx_mounts packages/sphinx-codelinks/src/sphinx_codelinks packages/sphinx-test-reports/src packages/ub-test-reports/src/ub_test_reports packages/ub-project/src/ub_project tools/src .github/scripts scripts --python .venvs/typing" executor = { type = "uv", frozen = true, no-group = "dev", group = ["typing"] } env = { UV_PROJECT_ENVIRONMENT = ".venvs/typing" } @@ -847,14 +859,22 @@ cmd = "uv build --package sphinx-test-reports --no-sources -o dist/sphinx-test-r [tool.poe.tasks.import-check-reports] help = "Import every sphinx-test-reports module from its built wheel, with dependencies as PUBLISHED" -# `--extra sphinx`, because this is the first member in the workspace whose RUNTIME -# dependencies are optional: the bare wheel's `Requires-Dist` is `lxml` and `ub-project`, and -# `sphinx_test_reports.test_reports` imports sphinx-needs. Without the extra the walk -# fails on every module that imports sphinx or docutils (the directives, the bridge, -# the environment and the exceptions). `--extra pytest` too, for `pytest_plugin`, which imports -# pytest and pluggy. The release workflow's compat cell gets the toolchain from this -# member's `compat-requirements.txt`, and pytest from the root `test` group it installs -cmd = "python tools/src/sn_tools/import_check.py sphinx-test-reports --extra sphinx --extra pytest" +# No extras: the wheel's `Requires-Dist` is the toolchain (sphinx, docutils, sphinx-needs) +# and the core, `ub-test-reports`, all hard. The walk covers `sphinx_test_reports` only -- +# the converter and the plugin are the core's, checked by `import-check-ub-test-reports`. +# RED until ub-test-reports is on PyPI, by design: the resolution is the published one, and +# that is the release order (the core first, then this package) +cmd = "python tools/src/sn_tools/import_check.py sphinx-test-reports" + +[tool.poe.tasks.build-ub-test-reports] +help = "Build the ub-test-reports sdist + wheel into dist/ub-test-reports" +cmd = "uv build --package ub-test-reports --no-sources -o dist/ub-test-reports" + +[tool.poe.tasks.import-check-ub-test-reports] +help = "Import every ub_test_reports module from its built wheel, with dependencies as PUBLISHED" +# `--extra pytest` for `pytest_plugin`, which imports pytest and pluggy; nothing else in the +# wheel needs more than its `Requires-Dist` (lxml, ub-project) +cmd = "python tools/src/sn_tools/import_check.py ub-test-reports --extra pytest" [tool.poe.tasks.build-ub-project] help = "Build the ub-project sdist + wheel into dist/ub-project" diff --git a/tools/tests/test_check_workspace.py b/tools/tests/test_check_workspace.py index 7cd254452..40dbcf386 100644 --- a/tools/tests/test_check_workspace.py +++ b/tools/tests/test_check_workspace.py @@ -919,8 +919,8 @@ def test_only_a_real_import_counts(workspace, capsys, line) -> None: def test_a_declaration_in_an_extra_satisfies_an_import(workspace, capsys) -> None: - """sphinx-test-reports' shape: its core install must stay docutils-free, so the - floor lives in the extra that installs the Sphinx toolchain.""" + """The shape sphinx-test-reports 2.0.0 had: its core install had to stay + docutils-free, so the floor lived in the extra that installed the Sphinx toolchain.""" root = workspace( { "acme-reports": { diff --git a/uv.lock b/uv.lock index f40a46454..17fcb2f15 100644 --- a/uv.lock +++ b/uv.lock @@ -23,6 +23,7 @@ members = [ "sphinx-needs-workspace-tools", "sphinx-test-reports", "ub-project", + "ub-test-reports", ] [[package]] @@ -3000,6 +3001,7 @@ dependencies = [ { name = "sphinx-needs-workspace-tools" }, { name = "sphinx-test-reports" }, { name = "ub-project" }, + { name = "ub-test-reports" }, ] [package.optional-dependencies] @@ -3148,6 +3150,7 @@ requires-dist = [ { name = "sphinx-test-reports", editable = "packages/sphinx-test-reports" }, { name = "sphinx-test-reports", extras = ["docs"], marker = "extra == 'docs-reports'", editable = "packages/sphinx-test-reports" }, { name = "ub-project", editable = "packages/ub-project" }, + { name = "ub-test-reports", editable = "packages/ub-test-reports" }, ] provides-extras = ["plotting", "docs", "docs-mounts", "docs-codelinks", "docs-reports", "codelinks-libclang", "theme-im", "theme-furo", "theme-pds", "theme-rtd"] @@ -3287,8 +3290,13 @@ name = "sphinx-test-reports" version = "2.0.0" source = { editable = "packages/sphinx-test-reports" } dependencies = [ - { name = "lxml" }, - { name = "ub-project" }, + { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or extra == 'group-22-sphinx-needs-workspace-sphinx-7' or extra == 'group-22-sphinx-needs-workspace-sphinx-8' or extra == 'group-22-sphinx-needs-workspace-typing'" }, + { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (python_full_version >= '3.12' and extra != 'group-22-sphinx-needs-workspace-sphinx-7' and extra != 'group-22-sphinx-needs-workspace-sphinx-8' and extra != 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-9' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-8') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-typing')" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or extra == 'group-22-sphinx-needs-workspace-sphinx-7' or extra == 'group-22-sphinx-needs-workspace-typing' or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-sphinx-9')" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and extra != 'group-22-sphinx-needs-workspace-sphinx-7' and extra != 'group-22-sphinx-needs-workspace-sphinx-9' and extra != 'group-22-sphinx-needs-workspace-typing') or extra == 'group-22-sphinx-needs-workspace-sphinx-8' or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-9' and extra == 'group-22-sphinx-needs-workspace-typing')" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (python_full_version >= '3.12' and extra != 'group-22-sphinx-needs-workspace-sphinx-7' and extra != 'group-22-sphinx-needs-workspace-sphinx-8' and extra != 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-9' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-8') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-typing')" }, + { name = "sphinx-needs" }, + { name = "ub-test-reports" }, ] [package.optional-dependencies] @@ -3304,32 +3312,23 @@ docs = [ { name = "sphinxcontrib-plantuml" }, ] pytest = [ - { name = "pytest" }, -] -sphinx = [ - { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or extra == 'group-22-sphinx-needs-workspace-sphinx-7' or extra == 'group-22-sphinx-needs-workspace-sphinx-8' or extra == 'group-22-sphinx-needs-workspace-typing'" }, - { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (python_full_version >= '3.12' and extra != 'group-22-sphinx-needs-workspace-sphinx-7' and extra != 'group-22-sphinx-needs-workspace-sphinx-8' and extra != 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-9' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-8') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-typing')" }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or extra == 'group-22-sphinx-needs-workspace-sphinx-7' or extra == 'group-22-sphinx-needs-workspace-typing' or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-sphinx-9')" }, - { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and extra != 'group-22-sphinx-needs-workspace-sphinx-7' and extra != 'group-22-sphinx-needs-workspace-sphinx-9' and extra != 'group-22-sphinx-needs-workspace-typing') or extra == 'group-22-sphinx-needs-workspace-sphinx-8' or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-9' and extra == 'group-22-sphinx-needs-workspace-typing')" }, - { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (python_full_version >= '3.12' and extra != 'group-22-sphinx-needs-workspace-sphinx-7' and extra != 'group-22-sphinx-needs-workspace-sphinx-8' and extra != 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-9' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-8') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-7' and extra == 'group-22-sphinx-needs-workspace-typing') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-sphinx-9') or (extra == 'group-22-sphinx-needs-workspace-sphinx-8' and extra == 'group-22-sphinx-needs-workspace-typing')" }, - { name = "sphinx-needs" }, + { name = "ub-test-reports", extra = ["pytest"] }, ] [package.metadata] requires-dist = [ - { name = "docutils", marker = "extra == 'sphinx'", specifier = ">=0.21" }, - { name = "lxml" }, + { name = "docutils", specifier = ">=0.21" }, { name = "packaging", marker = "extra == 'docs'" }, { name = "pillow", marker = "extra == 'docs'" }, - { name = "pytest", marker = "extra == 'pytest'", specifier = ">=7.0" }, + { name = "sphinx", specifier = ">=7.4" }, { name = "sphinx", marker = "extra == 'docs'" }, - { name = "sphinx", marker = "extra == 'sphinx'", specifier = ">=7.4" }, { name = "sphinx-design", marker = "extra == 'docs'" }, { name = "sphinx-immaterial", marker = "extra == 'docs'" }, + { name = "sphinx-needs", editable = "packages/sphinx-needs" }, { name = "sphinx-needs", marker = "extra == 'docs'", editable = "packages/sphinx-needs" }, - { name = "sphinx-needs", marker = "extra == 'sphinx'", editable = "packages/sphinx-needs" }, { name = "sphinxcontrib-plantuml", marker = "extra == 'docs'" }, - { name = "ub-project", editable = "packages/ub-project" }, + { name = "ub-test-reports", editable = "packages/ub-test-reports" }, + { name = "ub-test-reports", extras = ["pytest"], marker = "extra == 'pytest'", editable = "packages/ub-test-reports" }, ] provides-extras = ["sphinx", "pytest", "docs"] @@ -3854,6 +3853,28 @@ name = "ub-project" version = "1.1.0" source = { editable = "packages/ub-project" } +[[package]] +name = "ub-test-reports" +version = "1.0.0.dev0" +source = { editable = "packages/ub-test-reports" } +dependencies = [ + { name = "lxml" }, + { name = "ub-project" }, +] + +[package.optional-dependencies] +pytest = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "lxml" }, + { name = "pytest", marker = "extra == 'pytest'", specifier = ">=7.0" }, + { name = "ub-project", editable = "packages/ub-project" }, +] +provides-extras = ["pytest"] + [[package]] name = "urllib3" version = "2.8.0"