From 2ce40416a00f9c7cf6f5e1bec92e6d40c332dc36 Mon Sep 17 00:00:00 2001 From: Eric Minish Date: Sun, 15 Nov 2026 14:47:26 -0500 Subject: [PATCH 1/4] fix: one traced-run path for verify and the adapter; crash-safe trace writes (#154) verify.run and pytest_adapter.run duplicated the temp dir, env, runner and trace load, and refused in different orders and words. Both now go through pytest_adapter.run_traced / load_trace. A _workdir parameter replaces the tests' global patch of tempfile.TemporaryDirectory. - --json refusals print {"refused": true, "error": ...} in both CLIs - tgtrace flushes and fsyncs before os.replace; a non-regular target (/dev/stdout, a FIFO) is written directly - --trace-out is written atomically (write_json_atomic) - harness/trace.py treats an unreadable trace as no trace and removes stale .tmp.* files before a run - adapter trace-refusal tests move to test_pytest_adapter.py; the dying-write tests use an unserializable payload instead of patching json Fixes #154 --- CHANGELOG.md | 13 +++ harness/plugin/tgtrace.py | 30 +++++-- harness/trace.py | 25 +++++- testgraph/pytest_adapter.py | 154 +++++++++++++++++++++++++---------- testgraph/verify.py | 64 ++++----------- tests/test_ground_truth.py | 41 ++++++++++ tests/test_pytest_adapter.py | 118 +++++++++++++++++++++++++++ tests/test_verify.py | 136 ++++++++++--------------------- 8 files changed, 385 insertions(+), 196 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90c9091..70ffbd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,6 +128,19 @@ below is on `main` or in the branch that introduced this file. ### Fixed +- **A refusal under `--json` is JSON** (#154). `verify --json` and + `pytest_adapter --json` printed a refusal as prose, so a JSON consumer could not + parse the reason. They now print `{"refused": true, "error": "..."}`, still exit 2. +- **`verify` and the pytest adapter refuse a broken run in the same order and + words** (#154). Both now run pytest through one helper: a missing trace (with + pytest's stderr tail), then a truncated one, then one that is not an object. The + adapter's missing-JUnit refusal now carries the stderr tail too. +- **Trace files survive crashes** (#154). tgtrace flushes and fsyncs before its + atomic rename, so an OS crash can no longer leave a zero-length trace; + `pytest_adapter --trace-out` is written the same way instead of with a plain + open-and-dump; `TGTRACE_OUT=/dev/stdout` (or a FIFO) works again. + `harness/trace.py` reads an unreadable trace as no trace instead of raising, and + clears the `.tmp.` files a killed run leaves behind. - **`verify` works from an installed wheel.** The wheel shipped `testgraph/` only, so the README's headline `verify` command failed after `pip install testgraph` with "cannot find harness/plugin/tgtrace.py". The wheel now carries the trace plugin as diff --git a/harness/plugin/tgtrace.py b/harness/plugin/tgtrace.py index e2ee3fe..36f17ab 100644 --- a/harness/plugin/tgtrace.py +++ b/harness/plugin/tgtrace.py @@ -180,17 +180,35 @@ def pytest_unconfigure(config): "tests": {tid: sorted(map(list, syms)) for tid, syms in _results.items()}, "declared": {tid: list(j) for tid, j in sorted(_declared.items())}, } - directory = os.path.dirname(os.path.abspath(_out_path)) + _write(_out_path, payload) + + +def _write(out_path, payload): + """Write the payload so `out_path` is either absent or whole. + + Beside the target, then rename (#108): a run killed mid-dump used to leave + a truncated file where a reader expects a whole one. os.replace is atomic + on the same filesystem; the flush + fsync before it (#154) keeps an OS + crash from leaving a zero-length target, which ext4/xfs can otherwise do + when the rename reaches disk before the data. + + A target that exists and is NOT a regular file (`/dev/stdout`, a FIFO) has + no directory to put a temp file in and nothing to rename over, so it is + written directly: atomicity is meaningless for a stream.""" + if os.path.exists(out_path) and not os.path.isfile(out_path): + with open(out_path, "w") as f: + json.dump(payload, f, indent=1, sort_keys=True) + return + directory = os.path.dirname(os.path.abspath(out_path)) if directory: os.makedirs(directory, exist_ok=True) - # Write beside the target and rename (#108): a run killed mid-dump used to - # leave a truncated file where a reader expects a whole one. os.replace is - # atomic on the same filesystem, so the target is either absent or complete. - tmp = f"{_out_path}.tmp.{os.getpid()}" + tmp = f"{out_path}.tmp.{os.getpid()}" try: with open(tmp, "w") as f: json.dump(payload, f, indent=1, sort_keys=True) - os.replace(tmp, _out_path) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, out_path) except BaseException: try: os.remove(tmp) diff --git a/harness/trace.py b/harness/trace.py index 6bac53d..fe9242b 100644 --- a/harness/trace.py +++ b/harness/trace.py @@ -20,6 +20,7 @@ and is reported as `no_trace`, never as "nothing to select". """ import argparse +import glob import json import os import subprocess @@ -50,6 +51,13 @@ def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None): out = os.path.abspath(out) if os.path.exists(out): os.remove(out) + # A run SIGKILLed mid-dump leaves tgtrace's `.tmp.` behind + # (#154); nothing else ever removes it, so the traces dir collects them. + for stale in glob.glob(glob.escape(out) + ".tmp.*"): + try: + os.remove(stale) + except OSError: + pass env = dict(env if env is not None else os.environ) env["TGTRACE_OUT"] = out env["TGTRACE_ROOT"] = os.path.join(repo, root) @@ -69,8 +77,16 @@ def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None): return None, None payload = None if os.path.exists(out): - with open(out) as f: - payload = json.load(f) + # tgtrace writes atomically, but an OS crash or a hand-edited file can + # still leave something unreadable. That is the no-trace case the + # caller already handles, not a traceback (#154). + try: + with open(out) as f: + payload = json.load(f) + except (OSError, ValueError): + payload = None + if not isinstance(payload, dict): + payload = None return rc, payload @@ -93,8 +109,9 @@ def main(argv=None): why = ( f"timed out after {args.timeout}s" if rc is None - else f"exited {rc} before the plugin could report " - f"(a collection error, or the plugin was not loaded)" + else f"exited {rc} with no readable trace " + f"(a collection error, the plugin was not loaded, or the file is " + f"truncated or not a tgtrace object)" ) print(f"no trace written to {args.out} — pytest {why}", file=sys.stderr) return 1 diff --git a/testgraph/pytest_adapter.py b/testgraph/pytest_adapter.py index 2e1d020..d8a8e16 100644 --- a/testgraph/pytest_adapter.py +++ b/testgraph/pytest_adapter.py @@ -31,6 +31,7 @@ There is no second adapter to generalize against yet, and a one-adapter abstraction is a guess rather than a design (verification-direction D3). """ +import json import os import subprocess import sys @@ -264,8 +265,99 @@ def plugin_dir(start=None): return None +def load_trace(trace_path, proc): + """(trace, None) for a whole tgtrace payload, else (None, refusal). + + A missing, truncated or non-object trace is a broken run, not a verdict + (#108). One function so verify and this adapter refuse in the same order + and the same words (#154); the substrings "wrote no trace", "truncated or + not JSON" and "not a tgtrace payload" are what callers and tests key on.""" + if not os.path.isfile(trace_path): + # pytest's own stderr usually names the cause (a bad flag, the + # plugin's xdist refusal); the guess below is only a fallback. + tail = (getattr(proc, "stderr", "") or "").strip()[-400:] + return None, ( + f"pytest exited {getattr(proc, 'returncode', '?')} and tgtrace " + f"wrote no trace — the run was killed, or the plugin did not " + f"load; no verdict was recorded" + + (f"\n pytest stderr (tail): {tail}" if tail else "") + ) + try: + with open(trace_path) as fh: + trace = json.load(fh) + except (OSError, ValueError) as exc: + return None, ( + f"the tgtrace trace is truncated or not JSON ({exc}) — the run " + f"was cut short; no verdict was recorded" + ) + if not isinstance(trace, dict): + return None, ( + f"the trace file is not a tgtrace payload (a JSON " + f"{type(trace).__name__}, not an object); no verdict was recorded" + ) + return trace, None + + +def run_traced(repo, plugins, pytest_args, _runner=None, _workdir=None): + """Run pytest once under tgtrace with JUnit XML on; read both back. + + Returns `(proc, trace, verdicts, error)`. `verdicts` is None when pytest + wrote no JUnit XML: whether that refuses the run is the caller's call + (the adapter needs verdicts to attribute anything; verify reads a missing + report as no verdicts and reports the journeys incomplete). + + `_workdir` stands in for the temp dir in tests, so they need not patch + `tempfile.TemporaryDirectory` for every caller in the process (#154).""" + def go(tmp): + trace_path = os.path.join(tmp, "trace.json") + junit_path = os.path.join(tmp, "junit.xml") + env = dict(os.environ) + env["TGTRACE_OUT"] = trace_path + env["TGTRACE_ROOT"] = os.path.abspath(repo) + env["PYTHONPATH"] = os.pathsep.join( + p for p in (plugins, env.get("PYTHONPATH")) if p + ) + cmd = [sys.executable, "-m", "pytest", "-p", "tgtrace", + f"--junitxml={junit_path}", *pytest_args] + runner = _runner or (lambda: subprocess.run( + cmd, cwd=repo, env=env, capture_output=True, text=True)) + proc = runner() + trace, err = load_trace(trace_path, proc) + if err: + return proc, None, None, err + verdicts = parse_junit(junit_path) if os.path.isfile(junit_path) else None + return proc, trace, verdicts, None + + if _workdir is not None: + return go(_workdir) + with tempfile.TemporaryDirectory() as tmp: + return go(tmp) + + +def write_json_atomic(path, obj): + """Write `obj` as JSON so `path` is either absent or whole (#154). + + Temp file beside the target, flushed and fsynced, then renamed: a plain + `open(path, "w")` killed mid-dump leaves a truncated file that a later + reader (reconcile) refuses.""" + directory = os.path.dirname(os.path.abspath(path)) + tmp = os.path.join(directory, f".{os.path.basename(path)}.tmp.{os.getpid()}") + try: + with open(tmp, "w") as f: + json.dump(obj, f, indent=1, sort_keys=True) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + except BaseException: + try: + os.remove(tmp) + except OSError: + pass + raise + + def run(repo, registry_path=None, db_path=None, pytest_args=(), commit=None, - append=None, trace_out=None, _runner=None): + append=None, trace_out=None, _runner=None, _workdir=None): """Run the suite, attribute journeys from the trace, write the rows. Returns (summary, error). `error` is a string on any refusal; a suite whose @@ -302,47 +394,19 @@ def run(repo, registry_path=None, db_path=None, pytest_args=(), commit=None, f"would sit in `unasked` forever" ) - with tempfile.TemporaryDirectory() as tmp: - trace_path = os.path.join(tmp, "trace.json") - junit_path = os.path.join(tmp, "junit.xml") - env = dict(os.environ) - env["TGTRACE_OUT"] = trace_path - env["TGTRACE_ROOT"] = os.path.abspath(repo) - env["PYTHONPATH"] = os.pathsep.join( - p for p in (plugins, env.get("PYTHONPATH")) if p - ) - cmd = [ - sys.executable, "-m", "pytest", "-p", "tgtrace", - f"--junitxml={junit_path}", *pytest_args, - ] - runner = _runner or ( - lambda: subprocess.run(cmd, cwd=repo, env=env, capture_output=True, text=True) - ) - proc = runner() - - if not os.path.isfile(junit_path): - tail = (getattr(proc, "stderr", "") or "")[-400:] - return None, f"pytest wrote no JUnit XML (exit {getattr(proc, 'returncode', '?')}): {tail}" - if not os.path.isfile(trace_path): - return None, ( - "pytest ran but tgtrace wrote nothing — the plugin was not " - "loaded (check that -p tgtrace resolved on PYTHONPATH)" - ) - - import json - try: - with open(trace_path) as f: - trace = json.load(f) - except (OSError, ValueError) as exc: - # Same refusal as verify (#108): a cut-short run is not a verdict. - return None, f"the tgtrace trace is truncated or not JSON ({exc})" - if not isinstance(trace, dict): - return None, (f"the trace file is not a tgtrace payload (a JSON " - f"{type(trace).__name__}, not an object)") - if trace_out: - with open(trace_out, "w") as f: - json.dump(trace, f, indent=1, sort_keys=True) - verdicts = parse_junit(junit_path) + proc, trace, verdicts, err = run_traced(repo, plugins, pytest_args, + _runner=_runner, _workdir=_workdir) + if err: + return None, err + if verdicts is None: + # Attribution joins trace to verdict; with no report there is nothing + # to join, and every journey would read as covered-without-verdict. + tail = (getattr(proc, "stderr", "") or "").strip()[-400:] + return None, (f"pytest exited {getattr(proc, 'returncode', '?')} and wrote " + f"no JUnit XML; no verdict was recorded" + + (f"\n pytest stderr (tail): {tail}" if tail else "")) + if trace_out: + write_json_atomic(trace_out, trace) conn = dbmod.connect(db_path) try: @@ -478,7 +542,11 @@ def main(argv=None): append=(lambda row: False) if args.dry_run else None, ) if err: - print(f"testgraph pytest-adapter: {err}", file=sys.stderr) + if args.json: + # A JSON consumer must be able to parse the refusal too (#154). + print(_json.dumps({"refused": True, "error": err}, indent=2, sort_keys=True)) + else: + print(f"testgraph pytest-adapter: {err}", file=sys.stderr) return 2 summary["dry_run"] = bool(args.dry_run) diff --git a/testgraph/verify.py b/testgraph/verify.py index d81189d..8cc18e2 100644 --- a/testgraph/verify.py +++ b/testgraph/verify.py @@ -34,9 +34,7 @@ that rather than a green. """ import os -import subprocess import sys -import tempfile from . import db as dbmod from . import ledger @@ -158,7 +156,7 @@ def _summary(repo, head, selection, **extra): def run(repo, base, head="HEAD", registry_path=None, db_path=None, - pytest_args=(), append=None, _runner=None): + pytest_args=(), append=None, _runner=None, _workdir=None): """Select, run the declared tests, validate the declarations, report. Returns (summary, error). A red suite is not an error — it is the signal.""" @@ -197,52 +195,14 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, return _summary(repo, head, selection), None expr = marker_expression(journeys) - with tempfile.TemporaryDirectory() as tmp: - trace_path = os.path.join(tmp, "trace.json") - junit_path = os.path.join(tmp, "junit.xml") - env = dict(os.environ) - env["TGTRACE_OUT"] = trace_path - env["TGTRACE_ROOT"] = os.path.abspath(repo) - env["PYTHONPATH"] = os.pathsep.join( - p for p in (plugins, env.get("PYTHONPATH")) if p - ) - cmd = [sys.executable, "-m", "pytest", "-p", "tgtrace", - "-m", expr, f"--junitxml={junit_path}", *pytest_args] - runner = _runner or (lambda: subprocess.run( - cmd, cwd=repo, env=env, capture_output=True, text=True)) - proc = runner() - - import json - # A missing or unreadable trace is a broken run, not a verdict - # (#108). Read as `{}` it reported every selected journey as having - # no journey-level test (exit 3); a truncated one raised out of here - # and exited 1, the code for "tests failed". - if not os.path.isfile(trace_path): - # pytest's own stderr usually names the cause (a bad flag, the - # plugin's xdist refusal); the guess below is only a fallback. - tail = (getattr(proc, "stderr", "") or "").strip()[-400:] - return None, ( - f"pytest exited {getattr(proc, 'returncode', '?')} and tgtrace " - f"wrote no trace — the run was killed, or the plugin did not " - f"load; no verdict was recorded" - + (f"\n pytest stderr (tail): {tail}" if tail else "") - ) - try: - with open(trace_path) as fh: - trace = json.load(fh) - except (OSError, ValueError) as exc: - return None, ( - f"the tgtrace trace is truncated or not JSON ({exc}) — the run " - f"was cut short; no verdict was recorded" - ) - if not isinstance(trace, dict): - return None, ( - f"the trace file is not a tgtrace payload (a JSON " - f"{type(trace).__name__}, not an object); no verdict was recorded" - ) - verdicts = {} - if os.path.isfile(junit_path): - verdicts = pa.parse_junit(junit_path) + proc, trace, verdicts, err = pa.run_traced( + repo, plugins, ("-m", expr, *pytest_args), + _runner=_runner, _workdir=_workdir) + if err: + return None, err + # No JUnit XML reads as no verdicts: every selected journey then lands + # in `incomplete` or `uncovered`, never in a pass. + verdicts = verdicts or {} credited, unvalidated, per_journey = validate(conn, registry, trace, verdicts) uncovered = [j for j in journeys if j not in per_journey] @@ -430,7 +390,11 @@ def main(argv=None): append=(lambda row: False) if args.dry_run else None, ) if err: - print(err) + if args.json: + # A JSON consumer must be able to parse the refusal too (#154). + print(_json.dumps({"refused": True, "error": err}, indent=1, sort_keys=True)) + else: + print(err) return EXIT_REFUSED if args.json: print(_json.dumps(summary, indent=1, sort_keys=True, default=str)) diff --git a/tests/test_ground_truth.py b/tests/test_ground_truth.py index e2b8779..9a1bfaf 100644 --- a/tests/test_ground_truth.py +++ b/tests/test_ground_truth.py @@ -330,3 +330,44 @@ def test_a_worker_thread_is_recorded_by_the_setprofile_backend(self): if __name__ == "__main__": unittest.main() + + +class TraceRunnerTest(unittest.TestCase): + """#154: harness/trace.py read the trace with an unguarded `json.load`, + and never removed the `.tmp.` a SIGKILLed tgtrace leaves.""" + + def _fake_python(self, tmp, body): + # Stands in for the target's interpreter: ignores pytest's argv and + # writes `body` (or nothing) to TGTRACE_OUT. + path = os.path.join(tmp, "fakepy") + with open(path, "w") as fh: + fh.write("#!/bin/sh\n") + if body is not None: + fh.write(f"printf '%s' '{body}' > \"$TGTRACE_OUT\"\n") + os.chmod(path, 0o755) + return path + + def _run(self, body): + from harness import trace as htrace + tmp = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, tmp, ignore_errors=True) + out = os.path.join(tmp, "traces", "t.json") + os.makedirs(os.path.dirname(out)) + stale = out + ".tmp.4242" + open(stale, "w").close() + rc, payload = htrace.run(tmp, self._fake_python(tmp, body), ".", ".", out) + return rc, payload, stale + + def test_a_truncated_trace_is_the_no_trace_case(self): + rc, payload, _ = self._run('{"tests": {') + self.assertEqual(rc, 0) + self.assertIsNone(payload) + + def test_a_non_object_trace_is_the_no_trace_case(self): + rc, payload, _ = self._run("[]") + self.assertIsNone(payload) + + def test_a_stale_temp_file_is_removed_before_the_run(self): + rc, payload, stale = self._run('{"tests": {}}') + self.assertEqual(payload, {"tests": {}}) + self.assertFalse(os.path.exists(stale)) diff --git a/tests/test_pytest_adapter.py b/tests/test_pytest_adapter.py index 4297aef..7ade7c2 100644 --- a/tests/test_pytest_adapter.py +++ b/tests/test_pytest_adapter.py @@ -7,16 +7,24 @@ one outcome this project exists to prevent, so each of those paths is asserted to be REPORTED rather than swallowed. """ +import contextlib +import io +import json import os +import shutil import sqlite3 import sys +import tempfile +import types import unittest +import unittest.mock ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT_DIR) from testgraph.results import covers # noqa: E402 from testgraph import ledger # noqa: E402 from testgraph import pytest_adapter as pa # noqa: E402 +from test_core import _db_on_disk, _git_repo, _registry_file, build_fixture # noqa: E402 def build_conn(): @@ -404,3 +412,113 @@ def test_incomplete_join_is_flagged_in_both_directions(self): if __name__ == "__main__": unittest.main() + + +class RunTraceRefusalTests(unittest.TestCase): + """#108 / #154: a broken run is refused, never attributed. Each case + writes what a real pytest would have left in the work dir; `_workdir` + stands in for the temp dir, so nothing patches the stdlib.""" + + WHOLE = '{"tests": {}, "declared": {}}' + JUNIT = "" + + def setUp(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + self.tmp = tmp + self.db = _db_on_disk(tmp, build_fixture()) + self.registry = _registry_file(tmp, { + "J1": {"name": "ok", "entries": [{"name": "handler_a", "file": "app/svc.py"}]}, + }) + self.repo, _ = _git_repo(tmp, {"app/svc.py": 22}) + self.work = os.path.join(tmp, "work") + os.makedirs(self.work) + + def _run(self, trace=None, junit=None, stderr="", **kw): + work = self.work + + def runner(): + for name, text in (("trace.json", trace), ("junit.xml", junit)): + if text is not None: + with open(os.path.join(work, name), "w") as fh: + fh.write(text) + return types.SimpleNamespace(returncode=0, stderr=stderr) + + return pa.run(self.repo, registry_path=self.registry, db_path=self.db, + _runner=runner, _workdir=work, append=lambda r: True, **kw) + + def test_refusals(self): + cases = [ + ("truncated", dict(trace='{"tests": {', junit=JUNIT_OK), "truncated or not JSON"), + ("not an object", dict(trace="[]", junit=JUNIT_OK), "not a tgtrace payload"), + ("no trace", dict(junit=JUNIT_OK, stderr="boom: xdist"), "wrote no trace"), + ("no junit", dict(trace=self.WHOLE), "wrote no JUnit XML"), + ] + for label, kw, expect in cases: + with self.subTest(label): + for f in os.listdir(self.work): + os.remove(os.path.join(self.work, f)) + summary, err = self._run(**kw) + self.assertIsNone(summary) + self.assertIn(expect, err) + + def test_the_trace_check_comes_before_the_junit_check(self): + """#154: verify and the adapter refused in different orders. With + both files missing, both now name the trace first.""" + summary, err = self._run(stderr="ERROR: unrecognized arguments: --bogus") + self.assertIn("wrote no trace", err) + self.assertIn("--bogus", err) + + def test_a_whole_run_attributes(self): + summary, err = self._run(trace=self.WHOLE, junit=JUNIT_OK) + self.assertIsNone(err) + self.assertEqual(summary["rows_written"], 0) + + def test_trace_out_is_written_whole_and_leaves_no_temp_file(self): + out_dir = os.path.join(self.tmp, "out") + os.makedirs(out_dir) + out = os.path.join(out_dir, "t.json") + summary, err = self._run(trace=self.WHOLE, junit=JUNIT_OK, trace_out=out) + self.assertIsNone(err) + with open(out) as fh: + self.assertEqual(json.load(fh), json.loads(self.WHOLE)) + self.assertEqual(os.listdir(out_dir), ["t.json"]) + + +JUNIT_OK = RunTraceRefusalTests.JUNIT + + +class WriteJsonAtomicTests(unittest.TestCase): + """#154: `--trace-out` was copied with a plain open-and-dump.""" + + def test_a_dump_that_dies_leaves_neither_target_nor_temp(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + out = os.path.join(tmp, "t.json") + # json.dump streams chunks, so the temp file is part-written when the + # unserializable value is reached: the real mid-dump death, without + # patching the json module for the whole process. + with self.assertRaises(TypeError): + pa.write_json_atomic(out, {"a": 1, "z": object()}) + self.assertEqual(os.listdir(tmp), []) + + def test_an_existing_target_survives_a_failed_write(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + out = os.path.join(tmp, "t.json") + pa.write_json_atomic(out, {"old": True}) + with self.assertRaises(TypeError): + pa.write_json_atomic(out, {"z": object()}) + with open(out) as fh: + self.assertEqual(json.load(fh), {"old": True}) + + +class MainJsonRefusalTests(unittest.TestCase): + def test_a_json_refusal_is_json(self): + """#154: the refusal went to stderr as prose even under `--json`.""" + out = io.StringIO() + with unittest.mock.patch.object(pa, "run", return_value=(None, "trace broke")), \ + contextlib.redirect_stdout(out): + rc = pa.main(["--repo", "/x", "--json"]) + self.assertEqual(rc, 2) + self.assertEqual(json.loads(out.getvalue()), {"refused": True, "error": "trace broke"}) diff --git a/tests/test_verify.py b/tests/test_verify.py index da95f61..ebc04db 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -352,24 +352,15 @@ def __getattr__(self, n): work = os.path.join(tmp, "work") os.makedirs(work) - class FixedDir: - def __enter__(self): - return work - - def __exit__(self, *exc): - return False - def runner(): with open(os.path.join(work, "trace.json"), "w") as fh: fh.write('{"tests": {}, "declared": {}}') return types.SimpleNamespace(returncode=0) with unittest.mock.patch.object( - verify.dbmod, "connect", lambda p: Proxy(real(p))), \ - unittest.mock.patch.object(verify.tempfile, "TemporaryDirectory", - lambda: FixedDir()): + verify.dbmod, "connect", lambda p: Proxy(real(p))): summary, err = verify.run(repo, "HEAD~1", "HEAD", registry_path=registry, - db_path=db, _runner=runner, + db_path=db, _runner=runner, _workdir=work, append=lambda row: True) self.assertIsNone(err) self.assertIsNotNone(summary["marker_expression"]) @@ -406,23 +397,15 @@ def setUp(self): def _run(self, trace_text): work = self.work - class FixedDir: - def __enter__(self): - return work - - def __exit__(self, *exc): - return False - def runner(): if trace_text is not None: with open(os.path.join(work, "trace.json"), "w") as fh: fh.write(trace_text) return types.SimpleNamespace(returncode=0) - with unittest.mock.patch.object(verify.tempfile, "TemporaryDirectory", - lambda: FixedDir()): - return verify.run(self.repo, "HEAD~1", "HEAD", registry_path=self.registry, - db_path=self.db, _runner=runner, append=lambda r: True) + return verify.run(self.repo, "HEAD~1", "HEAD", registry_path=self.registry, + db_path=self.db, _runner=runner, _workdir=work, + append=lambda r: True) def test_a_truncated_trace_is_refused_not_a_crash(self): summary, err = self._run('{"tests": {"t::a": [["app/svc.py", "hand') @@ -442,22 +425,12 @@ def test_a_trace_that_is_not_an_object_is_refused(self): def test_the_refusal_carries_pytest_stderr(self): """Review H3: the real cause (an xdist refusal, a bad flag) was in pytest's stderr, which was dropped.""" - work = self.work - - class FixedDir: - def __enter__(self): - return work - - def __exit__(self, *exc): - return False - runner = lambda: types.SimpleNamespace( returncode=4, stderr="ERROR: usage: unrecognized arguments: --bogus") - with unittest.mock.patch.object(verify.tempfile, "TemporaryDirectory", - lambda: FixedDir()): - summary, err = verify.run(self.repo, "HEAD~1", "HEAD", - registry_path=self.registry, db_path=self.db, - _runner=runner, append=lambda r: True) + summary, err = verify.run(self.repo, "HEAD~1", "HEAD", + registry_path=self.registry, db_path=self.db, + _runner=runner, _workdir=self.work, + append=lambda r: True) self.assertIn("unrecognized arguments: --bogus", err) def test_a_missing_trace_is_refused_not_uncovered(self): @@ -471,6 +444,17 @@ def test_main_exits_refused(self): rc = verify.main(["--repo", self.repo, "--base", "HEAD~1"]) self.assertEqual(rc, verify.EXIT_REFUSED) + def test_a_json_refusal_is_json(self): + """#154: `--json` printed the refusal as prose, so a JSON consumer + could not parse the reason.""" + out = io.StringIO() + with unittest.mock.patch.object(verify, "run", return_value=(None, "trace broke")), \ + contextlib.redirect_stdout(out): + rc = verify.main(["--repo", self.repo, "--base", "HEAD~1", "--json"]) + self.assertEqual(rc, verify.EXIT_REFUSED) + self.assertEqual(json.loads(out.getvalue()), + {"refused": True, "error": "trace broke"}) + class AtomicTraceWriteTests(unittest.TestCase): """#108: the trace was written in place, so a run killed mid-write left a @@ -481,23 +465,31 @@ def test_a_write_that_dies_leaves_no_partial_file(self): tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) out = os.path.join(tmp, "trace.json") - - seen_target = [] - - def dying_dump(obj, fh, **kw): - fh.write('{"tests": {') - # a SIGKILL here runs no cleanup: the target must not exist yet - seen_target.append(os.path.exists(out)) - raise KeyboardInterrupt - - with unittest.mock.patch.object(tgtrace, "_out_path", out), \ - unittest.mock.patch.object(tgtrace, "_stop", lambda: None), \ - unittest.mock.patch.object(tgtrace.json, "dump", dying_dump): - with self.assertRaises(KeyboardInterrupt): - tgtrace.pytest_unconfigure(None) + # json.dump streams, so the temp file is part-written when it reaches + # the unserializable value: a real mid-dump death, with no patch on + # the json module the rest of the process shares (#154). + with self.assertRaises(TypeError): + tgtrace._write(out, {"tests": {}, "z": object()}) self.assertFalse(os.path.exists(out)) self.assertEqual(os.listdir(tmp), []) - self.assertEqual(seen_target, [False]) + + def test_a_stream_target_is_written_directly(self): + """#154: `TGTRACE_OUT=/dev/stdout` failed, because the temp file + cannot be created beside a device node. A FIFO stands in for it.""" + from harness.plugin import tgtrace + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + fifo = os.path.join(tmp, "pipe") + os.mkfifo(fifo) + got = [] + import threading + reader = threading.Thread( + target=lambda: got.append(open(fifo).read()), daemon=True) + reader.start() + tgtrace._write(fifo, {"tests": {}}) + reader.join(5) + self.assertEqual(json.loads(got[0]), {"tests": {}}) + self.assertEqual(sorted(os.listdir(tmp)), ["pipe"]) def test_a_whole_write_lands(self): from harness.plugin import tgtrace @@ -512,48 +504,6 @@ def test_a_whole_write_lands(self): self.assertEqual(os.listdir(tmp), ["trace.json"]) -class AdapterTruncatedTraceTests(unittest.TestCase): - def test_the_adapter_refuses_a_truncated_trace(self): - from testgraph import pytest_adapter as pa - tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) - db = _db_on_disk(tmp, build_fixture()) - registry = _registry_file(tmp, { - "J1": {"name": "ok", "entries": [{"name": "handler_a", "file": "app/svc.py"}]}, - }) - repo, _ = _git_repo(tmp, {"app/svc.py": 22}) - work = os.path.join(tmp, "work") - os.makedirs(work) - - class FixedDir: - def __enter__(self): - return work - - def __exit__(self, *exc): - return False - - def runner(): - with open(os.path.join(work, "junit.xml"), "w") as fh: - fh.write("") - with open(os.path.join(work, "trace.json"), "w") as fh: - fh.write(self.trace_text) - return types.SimpleNamespace(returncode=0, stderr="") - - with unittest.mock.patch.object(pa.tempfile, "TemporaryDirectory", - lambda: FixedDir()): - summary, err = pa.run(repo, registry_path=registry, db_path=db, - _runner=runner, append=lambda r: True) - self.assertIsNone(summary) - self.assertIn(self.expect, err) - - trace_text, expect = '{"tests": {', "truncated or not JSON" - - -class AdapterNonObjectTraceTests(AdapterTruncatedTraceTests): - """Review H1, adapter side: `[]` crashed at `trace.get("tests")`.""" - trace_text, expect = "[]", "not a tgtrace payload" - - class SummaryShapeTests(unittest.TestCase): """Every exit from `run` carries the same keys. From 756e9a35c2e09b59a6cdafed2e39a6b301a1205a Mon Sep 17 00:00:00 2001 From: Eric Minish Date: Sun, 15 Nov 2026 15:02:31 -0500 Subject: [PATCH 2/4] fix: stream targets, JUnit and shape refusals, safer temp sweep (#154 review) - F1/F2/F8: one stream rule in both writers (/dev/*, /proc/*, FIFOs are written in place) so /dev/stdout works even when redirected to a file; the adapter's temp is now .tmp., like tgtrace's; a parity test holds the two copies in step - F3: a truncated JUnit report is refused (exit 2), not a ParseError - F4: a trace object without tgtrace's tests/declared shape is refused; harness/trace.py treats it as no trace - F5: harness/trace.py refuses a non-regular --out instead of deleting it - F6: the stale-temp sweep takes only .tmp. of dead pids - F7: unittest.main() moved to the end of the two test files - F9: --trace-out is checked before the suite runs; verify refuses an unreadable registry instead of raising - C1: direct test that a missing JUnit report is no verdicts, not a pass --- CHANGELOG.md | 14 +++- harness/plugin/tgtrace.py | 28 ++++++-- harness/trace.py | 66 ++++++++++++++---- testgraph/pytest_adapter.py | 85 +++++++++++++++++++++-- testgraph/verify.py | 7 +- tests/test_ground_truth.py | 58 +++++++++++++++- tests/test_pytest_adapter.py | 130 ++++++++++++++++++++++++++++++++++- tests/test_verify.py | 38 ++++++++++ 8 files changed, 394 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70ffbd8..1e9deed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,9 +138,17 @@ below is on `main` or in the branch that introduced this file. - **Trace files survive crashes** (#154). tgtrace flushes and fsyncs before its atomic rename, so an OS crash can no longer leave a zero-length trace; `pytest_adapter --trace-out` is written the same way instead of with a plain - open-and-dump; `TGTRACE_OUT=/dev/stdout` (or a FIFO) works again. - `harness/trace.py` reads an unreadable trace as no trace instead of raising, and - clears the `.tmp.` files a killed run leaves behind. + open-and-dump. Both writers treat `/dev/*`, `/proc/*` and FIFOs as streams and + write them in place, so `TGTRACE_OUT=/dev/stdout` works even with stdout + redirected to a file, and `--trace-out` to a pipe still works. +- **More broken runs are refused instead of crashing** (#154). A truncated JUnit + report, a trace object without tgtrace's `tests`/`declared` shape, and an + unreadable registry in `verify` now refuse with exit 2; they used to raise and + exit 1, the code for "tests failed". `--trace-out` is checked before the suite + runs, not after. +- **`harness/trace.py`** reads an unreadable or foreign trace as no trace instead of + raising, refuses a non-regular `--out` (it deletes the path first and reads it + back), and removes only the `.tmp.` files left by dead processes. - **`verify` works from an installed wheel.** The wheel shipped `testgraph/` only, so the README's headline `verify` command failed after `pip install testgraph` with "cannot find harness/plugin/tgtrace.py". The wheel now carries the trace plugin as diff --git a/harness/plugin/tgtrace.py b/harness/plugin/tgtrace.py index 36f17ab..8ba0e42 100644 --- a/harness/plugin/tgtrace.py +++ b/harness/plugin/tgtrace.py @@ -183,6 +183,26 @@ def pytest_unconfigure(config): _write(_out_path, payload) +# Paths that name a stream however they resolve: `/dev/stdout` redirected to a +# file resolves to a REGULAR file, so `isfile` alone would send it down the +# temp-and-rename path, which cannot create a file in /dev (#154 review F2). +_STREAM_PREFIXES = ("/dev/", "/proc/") + + +def _is_stream(path): + """True when `path` must be written in place, not via temp + rename. + + Kept in step with `testgraph.pytest_adapter.is_stream` by + tests/test_pytest_adapter.py:WriterParityTests — this plugin imports + nothing from testgraph, so the rule exists twice.""" + if os.path.abspath(path).startswith(_STREAM_PREFIXES): + return True + # A directory is neither: writing to it fails either way, and callers + # that can refuse one up front (`--trace-out`) need to see it as such. + return (os.path.exists(path) and not os.path.isfile(path) + and not os.path.isdir(path)) + + def _write(out_path, payload): """Write the payload so `out_path` is either absent or whole. @@ -192,10 +212,10 @@ def _write(out_path, payload): crash from leaving a zero-length target, which ext4/xfs can otherwise do when the rename reaches disk before the data. - A target that exists and is NOT a regular file (`/dev/stdout`, a FIFO) has - no directory to put a temp file in and nothing to rename over, so it is - written directly: atomicity is meaningless for a stream.""" - if os.path.exists(out_path) and not os.path.isfile(out_path): + A stream target (`/dev/stdout`, `/proc/self/fd/N`, a FIFO; see + `_is_stream`) has no directory to put a temp file in and nothing to rename + over, so it is written directly: atomicity is meaningless for a stream.""" + if _is_stream(out_path): with open(out_path, "w") as f: json.dump(payload, f, indent=1, sort_keys=True) return diff --git a/harness/trace.py b/harness/trace.py index fe9242b..b41f309 100644 --- a/harness/trace.py +++ b/harness/trace.py @@ -20,9 +20,9 @@ and is reported as `no_trace`, never as "nothing to select". """ import argparse -import glob import json import os +import re import subprocess import sys @@ -34,6 +34,38 @@ PLUGIN_DIR = os.path.join(HERE, "plugin") +def _pid_alive(pid): + try: + os.kill(pid, 0) + except (ProcessLookupError, OverflowError): + return False + except PermissionError: + return True # exists, owned by someone else + return True + + +def _sweep_stale_temps(out): + """Remove `.tmp.` files whose writer is dead. + + A run SIGKILLed mid-dump leaves tgtrace's temp behind (#154) and nothing + else removes it. Only `.tmp.` names, and only dead pids: a broader + glob took unrelated files (`a.json.tmp.json`) and a CONCURRENT run's + in-progress temp, whose rename then failed (#154 review F6).""" + name = re.compile(re.escape(os.path.basename(out)) + r"\.tmp\.(\d+)$") + directory = os.path.dirname(out) + try: + siblings = os.listdir(directory) + except OSError: + return + for sib in siblings: + m = name.match(sib) + if m and not _pid_alive(int(m.group(1))): + try: + os.remove(os.path.join(directory, sib)) + except OSError: + pass + + def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None): """Run the suite under the plugin. Returns (returncode, payload_or_None). @@ -48,16 +80,17 @@ def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None): — the exact honeyslate case — would load the PREVIOUS run's trace and report it as fresh.""" repo = os.path.abspath(repo) + if out.startswith(("/dev/", "/proc/")) or ( + os.path.exists(out) and not os.path.isfile(out)): + # This runner deletes `out` first and reads it back after, so a stream + # can be neither: deleting a FIFO removes it from under its reader, + # and /dev/stdout cannot be unlinked at all (#154 review F5). + raise ValueError(f"--out {out} is not a regular file path; this " + f"runner reads the trace back, so it needs one") out = os.path.abspath(out) if os.path.exists(out): os.remove(out) - # A run SIGKILLed mid-dump leaves tgtrace's `.tmp.` behind - # (#154); nothing else ever removes it, so the traces dir collects them. - for stale in glob.glob(glob.escape(out) + ".tmp.*"): - try: - os.remove(stale) - except OSError: - pass + _sweep_stale_temps(out) env = dict(env if env is not None else os.environ) env["TGTRACE_OUT"] = out env["TGTRACE_ROOT"] = os.path.join(repo, root) @@ -85,7 +118,8 @@ def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None): payload = json.load(f) except (OSError, ValueError): payload = None - if not isinstance(payload, dict): + if not (isinstance(payload, dict) and isinstance(payload.get("tests"), dict) + and all(isinstance(v, list) for v in payload["tests"].values())): payload = None return rc, payload @@ -101,10 +135,14 @@ def main(argv=None): ap.add_argument("pytest_args", nargs="*", help="extra args passed to pytest") args = ap.parse_args(argv) - rc, payload = run( - args.repo, args.python, args.tests, args.root, args.out, - extra_args=args.pytest_args, timeout=args.timeout, - ) + try: + rc, payload = run( + args.repo, args.python, args.tests, args.root, args.out, + extra_args=args.pytest_args, timeout=args.timeout, + ) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 2 if payload is None: why = ( f"timed out after {args.timeout}s" @@ -129,7 +167,7 @@ def main(argv=None): ) return 1 print(f"traced {len(tests)} test(s), {len(symbols)} distinct symbol(s)") - print(f" backend: {payload['backend']} root: {payload['root']}") + print(f" backend: {payload.get('backend', '?')} root: {payload.get('root', '?')}") print(f" wrote {args.out}") empty = [t for t, s in tests.items() if not s] if empty: diff --git a/testgraph/pytest_adapter.py b/testgraph/pytest_adapter.py index d8a8e16..6204abc 100644 --- a/testgraph/pytest_adapter.py +++ b/testgraph/pytest_adapter.py @@ -295,9 +295,31 @@ def load_trace(trace_path, proc): f"the trace file is not a tgtrace payload (a JSON " f"{type(trace).__name__}, not an object); no verdict was recorded" ) + shape = payload_shape_error(trace) + if shape: + return None, (f"the trace file is not a tgtrace payload ({shape}); " + f"no verdict was recorded") return trace, None +def payload_shape_error(trace): + """What is wrong with a parsed trace object's shape, or None. + + tgtrace always writes `tests` as {nodeid: [[relpath, qualname], ...]} and + `declared` as {nodeid: [journey, ...]}. An object missing either shape + passed the old isinstance check and crashed later (#154 review F4).""" + tests = trace.get("tests") + if not isinstance(tests, dict): + return "`tests` is missing or not an object" + if any(not isinstance(v, list) for v in tests.values()): + return "a `tests` entry is not a list" + declared = trace.get("declared", {}) + if not isinstance(declared, dict) or any( + not isinstance(v, list) for v in declared.values()): + return "`declared` is not an object of lists" + return None + + def run_traced(repo, plugins, pytest_args, _runner=None, _workdir=None): """Run pytest once under tgtrace with JUnit XML on; read both back. @@ -325,7 +347,16 @@ def go(tmp): trace, err = load_trace(trace_path, proc) if err: return proc, None, None, err - verdicts = parse_junit(junit_path) if os.path.isfile(junit_path) else None + verdicts = None + if os.path.isfile(junit_path): + try: + verdicts = parse_junit(junit_path) + except (ET.ParseError, OSError) as exc: + # Killed while writing the report: a broken run, not a red + # suite. Raising here exited 1, the "tests failed" code (F3). + return proc, None, None, ( + f"pytest's JUnit XML is truncated or not XML ({exc}) — the " + f"run was cut short; no verdict was recorded") return proc, trace, verdicts, None if _workdir is not None: @@ -334,14 +365,55 @@ def go(tmp): return go(tmp) +_STREAM_PREFIXES = ("/dev/", "/proc/") + + +def is_stream(path): + """True when `path` must be written in place, not via temp + rename. + + `/dev/stdout` redirected to a file resolves to a REGULAR file, so the + prefix is checked on the unresolved path; anything else that exists and is + not a regular file (a FIFO) is a stream too. The same rule lives in + tgtrace (`_is_stream`), which cannot import this module; + tests/test_pytest_adapter.py:WriterParityTests holds the two in step.""" + if os.path.abspath(path).startswith(_STREAM_PREFIXES): + return True + # A directory is neither: writing to it fails either way, and callers + # that can refuse one up front (`--trace-out`) need to see it as such. + return (os.path.exists(path) and not os.path.isfile(path) + and not os.path.isdir(path)) + + +def trace_out_error(path): + """Why `--trace-out path` cannot be written, or None. + + Checked BEFORE the suite runs (#154 review F9): finding out after a + half-hour run that the directory does not exist throws the run away.""" + if is_stream(path): + return None + if os.path.isdir(path): + return f"--trace-out {path} is a directory" + parent = os.path.dirname(os.path.abspath(path)) + if not os.path.isdir(parent): + return f"--trace-out directory {parent} does not exist" + if not os.access(parent, os.W_OK): + return f"--trace-out directory {parent} is not writable" + return None + + def write_json_atomic(path, obj): """Write `obj` as JSON so `path` is either absent or whole (#154). Temp file beside the target, flushed and fsynced, then renamed: a plain `open(path, "w")` killed mid-dump leaves a truncated file that a later - reader (reconcile) refuses.""" - directory = os.path.dirname(os.path.abspath(path)) - tmp = os.path.join(directory, f".{os.path.basename(path)}.tmp.{os.getpid()}") + reader (reconcile) refuses. The temp is named `.tmp.`, as + tgtrace names its own, so harness/trace.py's sweep finds both. A stream + target is written in place (see `is_stream`).""" + if is_stream(path): + with open(path, "w") as f: + json.dump(obj, f, indent=1, sort_keys=True) + return + tmp = f"{path}.tmp.{os.getpid()}" try: with open(tmp, "w") as f: json.dump(obj, f, indent=1, sort_keys=True) @@ -387,6 +459,11 @@ def run(repo, registry_path=None, db_path=None, pytest_args=(), commit=None, if not os.path.isfile(db_path): return None, f"no codegraph index at {db_path} — run `codegraph init` in {repo}" + if trace_out: + bad = trace_out_error(trace_out) + if bad: + return None, bad + sha = ledger.resolve_commit(repo, commit or "HEAD") if sha is None: return None, ( diff --git a/testgraph/verify.py b/testgraph/verify.py index 8cc18e2..705764e 100644 --- a/testgraph/verify.py +++ b/testgraph/verify.py @@ -164,6 +164,12 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, if registry_path is None: return None, (f"no journey registry for {reg.repo_name(repo)} — looked " f"in {reg.where_it_looked(repo)}") + try: + registry = reg.load(registry_path) + except (OSError, ValueError) as exc: + # Raised out of here before: a traceback and exit 1, the code for + # "tests failed", and prose under --json (#154 review F9). + return None, f"cannot read registry {registry_path}: {exc}" db_path = db_path or os.path.join(repo, ".codegraph", "codegraph.db") if not os.path.isfile(db_path): return None, f"no codegraph index at {db_path} — run `codegraph init` in {repo}" @@ -187,7 +193,6 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, # the plain-string form is what a caller passing a hand-built selection uses. journeys = [j["id"] if isinstance(j, dict) else j for j in selection.get("journeys", [])] - registry = reg.load(registry_path) conn = dbmod.connect(db_path) try: diff --git a/tests/test_ground_truth.py b/tests/test_ground_truth.py index 9a1bfaf..e96e6ac 100644 --- a/tests/test_ground_truth.py +++ b/tests/test_ground_truth.py @@ -11,6 +11,7 @@ import sys import tempfile import unittest +import unittest.mock ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if ROOT not in sys.path: @@ -328,9 +329,6 @@ def test_a_worker_thread_is_recorded_by_the_setprofile_backend(self): self.assertIsNone(sys.getprofile()) -if __name__ == "__main__": - unittest.main() - class TraceRunnerTest(unittest.TestCase): """#154: harness/trace.py read the trace with an unguarded `json.load`, @@ -371,3 +369,57 @@ def test_a_stale_temp_file_is_removed_before_the_run(self): rc, payload, stale = self._run('{"tests": {}}') self.assertEqual(payload, {"tests": {}}) self.assertFalse(os.path.exists(stale)) + + + +class TraceRunnerReviewTest(TraceRunnerTest): + """Round-1 review of #154 (.review ledger F4, F5, F6).""" + + def test_f4_an_object_without_tgtrace_fields_is_the_no_trace_case(self): + for body in ("{}", '{"tests": 3}', '{"tests": {"t": 3}}'): + with self.subTest(body): + rc, payload, _ = self._run(body) + self.assertIsNone(payload) + + def test_f4_a_minimal_tgtrace_object_is_kept(self): + rc, payload, _ = self._run('{"tests": {"t": [["a", "b"]]}}') + self.assertEqual(payload, {"tests": {"t": [["a", "b"]]}}) + + def test_f4_main_survives_a_trace_without_backend(self): + from harness import trace as htrace + import contextlib, io + with unittest.mock.patch.object( + htrace, "run", return_value=(0, {"tests": {"t": [["a", "b"]]}})), \ + contextlib.redirect_stdout(io.StringIO()): + rc = htrace.main(["--repo", ".", "--python", "x", "--out", "o.json"]) + self.assertEqual(rc, 0) + + def test_f5_a_non_regular_out_is_refused_and_left_alone(self): + from harness import trace as htrace + tmp = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, tmp, ignore_errors=True) + fifo = os.path.join(tmp, "pipe") + os.mkfifo(fifo) + import stat + with self.assertRaises(ValueError): + htrace.run(tmp, "/bin/false", ".", ".", fifo) + self.assertTrue(stat.S_ISFIFO(os.stat(fifo).st_mode)) + + def test_f6_the_sweep_takes_only_dead_pids(self): + from harness import trace as htrace + tmp = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, tmp, ignore_errors=True) + out = os.path.join(tmp, "a.json") + live = f"{out}.tmp.{os.getpid()}" + keep = [live, out + ".tmp.json", out + ".tmp.12x"] + dead = out + ".tmp.999999999" + for f in keep + [dead]: + open(f, "w").close() + htrace.run(tmp, self._fake_python(tmp, '{"tests": {}}'), ".", ".", out) + for f in keep: + self.assertTrue(os.path.exists(f), f) + self.assertFalse(os.path.exists(dead)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pytest_adapter.py b/tests/test_pytest_adapter.py index 7ade7c2..5bb3b67 100644 --- a/tests/test_pytest_adapter.py +++ b/tests/test_pytest_adapter.py @@ -410,9 +410,6 @@ def test_incomplete_join_is_flagged_in_both_directions(self): self.assertIn("coverage may be overstated", out) -if __name__ == "__main__": - unittest.main() - class RunTraceRefusalTests(unittest.TestCase): """#108 / #154: a broken run is refused, never attributed. Each case @@ -522,3 +519,130 @@ def test_a_json_refusal_is_json(self): rc = pa.main(["--repo", "/x", "--json"]) self.assertEqual(rc, 2) self.assertEqual(json.loads(out.getvalue()), {"refused": True, "error": "trace broke"}) + + + +class ReviewRound1Tests(RunTraceRefusalTests): + """Round-1 review of #154 (.review ledger F1, F3, F4, F8, F9).""" + + def test_f3_a_truncated_junit_report_is_refused_not_a_crash(self): + for text in ("", ""): + with self.subTest(repr(text)): + for f in os.listdir(self.work): + os.remove(os.path.join(self.work, f)) + summary, err = self._run(trace=self.WHOLE, junit=text) + self.assertIsNone(summary) + self.assertIn("JUnit XML is truncated or not XML", err) + + def test_f4_an_object_without_tgtrace_fields_is_refused(self): + for text in ("{}", '{"tests": 3}', '{"tests": {}, "declared": 3}', + '{"tests": {"t": 3}}'): + with self.subTest(text): + for f in os.listdir(self.work): + os.remove(os.path.join(self.work, f)) + summary, err = self._run(trace=text, junit=JUNIT_OK) + self.assertIsNone(summary) + self.assertIn("not a tgtrace payload", err) + + def test_f9_a_trace_out_that_cannot_be_written_is_refused_before_the_run(self): + ran = [] + bad = [os.path.join(self.tmp, "no", "such", "t.json"), self.tmp] + for out in bad: + with self.subTest(out): + def runner(): + ran.append(out) + return types.SimpleNamespace(returncode=0, stderr="") + summary, err = pa.run(self.repo, registry_path=self.registry, + db_path=self.db, _runner=runner, + _workdir=self.work, append=lambda r: True, + trace_out=out) + self.assertIsNone(summary) + self.assertIn("--trace-out", err) + self.assertEqual(ran, [], "the suite ran before --trace-out was checked") + + def test_f1_trace_out_to_a_fifo_is_written_through(self): + fifo = os.path.join(self.tmp, "pipe") + os.mkfifo(fifo) + got = [] + import threading + t = threading.Thread(target=lambda: got.append(open(fifo).read()), daemon=True) + t.start() + summary, err = self._run(trace=self.WHOLE, junit=JUNIT_OK, trace_out=fifo) + t.join(5) + self.assertIsNone(err) + self.assertEqual(json.loads(got[0]), json.loads(self.WHOLE)) + self.assertTrue(__import__("stat").S_ISFIFO(os.stat(fifo).st_mode)) + + +class WriterParityTests(unittest.TestCase): + """F1/F2/F8: testgraph's atomic writer and tgtrace's are two copies of one + rule (tgtrace must stay standalone, so it cannot import the other). This + holds them to the same behaviour rather than a comment saying they match.""" + + def _writers(self): + sys.path.insert(0, os.path.join(ROOT_DIR, "harness", "plugin")) + try: + import tgtrace + finally: + sys.path.pop(0) + return {"adapter": pa.write_json_atomic, "tgtrace": tgtrace._write} + + def test_both_treat_dev_paths_as_streams(self): + """/dev/stdout redirected to a file resolves to a regular file, so an + `isfile` test alone sends it down the temp-and-rename path, which + cannot create a file in /dev (F2).""" + sys.path.insert(0, os.path.join(ROOT_DIR, "harness", "plugin")) + import tgtrace + sys.path.pop(0) + for path in ("/dev/stdout", "/dev/fd/1", "/proc/self/fd/1"): + with self.subTest(path): + self.assertTrue(pa.is_stream(path)) + self.assertTrue(tgtrace._is_stream(path)) + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + regular = os.path.join(tmp, "t.json") + open(regular, "w").close() + for path in (regular, os.path.join(tmp, "absent.json")): + self.assertFalse(pa.is_stream(path)) + self.assertFalse(tgtrace._is_stream(path)) + + def test_both_write_dev_stdout_redirected_to_a_file(self): + """F2, end to end: a child whose stdout IS a regular file writes + through /dev/stdout with each writer.""" + import subprocess + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + for name, code in ( + ("adapter", "from testgraph import pytest_adapter as w; w.write_json_atomic('/dev/stdout', {'ok': 1})"), + ("tgtrace", "import sys; sys.path.insert(0, 'harness/plugin'); import tgtrace as w; w._write('/dev/stdout', {'ok': 1})"), + ): + with self.subTest(name): + out = os.path.join(tmp, name + ".json") + with open(out, "w") as fh: + proc = subprocess.run([sys.executable, "-c", code], cwd=ROOT_DIR, + stdout=fh, stderr=subprocess.PIPE, text=True) + self.assertEqual(proc.returncode, 0, proc.stderr) + with open(out) as fh: + self.assertEqual(json.load(fh), {"ok": 1}) + + def test_both_name_their_temp_file_the_same_way(self): + """F8: the harness sweep matches `.tmp.`; a killed adapter + used to leave `..tmp.`, which nothing swept.""" + for name, write in self._writers().items(): + with self.subTest(name): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + out = os.path.join(tmp, "t.json") + seen = [] + real_replace = os.replace + + def spy(src, dst): + seen.append(os.path.basename(src)) + return real_replace(src, dst) + with unittest.mock.patch("os.replace", spy): + write(out, {"a": 1}) + self.assertEqual(seen, [f"t.json.tmp.{os.getpid()}"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_verify.py b/tests/test_verify.py index ebc04db..6d605f9 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -637,5 +637,43 @@ class C: self.assertEqual(res.declared_journeys(C), ("J1",)) + +class ReviewRound1VerifyTests(UnreadableTraceTests): + """Round-1 review of #154 (.review ledger C1, F3, F9).""" + + def test_c1_a_missing_junit_report_is_no_verdicts_not_a_pass(self): + summary, err = self._run('{"tests": {}, "declared": {}}') + self.assertIsNone(err) + self.assertNotEqual(verify.exit_code(summary), verify.EXIT_OK) + self.assertEqual(summary["collected"], 0) + + def test_f3_a_truncated_junit_report_refuses_verify(self): + work = self.work + + def runner(): + with open(os.path.join(work, "trace.json"), "w") as fh: + fh.write('{"tests": {}, "declared": {}}') + with open(os.path.join(work, "junit.xml"), "w") as fh: + fh.write("") + return types.SimpleNamespace(returncode=0, stderr="") + + summary, err = verify.run(self.repo, "HEAD~1", "HEAD", + registry_path=self.registry, db_path=self.db, + _runner=runner, _workdir=work, append=lambda r: True) + self.assertIsNone(summary) + self.assertIn("JUnit XML is truncated or not XML", err) + + def test_f9_an_unreadable_registry_is_a_json_refusal(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + rc = verify.main(["--repo", self.repo, "--base", "HEAD~1", + "--registry", os.path.join(self.tmp, "nope.json"), + "--db", self.db, "--json"]) + self.assertEqual(rc, verify.EXIT_REFUSED) + body = json.loads(out.getvalue()) + self.assertTrue(body["refused"]) + self.assertIn("nope.json", body["error"]) + + if __name__ == "__main__": unittest.main() From 557587196132b8ff55a5a26296e1df4dc40f46e2 Mon Sep 17 00:00:00 2001 From: Eric Minish Date: Sun, 15 Nov 2026 15:46:31 -0500 Subject: [PATCH 3/4] fix: refuse stream trace targets instead of writing them in place (#154 review round 2) Writing /dev/stdout in place corrupted it (a second open has its own offset, so pytest's output overwrote the JSON), and the /dev/ prefix rule also caught /dev/shm, an ordinary tmpfs. Stream support is dropped: - one rule, not_a_file_target: abspath, then /dev/ or /proc/ but not /dev/shm/, or an existing non-regular file. tgtrace refuses at pytest_configure; the adapter refuses --trace-out before the run; harness/trace.py imports the adapter's rule. The tgtrace copy is parity-tested, including a child whose stdout is a regular file. - writers keep only temp + fsync + rename - payload_shape_error checks each entry; declared: null stays accepted - verify refuses a registry that is not a JSON object - review test fixtures become mixins, so no inherited test runs twice --- CHANGELOG.md | 21 +++-- harness/plugin/tgtrace.py | 46 +++++------ harness/trace.py | 15 ++-- testgraph/pytest_adapter.py | 75 ++++++++++------- testgraph/verify.py | 5 ++ tests/test_ground_truth.py | 29 ++++++- tests/test_pytest_adapter.py | 156 ++++++++++++++++++++++++----------- tests/test_verify.py | 40 ++++++--- 8 files changed, 254 insertions(+), 133 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e9deed..3adbd34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,17 +138,20 @@ below is on `main` or in the branch that introduced this file. - **Trace files survive crashes** (#154). tgtrace flushes and fsyncs before its atomic rename, so an OS crash can no longer leave a zero-length trace; `pytest_adapter --trace-out` is written the same way instead of with a plain - open-and-dump. Both writers treat `/dev/*`, `/proc/*` and FIFOs as streams and - write them in place, so `TGTRACE_OUT=/dev/stdout` works even with stdout - redirected to a file, and `--trace-out` to a pipe still works. + open-and-dump. +- **Stream targets are refused up front** (#154). `TGTRACE_OUT`, `--trace-out` and + `harness/trace.py --out` must name a file: `/dev/stdout`, pipes and other device + paths are refused before any test runs, with a message saying why. Writing a + stream in place was tried and dropped, because a second open of `/dev/stdout` + has its own file offset and pytest's output overwrote the JSON. `/dev/shm` + counts as a file location. - **More broken runs are refused instead of crashing** (#154). A truncated JUnit - report, a trace object without tgtrace's `tests`/`declared` shape, and an - unreadable registry in `verify` now refuse with exit 2; they used to raise and - exit 1, the code for "tests failed". `--trace-out` is checked before the suite - runs, not after. + report, a trace whose `tests`/`declared` entries are not tgtrace's shape, and a + registry that is unreadable or not a JSON object now refuse with exit 2; they + used to raise and exit 1, the code for "tests failed". `--trace-out` is checked + before the suite runs, not after. - **`harness/trace.py`** reads an unreadable or foreign trace as no trace instead of - raising, refuses a non-regular `--out` (it deletes the path first and reads it - back), and removes only the `.tmp.` files left by dead processes. + raising, and removes only the `.tmp.` files left by dead processes. - **`verify` works from an installed wheel.** The wheel shipped `testgraph/` only, so the README's headline `verify` command failed after `pip install testgraph` with "cannot find harness/plugin/tgtrace.py". The wheel now carries the trace plugin as diff --git a/harness/plugin/tgtrace.py b/harness/plugin/tgtrace.py index 8ba0e42..40ca5f2 100644 --- a/harness/plugin/tgtrace.py +++ b/harness/plugin/tgtrace.py @@ -167,6 +167,14 @@ def pytest_configure(config): "overwrite one another's trace and the result would silently be " "empty. Re-run without -n/--numprocesses." ) + # Refused here, before any test runs, rather than failing at unconfigure + # after the whole suite (#154 review R2-2). + if _not_a_file_target(_out_path): + raise RuntimeError( + f"TGTRACE_OUT={_out_path} is not a regular file path: the trace is " + f"written to a temp file and renamed, so streams and devices " + f"(/dev/stdout, pipes) are refused. Point it at a file." + ) _start() @@ -183,24 +191,23 @@ def pytest_unconfigure(config): _write(_out_path, payload) -# Paths that name a stream however they resolve: `/dev/stdout` redirected to a -# file resolves to a REGULAR file, so `isfile` alone would send it down the -# temp-and-rename path, which cannot create a file in /dev (#154 review F2). -_STREAM_PREFIXES = ("/dev/", "/proc/") +# Device and process paths are streams however they resolve: `/dev/stdout` +# redirected to a file resolves to a REGULAR file. `/dev/shm` is ordinary +# tmpfs storage, not a stream (#154 review R2-1). +_DEVICE_PREFIXES = ("/dev/", "/proc/") +_FILE_UNDER_DEV = ("/dev/shm/",) -def _is_stream(path): - """True when `path` must be written in place, not via temp + rename. +def _not_a_file_target(path): + """True when `path` cannot hold a trace file: a stream or device path. - Kept in step with `testgraph.pytest_adapter.is_stream` by - tests/test_pytest_adapter.py:WriterParityTests — this plugin imports - nothing from testgraph, so the rule exists twice.""" - if os.path.abspath(path).startswith(_STREAM_PREFIXES): + A copy of `testgraph.pytest_adapter.not_a_file_target`, because this + plugin imports nothing from testgraph; WriterParityTests in + tests/test_pytest_adapter.py holds the two to identical answers.""" + p = os.path.abspath(path) + if p.startswith(_DEVICE_PREFIXES) and not p.startswith(_FILE_UNDER_DEV): return True - # A directory is neither: writing to it fails either way, and callers - # that can refuse one up front (`--trace-out`) need to see it as such. - return (os.path.exists(path) and not os.path.isfile(path) - and not os.path.isdir(path)) + return os.path.exists(p) and not os.path.isfile(p) def _write(out_path, payload): @@ -210,15 +217,8 @@ def _write(out_path, payload): a truncated file where a reader expects a whole one. os.replace is atomic on the same filesystem; the flush + fsync before it (#154) keeps an OS crash from leaving a zero-length target, which ext4/xfs can otherwise do - when the rename reaches disk before the data. - - A stream target (`/dev/stdout`, `/proc/self/fd/N`, a FIFO; see - `_is_stream`) has no directory to put a temp file in and nothing to rename - over, so it is written directly: atomicity is meaningless for a stream.""" - if _is_stream(out_path): - with open(out_path, "w") as f: - json.dump(payload, f, indent=1, sort_keys=True) - return + when the rename reaches disk before the data. Stream targets were refused + in `pytest_configure`.""" directory = os.path.dirname(os.path.abspath(out_path)) if directory: os.makedirs(directory, exist_ok=True) diff --git a/harness/trace.py b/harness/trace.py index b41f309..f5d5507 100644 --- a/harness/trace.py +++ b/harness/trace.py @@ -26,6 +26,11 @@ import subprocess import sys +_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if _ROOT not in sys.path: + sys.path.insert(0, _ROOT) +from testgraph import pytest_adapter as pa # noqa: E402 + HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) # ONLY this directory goes on the target's PYTHONPATH. `harness/` itself holds @@ -80,11 +85,12 @@ def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None): — the exact honeyslate case — would load the PREVIOUS run's trace and report it as fresh.""" repo = os.path.abspath(repo) - if out.startswith(("/dev/", "/proc/")) or ( - os.path.exists(out) and not os.path.isfile(out)): + if pa.not_a_file_target(out): # This runner deletes `out` first and reads it back after, so a stream # can be neither: deleting a FIFO removes it from under its reader, - # and /dev/stdout cannot be unlinked at all (#154 review F5). + # and /dev/stdout cannot be unlinked at all (#154 review F5). The rule + # is the adapter's, imported, so it cannot drift into a third copy + # (R2-4, R2-8). raise ValueError(f"--out {out} is not a regular file path; this " f"runner reads the trace back, so it needs one") out = os.path.abspath(out) @@ -118,8 +124,7 @@ def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None): payload = json.load(f) except (OSError, ValueError): payload = None - if not (isinstance(payload, dict) and isinstance(payload.get("tests"), dict) - and all(isinstance(v, list) for v in payload["tests"].values())): + if not isinstance(payload, dict) or pa.payload_shape_error(payload): payload = None return rc, payload diff --git a/testgraph/pytest_adapter.py b/testgraph/pytest_adapter.py index 6204abc..5f6a54e 100644 --- a/testgraph/pytest_adapter.py +++ b/testgraph/pytest_adapter.py @@ -306,17 +306,25 @@ def payload_shape_error(trace): """What is wrong with a parsed trace object's shape, or None. tgtrace always writes `tests` as {nodeid: [[relpath, qualname], ...]} and - `declared` as {nodeid: [journey, ...]}. An object missing either shape - passed the old isinstance check and crashed later (#154 review F4).""" + `declared` as {nodeid: [journey, ...]}. Anything looser passed and then + crashed in `tuple(s)` or a dict lookup, exit 1 rather than a refusal + (#154 review F4, R2-3). `declared` may be absent or null: verify has + always read it as `trace.get("declared") or {}`.""" tests = trace.get("tests") if not isinstance(tests, dict): return "`tests` is missing or not an object" - if any(not isinstance(v, list) for v in tests.values()): - return "a `tests` entry is not a list" - declared = trace.get("declared", {}) - if not isinstance(declared, dict) or any( - not isinstance(v, list) for v in declared.values()): - return "`declared` is not an object of lists" + for nodeid, symbols in tests.items(): + if not isinstance(symbols, list) or any( + not (isinstance(s, list) and len(s) == 2 + and all(isinstance(x, str) for x in s)) + for s in symbols): + return f"`tests[{nodeid!r}]` is not a list of [relpath, qualname] pairs" + declared = trace.get("declared") or {} + if not isinstance(declared, dict): + return "`declared` is not an object" + for nodeid, journeys in declared.items(): + if not isinstance(journeys, list) or any(not isinstance(j, str) for j in journeys): + return f"`declared[{nodeid!r}]` is not a list of journey ids" return None @@ -365,23 +373,30 @@ def go(tmp): return go(tmp) -_STREAM_PREFIXES = ("/dev/", "/proc/") +# Device and process paths are streams however they resolve: `/dev/stdout` +# redirected to a file resolves to a REGULAR file. `/dev/shm` is ordinary +# tmpfs storage, not a stream (#154 review R2-1). +_DEVICE_PREFIXES = ("/dev/", "/proc/") +_FILE_UNDER_DEV = ("/dev/shm/",) -def is_stream(path): - """True when `path` must be written in place, not via temp + rename. +def not_a_file_target(path): + """True when `path` cannot hold a trace file: a stream or device path. - `/dev/stdout` redirected to a file resolves to a REGULAR file, so the - prefix is checked on the unresolved path; anything else that exists and is - not a regular file (a FIFO) is a stream too. The same rule lives in - tgtrace (`_is_stream`), which cannot import this module; - tests/test_pytest_adapter.py:WriterParityTests holds the two in step.""" - if os.path.abspath(path).startswith(_STREAM_PREFIXES): + Traces are written to `.tmp.` and renamed over `path`, so the + reader sees either nothing or a whole file. A stream has nowhere to put + the temp file and nothing to rename over, and writing one in place was + tried and dropped: a second `open("/dev/stdout", "w")` gets its own file + offset, so pytest's output overwrote the JSON (#154 review R2-2). So + streams are refused up front, everywhere. #154 noted no caller uses one. + + The same rule is in tgtrace (`_not_a_file_target`), which cannot import + this module; tests/test_pytest_adapter.py:WriterParityTests holds the two + copies to identical answers. harness/trace.py imports this one.""" + p = os.path.abspath(path) + if p.startswith(_DEVICE_PREFIXES) and not p.startswith(_FILE_UNDER_DEV): return True - # A directory is neither: writing to it fails either way, and callers - # that can refuse one up front (`--trace-out`) need to see it as such. - return (os.path.exists(path) and not os.path.isfile(path) - and not os.path.isdir(path)) + return os.path.exists(p) and not os.path.isfile(p) def trace_out_error(path): @@ -389,10 +404,12 @@ def trace_out_error(path): Checked BEFORE the suite runs (#154 review F9): finding out after a half-hour run that the directory does not exist throws the run away.""" - if is_stream(path): - return None - if os.path.isdir(path): - return f"--trace-out {path} is a directory" + if path.endswith(os.sep): + return f"--trace-out {path} names a directory, not a file" + if not_a_file_target(path): + return (f"--trace-out {path} is not a regular file path; the trace is " + f"written to a temp file and renamed, so streams and devices " + f"(/dev/stdout, pipes) are refused") parent = os.path.dirname(os.path.abspath(path)) if not os.path.isdir(parent): return f"--trace-out directory {parent} does not exist" @@ -407,12 +424,8 @@ def write_json_atomic(path, obj): Temp file beside the target, flushed and fsynced, then renamed: a plain `open(path, "w")` killed mid-dump leaves a truncated file that a later reader (reconcile) refuses. The temp is named `.tmp.`, as - tgtrace names its own, so harness/trace.py's sweep finds both. A stream - target is written in place (see `is_stream`).""" - if is_stream(path): - with open(path, "w") as f: - json.dump(obj, f, indent=1, sort_keys=True) - return + tgtrace names its own, so harness/trace.py's sweep finds both. Callers + refuse stream targets first (`trace_out_error`).""" tmp = f"{path}.tmp.{os.getpid()}" try: with open(tmp, "w") as f: diff --git a/testgraph/verify.py b/testgraph/verify.py index 705764e..57c725d 100644 --- a/testgraph/verify.py +++ b/testgraph/verify.py @@ -170,6 +170,11 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, # Raised out of here before: a traceback and exit 1, the code for # "tests failed", and prose under --json (#154 review F9). return None, f"cannot read registry {registry_path}: {exc}" + if not isinstance(registry, dict): + # Parsed, but a list or a string crashed later in select with an + # AttributeError traceback (#154 review R2-6). + return None, (f"registry {registry_path} is a JSON " + f"{type(registry).__name__}, not an object") db_path = db_path or os.path.join(repo, ".codegraph", "codegraph.db") if not os.path.isfile(db_path): return None, f"no codegraph index at {db_path} — run `codegraph init` in {repo}" diff --git a/tests/test_ground_truth.py b/tests/test_ground_truth.py index e96e6ac..22b1815 100644 --- a/tests/test_ground_truth.py +++ b/tests/test_ground_truth.py @@ -330,7 +330,7 @@ def test_a_worker_thread_is_recorded_by_the_setprofile_backend(self): -class TraceRunnerTest(unittest.TestCase): +class _TraceRunnerFixture: """#154: harness/trace.py read the trace with an unguarded `json.load`, and never removed the `.tmp.` a SIGKILLed tgtrace leaves.""" @@ -356,6 +356,8 @@ def _run(self, body): rc, payload = htrace.run(tmp, self._fake_python(tmp, body), ".", ".", out) return rc, payload, stale + +class TraceRunnerTest(_TraceRunnerFixture, unittest.TestCase): def test_a_truncated_trace_is_the_no_trace_case(self): rc, payload, _ = self._run('{"tests": {') self.assertEqual(rc, 0) @@ -372,7 +374,7 @@ def test_a_stale_temp_file_is_removed_before_the_run(self): -class TraceRunnerReviewTest(TraceRunnerTest): +class TraceRunnerReviewTest(_TraceRunnerFixture, unittest.TestCase): """Round-1 review of #154 (.review ledger F4, F5, F6).""" def test_f4_an_object_without_tgtrace_fields_is_the_no_trace_case(self): @@ -404,6 +406,29 @@ def test_f5_a_non_regular_out_is_refused_and_left_alone(self): with self.assertRaises(ValueError): htrace.run(tmp, "/bin/false", ".", ".", fifo) self.assertTrue(stat.S_ISFIFO(os.stat(fifo).st_mode)) + # judged after abspath, so a relative or dotted path into /dev is + # refused too (R2-4) + cwd = os.getcwd() + self.addCleanup(os.chdir, cwd) + for out in ("/tmp/../dev/stdout",): + with self.assertRaises(ValueError): + htrace.run(tmp, "/bin/false", ".", ".", out) + os.chdir("/dev") + with self.assertRaises(ValueError): + htrace.run(tmp, "/bin/false", ".", ".", "stdout") + + @unittest.skipUnless(os.path.isdir("/dev/shm") and os.access("/dev/shm", os.W_OK), + "no writable /dev/shm") + def test_r2_1_an_out_under_dev_shm_is_a_file(self): + from harness import trace as htrace + tmp = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, tmp, ignore_errors=True) + shm = tempfile.mkdtemp(dir="/dev/shm") + self.addCleanup(__import__("shutil").rmtree, shm, ignore_errors=True) + out = os.path.join(shm, "t.json") + rc, payload = htrace.run(tmp, self._fake_python(tmp, '{"tests": {}}'), + ".", ".", out) + self.assertEqual(payload, {"tests": {}}) def test_f6_the_sweep_takes_only_dead_pids(self): from harness import trace as htrace diff --git a/tests/test_pytest_adapter.py b/tests/test_pytest_adapter.py index 5bb3b67..9de9dcd 100644 --- a/tests/test_pytest_adapter.py +++ b/tests/test_pytest_adapter.py @@ -411,7 +411,7 @@ def test_incomplete_join_is_flagged_in_both_directions(self): -class RunTraceRefusalTests(unittest.TestCase): +class _AdapterRunFixture: """#108 / #154: a broken run is refused, never attributed. Each case writes what a real pytest would have left in the work dir; `_workdir` stands in for the temp dir, so nothing patches the stdlib.""" @@ -444,6 +444,8 @@ def runner(): return pa.run(self.repo, registry_path=self.registry, db_path=self.db, _runner=runner, _workdir=work, append=lambda r: True, **kw) + +class RunTraceRefusalTests(_AdapterRunFixture, unittest.TestCase): def test_refusals(self): cases = [ ("truncated", dict(trace='{"tests": {', junit=JUNIT_OK), "truncated or not JSON"), @@ -482,7 +484,7 @@ def test_trace_out_is_written_whole_and_leaves_no_temp_file(self): self.assertEqual(os.listdir(out_dir), ["t.json"]) -JUNIT_OK = RunTraceRefusalTests.JUNIT +JUNIT_OK = _AdapterRunFixture.JUNIT class WriteJsonAtomicTests(unittest.TestCase): @@ -522,7 +524,7 @@ def test_a_json_refusal_is_json(self): -class ReviewRound1Tests(RunTraceRefusalTests): +class ReviewRound1Tests(_AdapterRunFixture, unittest.TestCase): """Round-1 review of #154 (.review ledger F1, F3, F4, F8, F9).""" def test_f3_a_truncated_junit_report_is_refused_not_a_crash(self): @@ -536,7 +538,12 @@ def test_f3_a_truncated_junit_report_is_refused_not_a_crash(self): def test_f4_an_object_without_tgtrace_fields_is_refused(self): for text in ("{}", '{"tests": 3}', '{"tests": {}, "declared": 3}', - '{"tests": {"t": 3}}'): + '{"tests": {"t": 3}}', + # element level (R2-3): each entry a [relpath, qualname] + # pair, each declared journey a string + '{"tests": {"t": [1]}}', '{"tests": {"t": [["x"]]}}', + '{"tests": {"t": [["a", 1]]}}', + '{"tests": {}, "declared": {"t": [["J1"]]}}'): with self.subTest(text): for f in os.listdir(self.work): os.remove(os.path.join(self.work, f)) @@ -546,7 +553,12 @@ def test_f4_an_object_without_tgtrace_fields_is_refused(self): def test_f9_a_trace_out_that_cannot_be_written_is_refused_before_the_run(self): ran = [] - bad = [os.path.join(self.tmp, "no", "such", "t.json"), self.tmp] + fifo = os.path.join(self.tmp, "pipe") + os.mkfifo(fifo) + bad = [os.path.join(self.tmp, "no", "such", "t.json"), self.tmp, + os.path.join(self.tmp, "newdir") + os.sep, # R2-5 + fifo, "/dev/stdout", "/dev/stdot", # R2-1/R2-2/R2-5 + "/tmp/../dev/stdout"] for out in bad: with self.subTest(out): def runner(): @@ -560,23 +572,35 @@ def runner(): self.assertIn("--trace-out", err) self.assertEqual(ran, [], "the suite ran before --trace-out was checked") - def test_f1_trace_out_to_a_fifo_is_written_through(self): - fifo = os.path.join(self.tmp, "pipe") - os.mkfifo(fifo) - got = [] - import threading - t = threading.Thread(target=lambda: got.append(open(fifo).read()), daemon=True) - t.start() - summary, err = self._run(trace=self.WHOLE, junit=JUNIT_OK, trace_out=fifo) - t.join(5) + def test_declared_null_is_accepted_as_empty(self): + """verify has always read `declared` as `.get("declared") or {}`; + the shape check must not turn that into a refusal (R2-3 parity).""" + summary, err = self._run(trace='{"tests": {}, "declared": null}', junit=JUNIT_OK) self.assertIsNone(err) - self.assertEqual(json.loads(got[0]), json.loads(self.WHOLE)) - self.assertTrue(__import__("stat").S_ISFIFO(os.stat(fifo).st_mode)) + + @unittest.skipUnless(os.path.isdir("/dev/shm") and os.access("/dev/shm", os.W_OK), + "no writable /dev/shm") + def test_r2_1_a_trace_out_under_dev_shm_is_a_file_written_atomically(self): + out_dir = tempfile.mkdtemp(dir="/dev/shm") + self.addCleanup(shutil.rmtree, out_dir, ignore_errors=True) + out = os.path.join(out_dir, "t.json") + seen = [] + real_replace = os.replace + + def spy(src, dst): + seen.append(os.path.basename(src)) + return real_replace(src, dst) + with unittest.mock.patch("os.replace", spy): + summary, err = self._run(trace=self.WHOLE, junit=JUNIT_OK, trace_out=out) + self.assertIsNone(err) + self.assertEqual(seen, [f"t.json.tmp.{os.getpid()}"]) + with open(out) as fh: + self.assertEqual(json.load(fh), json.loads(self.WHOLE)) class WriterParityTests(unittest.TestCase): - """F1/F2/F8: testgraph's atomic writer and tgtrace's are two copies of one - rule (tgtrace must stay standalone, so it cannot import the other). This + """F8/R2: testgraph's target rule and atomic writer and tgtrace's are two + copies (tgtrace must stay standalone, so it cannot import the other). This holds them to the same behaviour rather than a comment saying they match.""" def _writers(self): @@ -585,50 +609,82 @@ def _writers(self): import tgtrace finally: sys.path.pop(0) - return {"adapter": pa.write_json_atomic, "tgtrace": tgtrace._write} + return {"adapter": pa.write_json_atomic, "tgtrace": tgtrace._write, + "tgtrace_rule": tgtrace._not_a_file_target} - def test_both_treat_dev_paths_as_streams(self): - """/dev/stdout redirected to a file resolves to a regular file, so an - `isfile` test alone sends it down the temp-and-rename path, which - cannot create a file in /dev (F2).""" - sys.path.insert(0, os.path.join(ROOT_DIR, "harness", "plugin")) - import tgtrace - sys.path.pop(0) - for path in ("/dev/stdout", "/dev/fd/1", "/proc/self/fd/1"): - with self.subTest(path): - self.assertTrue(pa.is_stream(path)) - self.assertTrue(tgtrace._is_stream(path)) + def test_both_copies_of_the_target_rule_agree(self): + """One rule, two copies (tgtrace cannot import testgraph): same + answer on every case, and the answer is the intended one.""" + rules = self._writers() tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) regular = os.path.join(tmp, "t.json") open(regular, "w").close() - for path in (regular, os.path.join(tmp, "absent.json")): - self.assertFalse(pa.is_stream(path)) - self.assertFalse(tgtrace._is_stream(path)) - - def test_both_write_dev_stdout_redirected_to_a_file(self): - """F2, end to end: a child whose stdout IS a regular file writes - through /dev/stdout with each writer.""" + fifo = os.path.join(tmp, "pipe") + os.mkfifo(fifo) + cases = { + "/dev/stdout": True, "/dev/fd/1": True, "/proc/self/fd/1": True, + "/dev/stdot": True, "/dev/null": True, "/tmp/../dev/stdout": True, + fifo: True, tmp: True, + "/dev/shm/x.json": False, regular: False, + os.path.join(tmp, "absent.json"): False, + } + for path, expected in cases.items(): + with self.subTest(path): + self.assertEqual(pa.not_a_file_target(path), expected) + self.assertEqual(rules["tgtrace_rule"](path), expected) + # relative paths are judged after abspath (R2-4) + cwd = os.getcwd() + self.addCleanup(os.chdir, cwd) + os.chdir("/dev") + self.assertTrue(pa.not_a_file_target("stdout")) + self.assertTrue(rules["tgtrace_rule"]("stdout")) + + def test_relative_and_dotted_dev_paths_with_stdout_redirected_to_a_file(self): + """R2-4: with stdout redirected to a file, `/dev/stdout` resolves to a + regular file, so only the abspath + prefix test catches `stdout` from + cwd /dev or `/tmp/../dev/stdout`. Run in a child whose stdout IS a file.""" + import subprocess + code = ( + "import sys; sys.path[:0] = [sys.argv[1], sys.argv[1] + '/harness/plugin']\n" + "from testgraph import pytest_adapter as pa; import tgtrace\n" + "for p in ('stdout', '/tmp/../dev/stdout'):\n" + " sys.stderr.write(f'{pa.not_a_file_target(p)} {tgtrace._not_a_file_target(p)}\\n')\n" + ) + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + with open(os.path.join(tmp, "out.txt"), "w") as fh: + proc = subprocess.run([sys.executable, "-c", code, ROOT_DIR], cwd="/dev", + stdout=fh, stderr=subprocess.PIPE, text=True) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stderr.split(), ["True"] * 4) + + def test_tgtrace_refuses_a_stream_before_any_test_runs(self): + """R2-2: writing a stream in place corrupted it (a second open of + /dev/stdout has its own offset). Real pytest, stdout redirected to a + file: the run is refused at configure, and says why.""" import subprocess tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) - for name, code in ( - ("adapter", "from testgraph import pytest_adapter as w; w.write_json_atomic('/dev/stdout', {'ok': 1})"), - ("tgtrace", "import sys; sys.path.insert(0, 'harness/plugin'); import tgtrace as w; w._write('/dev/stdout', {'ok': 1})"), - ): - with self.subTest(name): - out = os.path.join(tmp, name + ".json") - with open(out, "w") as fh: - proc = subprocess.run([sys.executable, "-c", code], cwd=ROOT_DIR, - stdout=fh, stderr=subprocess.PIPE, text=True) - self.assertEqual(proc.returncode, 0, proc.stderr) - with open(out) as fh: - self.assertEqual(json.load(fh), {"ok": 1}) + with open(os.path.join(tmp, "test_x.py"), "w") as fh: + fh.write("def test_x():\n assert True\n") + env = dict(os.environ, TGTRACE_OUT="/dev/stdout", + PYTHONPATH=os.path.join(ROOT_DIR, "harness", "plugin")) + out = os.path.join(tmp, "out.txt") + with open(out, "w") as fh: + proc = subprocess.run([sys.executable, "-m", "pytest", "-p", "tgtrace", + "-q", "-p", "no:cacheprovider", tmp], + cwd=tmp, env=env, stdout=fh, stderr=subprocess.STDOUT) + with open(out) as fh: + text = fh.read() + self.assertNotEqual(proc.returncode, 0) + self.assertIn("not a regular file path", text) + self.assertNotIn("1 passed", text) def test_both_name_their_temp_file_the_same_way(self): """F8: the harness sweep matches `.tmp.`; a killed adapter used to leave `..tmp.`, which nothing swept.""" - for name, write in self._writers().items(): + for name, write in list(self._writers().items())[:2]: with self.subTest(name): tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) diff --git a/tests/test_verify.py b/tests/test_verify.py index 6d605f9..56f1188 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -369,7 +369,7 @@ def runner(): self.assertEqual(len(closed), len(opened)) -class UnreadableTraceTests(unittest.TestCase): +class _VerifyRunFixture: """Issue #108: a truncated trace made `json.load` raise, and the uncaught exception exited 1 — the code for "tests failed". A missing trace read as an empty one, so every selected journey was reported as having no @@ -407,6 +407,8 @@ def runner(): db_path=self.db, _runner=runner, _workdir=work, append=lambda r: True) + +class UnreadableTraceTests(_VerifyRunFixture, unittest.TestCase): def test_a_truncated_trace_is_refused_not_a_crash(self): summary, err = self._run('{"tests": {"t::a": [["app/svc.py", "hand') self.assertIsNone(summary) @@ -473,22 +475,25 @@ def test_a_write_that_dies_leaves_no_partial_file(self): self.assertFalse(os.path.exists(out)) self.assertEqual(os.listdir(tmp), []) - def test_a_stream_target_is_written_directly(self): - """#154: `TGTRACE_OUT=/dev/stdout` failed, because the temp file - cannot be created beside a device node. A FIFO stands in for it.""" + def test_a_stream_target_is_refused_at_configure(self): + """R2-2: a stream was written in place, and a second open of + /dev/stdout has its own offset, so pytest's output overwrote the + JSON. Now refused before any test runs; a FIFO is left alone.""" from harness.plugin import tgtrace tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) fifo = os.path.join(tmp, "pipe") os.mkfifo(fifo) - got = [] - import threading - reader = threading.Thread( - target=lambda: got.append(open(fifo).read()), daemon=True) - reader.start() - tgtrace._write(fifo, {"tests": {}}) - reader.join(5) - self.assertEqual(json.loads(got[0]), {"tests": {}}) + config = types.SimpleNamespace(option=types.SimpleNamespace(numprocesses=None)) + for target in (fifo, "/dev/stdout"): + with self.subTest(target), \ + unittest.mock.patch.object(tgtrace, "_out_path", target), \ + unittest.mock.patch.object(tgtrace, "_start", lambda: None), \ + unittest.mock.patch.dict(os.environ, {}, clear=False): + os.environ.pop("PYTEST_XDIST_WORKER", None) + with self.assertRaises(RuntimeError) as cm: + tgtrace.pytest_configure(config) + self.assertIn("not a regular file path", str(cm.exception)) self.assertEqual(sorted(os.listdir(tmp)), ["pipe"]) def test_a_whole_write_lands(self): @@ -638,7 +643,7 @@ class C: -class ReviewRound1VerifyTests(UnreadableTraceTests): +class ReviewRound1VerifyTests(_VerifyRunFixture, unittest.TestCase): """Round-1 review of #154 (.review ledger C1, F3, F9).""" def test_c1_a_missing_junit_report_is_no_verdicts_not_a_pass(self): @@ -663,6 +668,15 @@ def runner(): self.assertIsNone(summary) self.assertIn("JUnit XML is truncated or not XML", err) + def test_r2_6_a_registry_that_is_not_an_object_is_refused(self): + bad = os.path.join(self.tmp, "list.json") + with open(bad, "w") as fh: + fh.write("[]") + summary, err = verify.run(self.repo, "HEAD~1", "HEAD", registry_path=bad, + db_path=self.db, append=lambda r: True) + self.assertIsNone(summary) + self.assertIn("not an object", err) + def test_f9_an_unreadable_registry_is_a_json_refusal(self): out = io.StringIO() with contextlib.redirect_stdout(out): From dfa82dde809512cc0edaf73210a0799bcabf93eb Mon Sep 17 00:00:00 2001 From: Eric Minish Date: Sun, 15 Nov 2026 16:02:00 -0500 Subject: [PATCH 4/4] fix: tgtrace refusals write nothing; rule resolves the target directory (#154 review round 3) - R3-1: pytest calls unconfigure even after configure raised; tgtrace now writes and stops only if tracing started (it had replaced a refused FIFO on 3.11 and raised on 3.12+), for both the stream and xdist refusals - R3-2: the real-pytest tgtrace test skips without pytest (CI matrix) - R3-3: declared ids need only be hashable; covers(1) is a bad declaration for verify to report, not a broken trace - R3-4: the target rule resolves the directory with realpath, so a symlinked parent or //dev is still /dev - R3-5: registry.load refuses JSON that is not an object, for every caller - R3-6: x/, x/. and x/.. are refused; writers normalise the path once - R3-7: a directory --trace-out says so - R3-8/R3-9: one ROOT in harness/trace.py; tests pick writers by key --- harness/plugin/tgtrace.py | 20 +++++++- harness/trace.py | 7 ++- testgraph/pytest_adapter.py | 20 ++++++-- testgraph/registry.py | 8 +++- testgraph/verify.py | 6 +-- tests/test_pytest_adapter.py | 88 ++++++++++++++++++++++++++++++++---- tests/test_verify.py | 31 +++++++++++++ 7 files changed, 156 insertions(+), 24 deletions(-) diff --git a/harness/plugin/tgtrace.py b/harness/plugin/tgtrace.py index 40ca5f2..74d05cc 100644 --- a/harness/plugin/tgtrace.py +++ b/harness/plugin/tgtrace.py @@ -58,6 +58,7 @@ ) _current = None # set() while a test body is running, else None +_started = False # tracing began; unconfigure does nothing otherwise _results = {} _declared = {} # nodeid -> [journey id, ...] the test CLAIMS to cover @@ -141,7 +142,9 @@ def _stop_profile(): def _start(): + global _started _start_monitoring() if _HAVE_MONITORING else _start_profile() + _started = True def _stop(): @@ -179,8 +182,14 @@ def pytest_configure(config): def pytest_unconfigure(config): - if not _out_path: + # pytest calls this even when pytest_configure RAISED (an xdist or + # stream-target refusal). Stopping a tracer that never started raised on + # sys.monitoring, and writing the empty payload replaced the refused + # target on setprofile (#154 review R3-1). + global _started + if not _out_path or not _started: return + _started = False _stop() payload = { "root": _root, @@ -204,7 +213,13 @@ def _not_a_file_target(path): A copy of `testgraph.pytest_adapter.not_a_file_target`, because this plugin imports nothing from testgraph; WriterParityTests in tests/test_pytest_adapter.py holds the two to identical answers.""" + if path.endswith(os.sep) or os.path.basename(path) in (".", ".."): + return True # names a directory, however it is spelled (R3-6) p = os.path.abspath(path) + # Resolve the DIRECTORY, not the leaf: `/dev/stdout` itself resolves into + # whatever stdout is redirected to, but a symlinked parent (`d -> /dev`) + # or a POSIX `//dev` must still read as /dev (#154 review R3-4). + p = os.path.join(os.path.realpath(os.path.dirname(p)), os.path.basename(p)) if p.startswith(_DEVICE_PREFIXES) and not p.startswith(_FILE_UNDER_DEV): return True return os.path.exists(p) and not os.path.isfile(p) @@ -219,7 +234,8 @@ def _write(out_path, payload): crash from leaving a zero-length target, which ext4/xfs can otherwise do when the rename reaches disk before the data. Stream targets were refused in `pytest_configure`.""" - directory = os.path.dirname(os.path.abspath(out_path)) + out_path = os.path.abspath(out_path) # one spelling for check and write (R3-6) + directory = os.path.dirname(out_path) if directory: os.makedirs(directory, exist_ok=True) tmp = f"{out_path}.tmp.{os.getpid()}" diff --git a/harness/trace.py b/harness/trace.py index f5d5507..022a518 100644 --- a/harness/trace.py +++ b/harness/trace.py @@ -26,13 +26,12 @@ import subprocess import sys -_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if _ROOT not in sys.path: - sys.path.insert(0, _ROOT) -from testgraph import pytest_adapter as pa # noqa: E402 HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) +if ROOT not in sys.path: + sys.path.insert(0, ROOT) +from testgraph import pytest_adapter as pa # noqa: E402 # ONLY this directory goes on the target's PYTHONPATH. `harness/` itself holds # `trace.py`, which would shadow the stdlib `trace` module for every import in # the traced suite. diff --git a/testgraph/pytest_adapter.py b/testgraph/pytest_adapter.py index 5f6a54e..771d39e 100644 --- a/testgraph/pytest_adapter.py +++ b/testgraph/pytest_adapter.py @@ -323,7 +323,11 @@ def payload_shape_error(trace): if not isinstance(declared, dict): return "`declared` is not an object" for nodeid, journeys in declared.items(): - if not isinstance(journeys, list) or any(not isinstance(j, str) for j in journeys): + # Each id must be hashable (it is looked up). It need NOT be a string: + # tgtrace writes whatever `covers(...)` was given, and `covers(1)` is a + # bad declaration that verify reports, not a broken trace (R3-3). + if not isinstance(journeys, list) or any( + isinstance(j, (list, dict)) for j in journeys): return f"`declared[{nodeid!r}]` is not a list of journey ids" return None @@ -393,7 +397,13 @@ def not_a_file_target(path): The same rule is in tgtrace (`_not_a_file_target`), which cannot import this module; tests/test_pytest_adapter.py:WriterParityTests holds the two copies to identical answers. harness/trace.py imports this one.""" + if path.endswith(os.sep) or os.path.basename(path) in (".", ".."): + return True # names a directory, however it is spelled (R3-6) p = os.path.abspath(path) + # Resolve the DIRECTORY, not the leaf: `/dev/stdout` itself resolves into + # whatever stdout is redirected to, but a symlinked parent (`d -> /dev`) + # or a POSIX `//dev` must still read as /dev (#154 review R3-4). + p = os.path.join(os.path.realpath(os.path.dirname(p)), os.path.basename(p)) if p.startswith(_DEVICE_PREFIXES) and not p.startswith(_FILE_UNDER_DEV): return True return os.path.exists(p) and not os.path.isfile(p) @@ -404,8 +414,9 @@ def trace_out_error(path): Checked BEFORE the suite runs (#154 review F9): finding out after a half-hour run that the directory does not exist throws the run away.""" - if path.endswith(os.sep): - return f"--trace-out {path} names a directory, not a file" + if (path.endswith(os.sep) or os.path.basename(path) in (".", "..") + or os.path.isdir(path)): + return f"--trace-out {path} is a directory, not a file" if not_a_file_target(path): return (f"--trace-out {path} is not a regular file path; the trace is " f"written to a temp file and renamed, so streams and devices " @@ -426,6 +437,9 @@ def write_json_atomic(path, obj): reader (reconcile) refuses. The temp is named `.tmp.`, as tgtrace names its own, so harness/trace.py's sweep finds both. Callers refuse stream targets first (`trace_out_error`).""" + # One spelling for the check and the write (R3-6): `gone/../t.json` + # passes a check on abspath but fails as a raw temp path. + path = os.path.abspath(path) tmp = f"{path}.tmp.{os.getpid()}" try: with open(tmp, "w") as f: diff --git a/testgraph/registry.py b/testgraph/registry.py index 7bf6031..9b4c921 100644 --- a/testgraph/registry.py +++ b/testgraph/registry.py @@ -12,8 +12,14 @@ def load(path): + """Parse a registry file. Raises OSError or ValueError, including for JSON + that is not an object: a list used to load fine and crash later, in + whichever caller first indexed it (#154 review R3-5).""" with open(path) as f: - return json.load(f) + data = json.load(f) + if not isinstance(data, dict): + raise ValueError(f"the registry is a JSON {type(data).__name__}, not an object") + return data # Where a registry lives, in search order. See `search_dirs`. diff --git a/testgraph/verify.py b/testgraph/verify.py index 57c725d..958b00f 100644 --- a/testgraph/verify.py +++ b/testgraph/verify.py @@ -170,11 +170,7 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, # Raised out of here before: a traceback and exit 1, the code for # "tests failed", and prose under --json (#154 review F9). return None, f"cannot read registry {registry_path}: {exc}" - if not isinstance(registry, dict): - # Parsed, but a list or a string crashed later in select with an - # AttributeError traceback (#154 review R2-6). - return None, (f"registry {registry_path} is a JSON " - f"{type(registry).__name__}, not an object") + db_path = db_path or os.path.join(repo, ".codegraph", "codegraph.db") if not os.path.isfile(db_path): return None, f"no codegraph index at {db_path} — run `codegraph init` in {repo}" diff --git a/tests/test_pytest_adapter.py b/tests/test_pytest_adapter.py index 9de9dcd..cf5de89 100644 --- a/tests/test_pytest_adapter.py +++ b/tests/test_pytest_adapter.py @@ -8,6 +8,7 @@ to be REPORTED rather than swallowed. """ import contextlib +import importlib.util import io import json import os @@ -26,6 +27,8 @@ from testgraph import pytest_adapter as pa # noqa: E402 from test_core import _db_on_disk, _git_repo, _registry_file, build_fixture # noqa: E402 +HAVE_PYTEST = importlib.util.find_spec("pytest") is not None + def build_conn(): """Two journeys' worth of graph: J1 reaches handler->helper, J2 is a @@ -572,6 +575,57 @@ def runner(): self.assertIn("--trace-out", err) self.assertEqual(ran, [], "the suite ran before --trace-out was checked") + def test_r3_3_a_non_string_declaration_is_not_a_broken_trace(self): + """`covers(1)` is a bad decorator, not a broken run: tgtrace writes + what `tg_journeys` holds, so an int must not refuse every verdict.""" + summary, err = self._run(trace='{"tests": {}, "declared": {"t": [1]}}', + junit=JUNIT_OK) + self.assertIsNone(err) + + def test_r3_7_a_directory_trace_out_says_directory(self): + summary, err = self._run(trace=self.WHOLE, junit=JUNIT_OK, trace_out=self.tmp) + self.assertIsNone(summary) + self.assertIn("is a directory", err) + + def test_r3_6_a_trace_out_naming_a_directory_is_refused_before_the_run(self): + ran = [] + + def runner(): + ran.append(1) + return types.SimpleNamespace(returncode=0, stderr="") + for out in (os.path.join(self.tmp, "x", "new.json") + "/.", + os.path.join(self.tmp, "nosuchdir") + "/."): + with self.subTest(out): + summary, err = pa.run(self.repo, registry_path=self.registry, + db_path=self.db, _runner=runner, + _workdir=self.work, append=lambda r: True, + trace_out=out) + self.assertIsNone(summary) + self.assertIn("--trace-out", err) + self.assertEqual(ran, []) + + def test_r3_6_a_dotted_trace_out_is_written_where_the_check_looked(self): + os.makedirs(os.path.join(self.tmp, "real")) + out = os.path.join(self.tmp, "gone", "..", "real", "t.json") + summary, err = self._run(trace=self.WHOLE, junit=JUNIT_OK, trace_out=out) + self.assertIsNone(err) + self.assertTrue(os.path.isfile(os.path.join(self.tmp, "real", "t.json"))) + + def test_r3_5_a_registry_that_is_not_an_object_is_refused_before_the_run(self): + bad = os.path.join(self.tmp, "list.json") + with open(bad, "w") as fh: + fh.write("[]") + ran = [] + + def runner(): + ran.append(1) + return types.SimpleNamespace(returncode=0, stderr="") + summary, err = pa.run(self.repo, registry_path=bad, db_path=self.db, + _runner=runner, _workdir=self.work, append=lambda r: True) + self.assertIsNone(summary) + self.assertIn("not an object", err) + self.assertEqual(ran, []) + def test_declared_null_is_accepted_as_empty(self): """verify has always read `declared` as `.get("declared") or {}`; the shape check must not turn that into a refusal (R2-3 parity).""" @@ -609,13 +663,20 @@ def _writers(self): import tgtrace finally: sys.path.pop(0) - return {"adapter": pa.write_json_atomic, "tgtrace": tgtrace._write, - "tgtrace_rule": tgtrace._not_a_file_target} + return {"adapter": pa.write_json_atomic, "tgtrace": tgtrace._write} + + def _tgtrace_rule(self): + sys.path.insert(0, os.path.join(ROOT_DIR, "harness", "plugin")) + try: + import tgtrace + finally: + sys.path.pop(0) + return tgtrace._not_a_file_target def test_both_copies_of_the_target_rule_agree(self): """One rule, two copies (tgtrace cannot import testgraph): same answer on every case, and the answer is the intended one.""" - rules = self._writers() + tg_rule = self._tgtrace_rule() tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) regular = os.path.join(tmp, "t.json") @@ -628,17 +689,23 @@ def test_both_copies_of_the_target_rule_agree(self): fifo: True, tmp: True, "/dev/shm/x.json": False, regular: False, os.path.join(tmp, "absent.json"): False, + # a path that names a directory, however spelled (R3-6) + os.path.join(tmp, "new.json") + "/.": True, + os.path.join(tmp, "new.json") + "/": True, + os.path.join(tmp, "a", ".."): True, + # a dotted path to a real file location is still a file + os.path.join(tmp, "a", "..", "b.json"): False, } for path, expected in cases.items(): with self.subTest(path): self.assertEqual(pa.not_a_file_target(path), expected) - self.assertEqual(rules["tgtrace_rule"](path), expected) + self.assertEqual(tg_rule(path), expected) # relative paths are judged after abspath (R2-4) cwd = os.getcwd() self.addCleanup(os.chdir, cwd) os.chdir("/dev") self.assertTrue(pa.not_a_file_target("stdout")) - self.assertTrue(rules["tgtrace_rule"]("stdout")) + self.assertTrue(tg_rule("stdout")) def test_relative_and_dotted_dev_paths_with_stdout_redirected_to_a_file(self): """R2-4: with stdout redirected to a file, `/dev/stdout` resolves to a @@ -648,17 +715,20 @@ def test_relative_and_dotted_dev_paths_with_stdout_redirected_to_a_file(self): code = ( "import sys; sys.path[:0] = [sys.argv[1], sys.argv[1] + '/harness/plugin']\n" "from testgraph import pytest_adapter as pa; import tgtrace\n" - "for p in ('stdout', '/tmp/../dev/stdout'):\n" + "for p in ('stdout', '/tmp/../dev/stdout', '//dev/stdout', sys.argv[2] + '/stdout'):\n" " sys.stderr.write(f'{pa.not_a_file_target(p)} {tgtrace._not_a_file_target(p)}\\n')\n" ) tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + link = os.path.join(tmp, "devlnk") + os.symlink("/dev", link) # a symlinked parent (R3-4) with open(os.path.join(tmp, "out.txt"), "w") as fh: - proc = subprocess.run([sys.executable, "-c", code, ROOT_DIR], cwd="/dev", + proc = subprocess.run([sys.executable, "-c", code, ROOT_DIR, link], cwd="/dev", stdout=fh, stderr=subprocess.PIPE, text=True) self.assertEqual(proc.returncode, 0, proc.stderr) - self.assertEqual(proc.stderr.split(), ["True"] * 4) + self.assertEqual(proc.stderr.split(), ["True"] * 8) + @unittest.skipUnless(HAVE_PYTEST, "pytest is not installed") # R3-2 def test_tgtrace_refuses_a_stream_before_any_test_runs(self): """R2-2: writing a stream in place corrupted it (a second open of /dev/stdout has its own offset). Real pytest, stdout redirected to a @@ -684,7 +754,7 @@ def test_tgtrace_refuses_a_stream_before_any_test_runs(self): def test_both_name_their_temp_file_the_same_way(self): """F8: the harness sweep matches `.tmp.`; a killed adapter used to leave `..tmp.`, which nothing swept.""" - for name, write in list(self._writers().items())[:2]: + for name, write in self._writers().items(): with self.subTest(name): tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) diff --git a/tests/test_verify.py b/tests/test_verify.py index 56f1188..a458fcb 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -496,12 +496,43 @@ def test_a_stream_target_is_refused_at_configure(self): self.assertIn("not a regular file path", str(cm.exception)) self.assertEqual(sorted(os.listdir(tmp)), ["pipe"]) + def test_r3_1_unconfigure_after_a_refusal_touches_nothing(self): + """pytest calls pytest_unconfigure even when pytest_configure raised. + It used to run _stop + _write anyway: on the setprofile backend (3.11) + that replaced the refused FIFO with a regular file; on sys.monitoring + it raised `tool 3 is not in use`, exit 1. Both backends, both + refusals (stream target, xdist).""" + from harness.plugin import tgtrace + import stat + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + fifo = os.path.join(tmp, "pipe") + os.mkfifo(fifo) + plain = os.path.join(tmp, "t.json") + for backend in (True, False): + for target, xdist in ((fifo, None), (plain, 2)): + with self.subTest(monitoring=backend, target=target), \ + unittest.mock.patch.object(tgtrace, "_HAVE_MONITORING", backend), \ + unittest.mock.patch.object(tgtrace, "_out_path", target), \ + unittest.mock.patch.object(tgtrace, "_started", False): + os.environ.pop("PYTEST_XDIST_WORKER", None) + config = types.SimpleNamespace( + option=types.SimpleNamespace(numprocesses=xdist)) + with self.assertRaises(RuntimeError): + tgtrace.pytest_configure(config) + tgtrace.pytest_unconfigure(config) # must not raise or write + self.assertTrue(stat.S_ISFIFO(os.stat(fifo).st_mode)) + self.assertFalse(os.path.exists(plain)) + def test_a_whole_write_lands(self): from harness.plugin import tgtrace tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) out = os.path.join(tmp, "trace.json") + # `_started`: unconfigure writes only after configure started tracing + # (R3-1); this simulates a run that did. with unittest.mock.patch.object(tgtrace, "_out_path", out), \ + unittest.mock.patch.object(tgtrace, "_started", True), \ unittest.mock.patch.object(tgtrace, "_stop", lambda: None): tgtrace.pytest_unconfigure(None) with open(out) as fh: