diff --git a/CHANGELOG.md b/CHANGELOG.md index 90c9091..3adbd34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,6 +128,30 @@ below is on `main` or in the branch that introduced this file. ### Fixed +- **A refusal under `--json` is JSON** (#154). `verify --json` and + `pytest_adapter --json` printed a refusal as prose, so a JSON consumer could not + parse the reason. They now print `{"refused": true, "error": "..."}`, still exit 2. +- **`verify` and the pytest adapter refuse a broken run in the same order and + words** (#154). Both now run pytest through one helper: a missing trace (with + pytest's stderr tail), then a truncated one, then one that is not an object. The + adapter's missing-JUnit refusal now carries the stderr tail too. +- **Trace files survive crashes** (#154). tgtrace flushes and fsyncs before its + atomic rename, so an OS crash can no longer leave a zero-length trace; + `pytest_adapter --trace-out` is written the same way instead of with a plain + open-and-dump. +- **Stream targets are refused up front** (#154). `TGTRACE_OUT`, `--trace-out` and + `harness/trace.py --out` must name a file: `/dev/stdout`, pipes and other device + paths are refused before any test runs, with a message saying why. Writing a + stream in place was tried and dropped, because a second open of `/dev/stdout` + has its own file offset and pytest's output overwrote the JSON. `/dev/shm` + counts as a file location. +- **More broken runs are refused instead of crashing** (#154). A truncated JUnit + report, a trace whose `tests`/`declared` entries are not tgtrace's shape, and a + registry that is unreadable or not a JSON object now refuse with exit 2; they + used to raise and exit 1, the code for "tests failed". `--trace-out` is checked + before the suite runs, not after. +- **`harness/trace.py`** reads an unreadable or foreign trace as no trace instead of + raising, and removes only the `.tmp.` files left by dead processes. - **`verify` works from an installed wheel.** The wheel shipped `testgraph/` only, so the README's headline `verify` command failed after `pip install testgraph` with "cannot find harness/plugin/tgtrace.py". The wheel now carries the trace plugin as diff --git a/harness/plugin/tgtrace.py b/harness/plugin/tgtrace.py index e2ee3fe..74d05cc 100644 --- a/harness/plugin/tgtrace.py +++ b/harness/plugin/tgtrace.py @@ -58,6 +58,7 @@ ) _current = None # set() while a test body is running, else None +_started = False # tracing began; unconfigure does nothing otherwise _results = {} _declared = {} # nodeid -> [journey id, ...] the test CLAIMS to cover @@ -141,7 +142,9 @@ def _stop_profile(): def _start(): + global _started _start_monitoring() if _HAVE_MONITORING else _start_profile() + _started = True def _stop(): @@ -167,12 +170,26 @@ def pytest_configure(config): "overwrite one another's trace and the result would silently be " "empty. Re-run without -n/--numprocesses." ) + # Refused here, before any test runs, rather than failing at unconfigure + # after the whole suite (#154 review R2-2). + if _not_a_file_target(_out_path): + raise RuntimeError( + f"TGTRACE_OUT={_out_path} is not a regular file path: the trace is " + f"written to a temp file and renamed, so streams and devices " + f"(/dev/stdout, pipes) are refused. Point it at a file." + ) _start() def pytest_unconfigure(config): - if not _out_path: + # pytest calls this even when pytest_configure RAISED (an xdist or + # stream-target refusal). Stopping a tracer that never started raised on + # sys.monitoring, and writing the empty payload replaced the refused + # target on setprofile (#154 review R3-1). + global _started + if not _out_path or not _started: return + _started = False _stop() payload = { "root": _root, @@ -180,17 +197,54 @@ def pytest_unconfigure(config): "tests": {tid: sorted(map(list, syms)) for tid, syms in _results.items()}, "declared": {tid: list(j) for tid, j in sorted(_declared.items())}, } - directory = os.path.dirname(os.path.abspath(_out_path)) + _write(_out_path, payload) + + +# Device and process paths are streams however they resolve: `/dev/stdout` +# redirected to a file resolves to a REGULAR file. `/dev/shm` is ordinary +# tmpfs storage, not a stream (#154 review R2-1). +_DEVICE_PREFIXES = ("/dev/", "/proc/") +_FILE_UNDER_DEV = ("/dev/shm/",) + + +def _not_a_file_target(path): + """True when `path` cannot hold a trace file: a stream or device path. + + A copy of `testgraph.pytest_adapter.not_a_file_target`, because this + plugin imports nothing from testgraph; WriterParityTests in + tests/test_pytest_adapter.py holds the two to identical answers.""" + if path.endswith(os.sep) or os.path.basename(path) in (".", ".."): + return True # names a directory, however it is spelled (R3-6) + p = os.path.abspath(path) + # Resolve the DIRECTORY, not the leaf: `/dev/stdout` itself resolves into + # whatever stdout is redirected to, but a symlinked parent (`d -> /dev`) + # or a POSIX `//dev` must still read as /dev (#154 review R3-4). + p = os.path.join(os.path.realpath(os.path.dirname(p)), os.path.basename(p)) + if p.startswith(_DEVICE_PREFIXES) and not p.startswith(_FILE_UNDER_DEV): + return True + return os.path.exists(p) and not os.path.isfile(p) + + +def _write(out_path, payload): + """Write the payload so `out_path` is either absent or whole. + + Beside the target, then rename (#108): a run killed mid-dump used to leave + a truncated file where a reader expects a whole one. os.replace is atomic + on the same filesystem; the flush + fsync before it (#154) keeps an OS + crash from leaving a zero-length target, which ext4/xfs can otherwise do + when the rename reaches disk before the data. Stream targets were refused + in `pytest_configure`.""" + out_path = os.path.abspath(out_path) # one spelling for check and write (R3-6) + directory = os.path.dirname(out_path) if directory: os.makedirs(directory, exist_ok=True) - # Write beside the target and rename (#108): a run killed mid-dump used to - # leave a truncated file where a reader expects a whole one. os.replace is - # atomic on the same filesystem, so the target is either absent or complete. - tmp = f"{_out_path}.tmp.{os.getpid()}" + tmp = f"{out_path}.tmp.{os.getpid()}" try: with open(tmp, "w") as f: json.dump(payload, f, indent=1, sort_keys=True) - os.replace(tmp, _out_path) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, out_path) except BaseException: try: os.remove(tmp) diff --git a/harness/trace.py b/harness/trace.py index 6bac53d..022a518 100644 --- a/harness/trace.py +++ b/harness/trace.py @@ -22,17 +22,54 @@ import argparse import json import os +import re import subprocess import sys + HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) +if ROOT not in sys.path: + sys.path.insert(0, ROOT) +from testgraph import pytest_adapter as pa # noqa: E402 # ONLY this directory goes on the target's PYTHONPATH. `harness/` itself holds # `trace.py`, which would shadow the stdlib `trace` module for every import in # the traced suite. PLUGIN_DIR = os.path.join(HERE, "plugin") +def _pid_alive(pid): + try: + os.kill(pid, 0) + except (ProcessLookupError, OverflowError): + return False + except PermissionError: + return True # exists, owned by someone else + return True + + +def _sweep_stale_temps(out): + """Remove `.tmp.` files whose writer is dead. + + A run SIGKILLed mid-dump leaves tgtrace's temp behind (#154) and nothing + else removes it. Only `.tmp.` names, and only dead pids: a broader + glob took unrelated files (`a.json.tmp.json`) and a CONCURRENT run's + in-progress temp, whose rename then failed (#154 review F6).""" + name = re.compile(re.escape(os.path.basename(out)) + r"\.tmp\.(\d+)$") + directory = os.path.dirname(out) + try: + siblings = os.listdir(directory) + except OSError: + return + for sib in siblings: + m = name.match(sib) + if m and not _pid_alive(int(m.group(1))): + try: + os.remove(os.path.join(directory, sib)) + except OSError: + pass + + def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None): """Run the suite under the plugin. Returns (returncode, payload_or_None). @@ -47,9 +84,18 @@ def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None): — the exact honeyslate case — would load the PREVIOUS run's trace and report it as fresh.""" repo = os.path.abspath(repo) + if pa.not_a_file_target(out): + # This runner deletes `out` first and reads it back after, so a stream + # can be neither: deleting a FIFO removes it from under its reader, + # and /dev/stdout cannot be unlinked at all (#154 review F5). The rule + # is the adapter's, imported, so it cannot drift into a third copy + # (R2-4, R2-8). + raise ValueError(f"--out {out} is not a regular file path; this " + f"runner reads the trace back, so it needs one") out = os.path.abspath(out) if os.path.exists(out): os.remove(out) + _sweep_stale_temps(out) env = dict(env if env is not None else os.environ) env["TGTRACE_OUT"] = out env["TGTRACE_ROOT"] = os.path.join(repo, root) @@ -69,8 +115,16 @@ def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None): return None, None payload = None if os.path.exists(out): - with open(out) as f: - payload = json.load(f) + # tgtrace writes atomically, but an OS crash or a hand-edited file can + # still leave something unreadable. That is the no-trace case the + # caller already handles, not a traceback (#154). + try: + with open(out) as f: + payload = json.load(f) + except (OSError, ValueError): + payload = None + if not isinstance(payload, dict) or pa.payload_shape_error(payload): + payload = None return rc, payload @@ -85,16 +139,21 @@ def main(argv=None): ap.add_argument("pytest_args", nargs="*", help="extra args passed to pytest") args = ap.parse_args(argv) - rc, payload = run( - args.repo, args.python, args.tests, args.root, args.out, - extra_args=args.pytest_args, timeout=args.timeout, - ) + try: + rc, payload = run( + args.repo, args.python, args.tests, args.root, args.out, + extra_args=args.pytest_args, timeout=args.timeout, + ) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 2 if payload is None: why = ( f"timed out after {args.timeout}s" if rc is None - else f"exited {rc} before the plugin could report " - f"(a collection error, or the plugin was not loaded)" + else f"exited {rc} with no readable trace " + f"(a collection error, the plugin was not loaded, or the file is " + f"truncated or not a tgtrace object)" ) print(f"no trace written to {args.out} — pytest {why}", file=sys.stderr) return 1 @@ -112,7 +171,7 @@ def main(argv=None): ) return 1 print(f"traced {len(tests)} test(s), {len(symbols)} distinct symbol(s)") - print(f" backend: {payload['backend']} root: {payload['root']}") + print(f" backend: {payload.get('backend', '?')} root: {payload.get('root', '?')}") print(f" wrote {args.out}") empty = [t for t, s in tests.items() if not s] if empty: diff --git a/testgraph/pytest_adapter.py b/testgraph/pytest_adapter.py index 2e1d020..771d39e 100644 --- a/testgraph/pytest_adapter.py +++ b/testgraph/pytest_adapter.py @@ -31,6 +31,7 @@ There is no second adapter to generalize against yet, and a one-adapter abstraction is a guess rather than a design (verification-direction D3). """ +import json import os import subprocess import sys @@ -264,8 +265,198 @@ def plugin_dir(start=None): return None +def load_trace(trace_path, proc): + """(trace, None) for a whole tgtrace payload, else (None, refusal). + + A missing, truncated or non-object trace is a broken run, not a verdict + (#108). One function so verify and this adapter refuse in the same order + and the same words (#154); the substrings "wrote no trace", "truncated or + not JSON" and "not a tgtrace payload" are what callers and tests key on.""" + if not os.path.isfile(trace_path): + # pytest's own stderr usually names the cause (a bad flag, the + # plugin's xdist refusal); the guess below is only a fallback. + tail = (getattr(proc, "stderr", "") or "").strip()[-400:] + return None, ( + f"pytest exited {getattr(proc, 'returncode', '?')} and tgtrace " + f"wrote no trace — the run was killed, or the plugin did not " + f"load; no verdict was recorded" + + (f"\n pytest stderr (tail): {tail}" if tail else "") + ) + try: + with open(trace_path) as fh: + trace = json.load(fh) + except (OSError, ValueError) as exc: + return None, ( + f"the tgtrace trace is truncated or not JSON ({exc}) — the run " + f"was cut short; no verdict was recorded" + ) + if not isinstance(trace, dict): + return None, ( + f"the trace file is not a tgtrace payload (a JSON " + f"{type(trace).__name__}, not an object); no verdict was recorded" + ) + shape = payload_shape_error(trace) + if shape: + return None, (f"the trace file is not a tgtrace payload ({shape}); " + f"no verdict was recorded") + return trace, None + + +def payload_shape_error(trace): + """What is wrong with a parsed trace object's shape, or None. + + tgtrace always writes `tests` as {nodeid: [[relpath, qualname], ...]} and + `declared` as {nodeid: [journey, ...]}. Anything looser passed and then + crashed in `tuple(s)` or a dict lookup, exit 1 rather than a refusal + (#154 review F4, R2-3). `declared` may be absent or null: verify has + always read it as `trace.get("declared") or {}`.""" + tests = trace.get("tests") + if not isinstance(tests, dict): + return "`tests` is missing or not an object" + for nodeid, symbols in tests.items(): + if not isinstance(symbols, list) or any( + not (isinstance(s, list) and len(s) == 2 + and all(isinstance(x, str) for x in s)) + for s in symbols): + return f"`tests[{nodeid!r}]` is not a list of [relpath, qualname] pairs" + declared = trace.get("declared") or {} + if not isinstance(declared, dict): + return "`declared` is not an object" + for nodeid, journeys in declared.items(): + # Each id must be hashable (it is looked up). It need NOT be a string: + # tgtrace writes whatever `covers(...)` was given, and `covers(1)` is a + # bad declaration that verify reports, not a broken trace (R3-3). + if not isinstance(journeys, list) or any( + isinstance(j, (list, dict)) for j in journeys): + return f"`declared[{nodeid!r}]` is not a list of journey ids" + return None + + +def run_traced(repo, plugins, pytest_args, _runner=None, _workdir=None): + """Run pytest once under tgtrace with JUnit XML on; read both back. + + Returns `(proc, trace, verdicts, error)`. `verdicts` is None when pytest + wrote no JUnit XML: whether that refuses the run is the caller's call + (the adapter needs verdicts to attribute anything; verify reads a missing + report as no verdicts and reports the journeys incomplete). + + `_workdir` stands in for the temp dir in tests, so they need not patch + `tempfile.TemporaryDirectory` for every caller in the process (#154).""" + def go(tmp): + trace_path = os.path.join(tmp, "trace.json") + junit_path = os.path.join(tmp, "junit.xml") + env = dict(os.environ) + env["TGTRACE_OUT"] = trace_path + env["TGTRACE_ROOT"] = os.path.abspath(repo) + env["PYTHONPATH"] = os.pathsep.join( + p for p in (plugins, env.get("PYTHONPATH")) if p + ) + cmd = [sys.executable, "-m", "pytest", "-p", "tgtrace", + f"--junitxml={junit_path}", *pytest_args] + runner = _runner or (lambda: subprocess.run( + cmd, cwd=repo, env=env, capture_output=True, text=True)) + proc = runner() + trace, err = load_trace(trace_path, proc) + if err: + return proc, None, None, err + verdicts = None + if os.path.isfile(junit_path): + try: + verdicts = parse_junit(junit_path) + except (ET.ParseError, OSError) as exc: + # Killed while writing the report: a broken run, not a red + # suite. Raising here exited 1, the "tests failed" code (F3). + return proc, None, None, ( + f"pytest's JUnit XML is truncated or not XML ({exc}) — the " + f"run was cut short; no verdict was recorded") + return proc, trace, verdicts, None + + if _workdir is not None: + return go(_workdir) + with tempfile.TemporaryDirectory() as tmp: + return go(tmp) + + +# Device and process paths are streams however they resolve: `/dev/stdout` +# redirected to a file resolves to a REGULAR file. `/dev/shm` is ordinary +# tmpfs storage, not a stream (#154 review R2-1). +_DEVICE_PREFIXES = ("/dev/", "/proc/") +_FILE_UNDER_DEV = ("/dev/shm/",) + + +def not_a_file_target(path): + """True when `path` cannot hold a trace file: a stream or device path. + + Traces are written to `.tmp.` and renamed over `path`, so the + reader sees either nothing or a whole file. A stream has nowhere to put + the temp file and nothing to rename over, and writing one in place was + tried and dropped: a second `open("/dev/stdout", "w")` gets its own file + offset, so pytest's output overwrote the JSON (#154 review R2-2). So + streams are refused up front, everywhere. #154 noted no caller uses one. + + The same rule is in tgtrace (`_not_a_file_target`), which cannot import + this module; tests/test_pytest_adapter.py:WriterParityTests holds the two + copies to identical answers. harness/trace.py imports this one.""" + if path.endswith(os.sep) or os.path.basename(path) in (".", ".."): + return True # names a directory, however it is spelled (R3-6) + p = os.path.abspath(path) + # Resolve the DIRECTORY, not the leaf: `/dev/stdout` itself resolves into + # whatever stdout is redirected to, but a symlinked parent (`d -> /dev`) + # or a POSIX `//dev` must still read as /dev (#154 review R3-4). + p = os.path.join(os.path.realpath(os.path.dirname(p)), os.path.basename(p)) + if p.startswith(_DEVICE_PREFIXES) and not p.startswith(_FILE_UNDER_DEV): + return True + return os.path.exists(p) and not os.path.isfile(p) + + +def trace_out_error(path): + """Why `--trace-out path` cannot be written, or None. + + Checked BEFORE the suite runs (#154 review F9): finding out after a + half-hour run that the directory does not exist throws the run away.""" + if (path.endswith(os.sep) or os.path.basename(path) in (".", "..") + or os.path.isdir(path)): + return f"--trace-out {path} is a directory, not a file" + if not_a_file_target(path): + return (f"--trace-out {path} is not a regular file path; the trace is " + f"written to a temp file and renamed, so streams and devices " + f"(/dev/stdout, pipes) are refused") + parent = os.path.dirname(os.path.abspath(path)) + if not os.path.isdir(parent): + return f"--trace-out directory {parent} does not exist" + if not os.access(parent, os.W_OK): + return f"--trace-out directory {parent} is not writable" + return None + + +def write_json_atomic(path, obj): + """Write `obj` as JSON so `path` is either absent or whole (#154). + + Temp file beside the target, flushed and fsynced, then renamed: a plain + `open(path, "w")` killed mid-dump leaves a truncated file that a later + reader (reconcile) refuses. The temp is named `.tmp.`, as + tgtrace names its own, so harness/trace.py's sweep finds both. Callers + refuse stream targets first (`trace_out_error`).""" + # One spelling for the check and the write (R3-6): `gone/../t.json` + # passes a check on abspath but fails as a raw temp path. + path = os.path.abspath(path) + tmp = f"{path}.tmp.{os.getpid()}" + try: + with open(tmp, "w") as f: + json.dump(obj, f, indent=1, sort_keys=True) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + except BaseException: + try: + os.remove(tmp) + except OSError: + pass + raise + + def run(repo, registry_path=None, db_path=None, pytest_args=(), commit=None, - append=None, trace_out=None, _runner=None): + append=None, trace_out=None, _runner=None, _workdir=None): """Run the suite, attribute journeys from the trace, write the rows. Returns (summary, error). `error` is a string on any refusal; a suite whose @@ -295,6 +486,11 @@ def run(repo, registry_path=None, db_path=None, pytest_args=(), commit=None, if not os.path.isfile(db_path): return None, f"no codegraph index at {db_path} — run `codegraph init` in {repo}" + if trace_out: + bad = trace_out_error(trace_out) + if bad: + return None, bad + sha = ledger.resolve_commit(repo, commit or "HEAD") if sha is None: return None, ( @@ -302,47 +498,19 @@ def run(repo, registry_path=None, db_path=None, pytest_args=(), commit=None, f"would sit in `unasked` forever" ) - with tempfile.TemporaryDirectory() as tmp: - trace_path = os.path.join(tmp, "trace.json") - junit_path = os.path.join(tmp, "junit.xml") - env = dict(os.environ) - env["TGTRACE_OUT"] = trace_path - env["TGTRACE_ROOT"] = os.path.abspath(repo) - env["PYTHONPATH"] = os.pathsep.join( - p for p in (plugins, env.get("PYTHONPATH")) if p - ) - cmd = [ - sys.executable, "-m", "pytest", "-p", "tgtrace", - f"--junitxml={junit_path}", *pytest_args, - ] - runner = _runner or ( - lambda: subprocess.run(cmd, cwd=repo, env=env, capture_output=True, text=True) - ) - proc = runner() - - if not os.path.isfile(junit_path): - tail = (getattr(proc, "stderr", "") or "")[-400:] - return None, f"pytest wrote no JUnit XML (exit {getattr(proc, 'returncode', '?')}): {tail}" - if not os.path.isfile(trace_path): - return None, ( - "pytest ran but tgtrace wrote nothing — the plugin was not " - "loaded (check that -p tgtrace resolved on PYTHONPATH)" - ) - - import json - try: - with open(trace_path) as f: - trace = json.load(f) - except (OSError, ValueError) as exc: - # Same refusal as verify (#108): a cut-short run is not a verdict. - return None, f"the tgtrace trace is truncated or not JSON ({exc})" - if not isinstance(trace, dict): - return None, (f"the trace file is not a tgtrace payload (a JSON " - f"{type(trace).__name__}, not an object)") - if trace_out: - with open(trace_out, "w") as f: - json.dump(trace, f, indent=1, sort_keys=True) - verdicts = parse_junit(junit_path) + proc, trace, verdicts, err = run_traced(repo, plugins, pytest_args, + _runner=_runner, _workdir=_workdir) + if err: + return None, err + if verdicts is None: + # Attribution joins trace to verdict; with no report there is nothing + # to join, and every journey would read as covered-without-verdict. + tail = (getattr(proc, "stderr", "") or "").strip()[-400:] + return None, (f"pytest exited {getattr(proc, 'returncode', '?')} and wrote " + f"no JUnit XML; no verdict was recorded" + + (f"\n pytest stderr (tail): {tail}" if tail else "")) + if trace_out: + write_json_atomic(trace_out, trace) conn = dbmod.connect(db_path) try: @@ -478,7 +646,11 @@ def main(argv=None): append=(lambda row: False) if args.dry_run else None, ) if err: - print(f"testgraph pytest-adapter: {err}", file=sys.stderr) + if args.json: + # A JSON consumer must be able to parse the refusal too (#154). + print(_json.dumps({"refused": True, "error": err}, indent=2, sort_keys=True)) + else: + print(f"testgraph pytest-adapter: {err}", file=sys.stderr) return 2 summary["dry_run"] = bool(args.dry_run) diff --git a/testgraph/registry.py b/testgraph/registry.py index 7bf6031..9b4c921 100644 --- a/testgraph/registry.py +++ b/testgraph/registry.py @@ -12,8 +12,14 @@ def load(path): + """Parse a registry file. Raises OSError or ValueError, including for JSON + that is not an object: a list used to load fine and crash later, in + whichever caller first indexed it (#154 review R3-5).""" with open(path) as f: - return json.load(f) + data = json.load(f) + if not isinstance(data, dict): + raise ValueError(f"the registry is a JSON {type(data).__name__}, not an object") + return data # Where a registry lives, in search order. See `search_dirs`. diff --git a/testgraph/verify.py b/testgraph/verify.py index d81189d..958b00f 100644 --- a/testgraph/verify.py +++ b/testgraph/verify.py @@ -34,9 +34,7 @@ that rather than a green. """ import os -import subprocess import sys -import tempfile from . import db as dbmod from . import ledger @@ -158,7 +156,7 @@ def _summary(repo, head, selection, **extra): def run(repo, base, head="HEAD", registry_path=None, db_path=None, - pytest_args=(), append=None, _runner=None): + pytest_args=(), append=None, _runner=None, _workdir=None): """Select, run the declared tests, validate the declarations, report. Returns (summary, error). A red suite is not an error — it is the signal.""" @@ -166,6 +164,13 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, if registry_path is None: return None, (f"no journey registry for {reg.repo_name(repo)} — looked " f"in {reg.where_it_looked(repo)}") + try: + registry = reg.load(registry_path) + except (OSError, ValueError) as exc: + # Raised out of here before: a traceback and exit 1, the code for + # "tests failed", and prose under --json (#154 review F9). + return None, f"cannot read registry {registry_path}: {exc}" + db_path = db_path or os.path.join(repo, ".codegraph", "codegraph.db") if not os.path.isfile(db_path): return None, f"no codegraph index at {db_path} — run `codegraph init` in {repo}" @@ -189,7 +194,6 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, # the plain-string form is what a caller passing a hand-built selection uses. journeys = [j["id"] if isinstance(j, dict) else j for j in selection.get("journeys", [])] - registry = reg.load(registry_path) conn = dbmod.connect(db_path) try: @@ -197,52 +201,14 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, return _summary(repo, head, selection), None expr = marker_expression(journeys) - with tempfile.TemporaryDirectory() as tmp: - trace_path = os.path.join(tmp, "trace.json") - junit_path = os.path.join(tmp, "junit.xml") - env = dict(os.environ) - env["TGTRACE_OUT"] = trace_path - env["TGTRACE_ROOT"] = os.path.abspath(repo) - env["PYTHONPATH"] = os.pathsep.join( - p for p in (plugins, env.get("PYTHONPATH")) if p - ) - cmd = [sys.executable, "-m", "pytest", "-p", "tgtrace", - "-m", expr, f"--junitxml={junit_path}", *pytest_args] - runner = _runner or (lambda: subprocess.run( - cmd, cwd=repo, env=env, capture_output=True, text=True)) - proc = runner() - - import json - # A missing or unreadable trace is a broken run, not a verdict - # (#108). Read as `{}` it reported every selected journey as having - # no journey-level test (exit 3); a truncated one raised out of here - # and exited 1, the code for "tests failed". - if not os.path.isfile(trace_path): - # pytest's own stderr usually names the cause (a bad flag, the - # plugin's xdist refusal); the guess below is only a fallback. - tail = (getattr(proc, "stderr", "") or "").strip()[-400:] - return None, ( - f"pytest exited {getattr(proc, 'returncode', '?')} and tgtrace " - f"wrote no trace — the run was killed, or the plugin did not " - f"load; no verdict was recorded" - + (f"\n pytest stderr (tail): {tail}" if tail else "") - ) - try: - with open(trace_path) as fh: - trace = json.load(fh) - except (OSError, ValueError) as exc: - return None, ( - f"the tgtrace trace is truncated or not JSON ({exc}) — the run " - f"was cut short; no verdict was recorded" - ) - if not isinstance(trace, dict): - return None, ( - f"the trace file is not a tgtrace payload (a JSON " - f"{type(trace).__name__}, not an object); no verdict was recorded" - ) - verdicts = {} - if os.path.isfile(junit_path): - verdicts = pa.parse_junit(junit_path) + proc, trace, verdicts, err = pa.run_traced( + repo, plugins, ("-m", expr, *pytest_args), + _runner=_runner, _workdir=_workdir) + if err: + return None, err + # No JUnit XML reads as no verdicts: every selected journey then lands + # in `incomplete` or `uncovered`, never in a pass. + verdicts = verdicts or {} credited, unvalidated, per_journey = validate(conn, registry, trace, verdicts) uncovered = [j for j in journeys if j not in per_journey] @@ -430,7 +396,11 @@ def main(argv=None): append=(lambda row: False) if args.dry_run else None, ) if err: - print(err) + if args.json: + # A JSON consumer must be able to parse the refusal too (#154). + print(_json.dumps({"refused": True, "error": err}, indent=1, sort_keys=True)) + else: + print(err) return EXIT_REFUSED if args.json: print(_json.dumps(summary, indent=1, sort_keys=True, default=str)) diff --git a/tests/test_ground_truth.py b/tests/test_ground_truth.py index e2b8779..22b1815 100644 --- a/tests/test_ground_truth.py +++ b/tests/test_ground_truth.py @@ -11,6 +11,7 @@ import sys import tempfile import unittest +import unittest.mock ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if ROOT not in sys.path: @@ -328,5 +329,122 @@ def test_a_worker_thread_is_recorded_by_the_setprofile_backend(self): self.assertIsNone(sys.getprofile()) + +class _TraceRunnerFixture: + """#154: harness/trace.py read the trace with an unguarded `json.load`, + and never removed the `.tmp.` a SIGKILLed tgtrace leaves.""" + + def _fake_python(self, tmp, body): + # Stands in for the target's interpreter: ignores pytest's argv and + # writes `body` (or nothing) to TGTRACE_OUT. + path = os.path.join(tmp, "fakepy") + with open(path, "w") as fh: + fh.write("#!/bin/sh\n") + if body is not None: + fh.write(f"printf '%s' '{body}' > \"$TGTRACE_OUT\"\n") + os.chmod(path, 0o755) + return path + + def _run(self, body): + from harness import trace as htrace + tmp = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, tmp, ignore_errors=True) + out = os.path.join(tmp, "traces", "t.json") + os.makedirs(os.path.dirname(out)) + stale = out + ".tmp.4242" + open(stale, "w").close() + rc, payload = htrace.run(tmp, self._fake_python(tmp, body), ".", ".", out) + return rc, payload, stale + + +class TraceRunnerTest(_TraceRunnerFixture, unittest.TestCase): + def test_a_truncated_trace_is_the_no_trace_case(self): + rc, payload, _ = self._run('{"tests": {') + self.assertEqual(rc, 0) + self.assertIsNone(payload) + + def test_a_non_object_trace_is_the_no_trace_case(self): + rc, payload, _ = self._run("[]") + self.assertIsNone(payload) + + def test_a_stale_temp_file_is_removed_before_the_run(self): + rc, payload, stale = self._run('{"tests": {}}') + self.assertEqual(payload, {"tests": {}}) + self.assertFalse(os.path.exists(stale)) + + + +class TraceRunnerReviewTest(_TraceRunnerFixture, unittest.TestCase): + """Round-1 review of #154 (.review ledger F4, F5, F6).""" + + def test_f4_an_object_without_tgtrace_fields_is_the_no_trace_case(self): + for body in ("{}", '{"tests": 3}', '{"tests": {"t": 3}}'): + with self.subTest(body): + rc, payload, _ = self._run(body) + self.assertIsNone(payload) + + def test_f4_a_minimal_tgtrace_object_is_kept(self): + rc, payload, _ = self._run('{"tests": {"t": [["a", "b"]]}}') + self.assertEqual(payload, {"tests": {"t": [["a", "b"]]}}) + + def test_f4_main_survives_a_trace_without_backend(self): + from harness import trace as htrace + import contextlib, io + with unittest.mock.patch.object( + htrace, "run", return_value=(0, {"tests": {"t": [["a", "b"]]}})), \ + contextlib.redirect_stdout(io.StringIO()): + rc = htrace.main(["--repo", ".", "--python", "x", "--out", "o.json"]) + self.assertEqual(rc, 0) + + def test_f5_a_non_regular_out_is_refused_and_left_alone(self): + from harness import trace as htrace + tmp = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, tmp, ignore_errors=True) + fifo = os.path.join(tmp, "pipe") + os.mkfifo(fifo) + import stat + with self.assertRaises(ValueError): + htrace.run(tmp, "/bin/false", ".", ".", fifo) + self.assertTrue(stat.S_ISFIFO(os.stat(fifo).st_mode)) + # judged after abspath, so a relative or dotted path into /dev is + # refused too (R2-4) + cwd = os.getcwd() + self.addCleanup(os.chdir, cwd) + for out in ("/tmp/../dev/stdout",): + with self.assertRaises(ValueError): + htrace.run(tmp, "/bin/false", ".", ".", out) + os.chdir("/dev") + with self.assertRaises(ValueError): + htrace.run(tmp, "/bin/false", ".", ".", "stdout") + + @unittest.skipUnless(os.path.isdir("/dev/shm") and os.access("/dev/shm", os.W_OK), + "no writable /dev/shm") + def test_r2_1_an_out_under_dev_shm_is_a_file(self): + from harness import trace as htrace + tmp = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, tmp, ignore_errors=True) + shm = tempfile.mkdtemp(dir="/dev/shm") + self.addCleanup(__import__("shutil").rmtree, shm, ignore_errors=True) + out = os.path.join(shm, "t.json") + rc, payload = htrace.run(tmp, self._fake_python(tmp, '{"tests": {}}'), + ".", ".", out) + self.assertEqual(payload, {"tests": {}}) + + def test_f6_the_sweep_takes_only_dead_pids(self): + from harness import trace as htrace + tmp = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, tmp, ignore_errors=True) + out = os.path.join(tmp, "a.json") + live = f"{out}.tmp.{os.getpid()}" + keep = [live, out + ".tmp.json", out + ".tmp.12x"] + dead = out + ".tmp.999999999" + for f in keep + [dead]: + open(f, "w").close() + htrace.run(tmp, self._fake_python(tmp, '{"tests": {}}'), ".", ".", out) + for f in keep: + self.assertTrue(os.path.exists(f), f) + self.assertFalse(os.path.exists(dead)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_pytest_adapter.py b/tests/test_pytest_adapter.py index 4297aef..cf5de89 100644 --- a/tests/test_pytest_adapter.py +++ b/tests/test_pytest_adapter.py @@ -7,16 +7,27 @@ one outcome this project exists to prevent, so each of those paths is asserted to be REPORTED rather than swallowed. """ +import contextlib +import importlib.util +import io +import json import os +import shutil import sqlite3 import sys +import tempfile +import types import unittest +import unittest.mock ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT_DIR) from testgraph.results import covers # noqa: E402 from testgraph import ledger # noqa: E402 from testgraph import pytest_adapter as pa # noqa: E402 +from test_core import _db_on_disk, _git_repo, _registry_file, build_fixture # noqa: E402 + +HAVE_PYTEST = importlib.util.find_spec("pytest") is not None def build_conn(): @@ -402,5 +413,362 @@ def test_incomplete_join_is_flagged_in_both_directions(self): self.assertIn("coverage may be overstated", out) + +class _AdapterRunFixture: + """#108 / #154: a broken run is refused, never attributed. Each case + writes what a real pytest would have left in the work dir; `_workdir` + stands in for the temp dir, so nothing patches the stdlib.""" + + WHOLE = '{"tests": {}, "declared": {}}' + JUNIT = "" + + def setUp(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + self.tmp = tmp + self.db = _db_on_disk(tmp, build_fixture()) + self.registry = _registry_file(tmp, { + "J1": {"name": "ok", "entries": [{"name": "handler_a", "file": "app/svc.py"}]}, + }) + self.repo, _ = _git_repo(tmp, {"app/svc.py": 22}) + self.work = os.path.join(tmp, "work") + os.makedirs(self.work) + + def _run(self, trace=None, junit=None, stderr="", **kw): + work = self.work + + def runner(): + for name, text in (("trace.json", trace), ("junit.xml", junit)): + if text is not None: + with open(os.path.join(work, name), "w") as fh: + fh.write(text) + return types.SimpleNamespace(returncode=0, stderr=stderr) + + return pa.run(self.repo, registry_path=self.registry, db_path=self.db, + _runner=runner, _workdir=work, append=lambda r: True, **kw) + + +class RunTraceRefusalTests(_AdapterRunFixture, unittest.TestCase): + def test_refusals(self): + cases = [ + ("truncated", dict(trace='{"tests": {', junit=JUNIT_OK), "truncated or not JSON"), + ("not an object", dict(trace="[]", junit=JUNIT_OK), "not a tgtrace payload"), + ("no trace", dict(junit=JUNIT_OK, stderr="boom: xdist"), "wrote no trace"), + ("no junit", dict(trace=self.WHOLE), "wrote no JUnit XML"), + ] + for label, kw, expect in cases: + with self.subTest(label): + for f in os.listdir(self.work): + os.remove(os.path.join(self.work, f)) + summary, err = self._run(**kw) + self.assertIsNone(summary) + self.assertIn(expect, err) + + def test_the_trace_check_comes_before_the_junit_check(self): + """#154: verify and the adapter refused in different orders. With + both files missing, both now name the trace first.""" + summary, err = self._run(stderr="ERROR: unrecognized arguments: --bogus") + self.assertIn("wrote no trace", err) + self.assertIn("--bogus", err) + + def test_a_whole_run_attributes(self): + summary, err = self._run(trace=self.WHOLE, junit=JUNIT_OK) + self.assertIsNone(err) + self.assertEqual(summary["rows_written"], 0) + + def test_trace_out_is_written_whole_and_leaves_no_temp_file(self): + out_dir = os.path.join(self.tmp, "out") + os.makedirs(out_dir) + out = os.path.join(out_dir, "t.json") + summary, err = self._run(trace=self.WHOLE, junit=JUNIT_OK, trace_out=out) + self.assertIsNone(err) + with open(out) as fh: + self.assertEqual(json.load(fh), json.loads(self.WHOLE)) + self.assertEqual(os.listdir(out_dir), ["t.json"]) + + +JUNIT_OK = _AdapterRunFixture.JUNIT + + +class WriteJsonAtomicTests(unittest.TestCase): + """#154: `--trace-out` was copied with a plain open-and-dump.""" + + def test_a_dump_that_dies_leaves_neither_target_nor_temp(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + out = os.path.join(tmp, "t.json") + # json.dump streams chunks, so the temp file is part-written when the + # unserializable value is reached: the real mid-dump death, without + # patching the json module for the whole process. + with self.assertRaises(TypeError): + pa.write_json_atomic(out, {"a": 1, "z": object()}) + self.assertEqual(os.listdir(tmp), []) + + def test_an_existing_target_survives_a_failed_write(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + out = os.path.join(tmp, "t.json") + pa.write_json_atomic(out, {"old": True}) + with self.assertRaises(TypeError): + pa.write_json_atomic(out, {"z": object()}) + with open(out) as fh: + self.assertEqual(json.load(fh), {"old": True}) + + +class MainJsonRefusalTests(unittest.TestCase): + def test_a_json_refusal_is_json(self): + """#154: the refusal went to stderr as prose even under `--json`.""" + out = io.StringIO() + with unittest.mock.patch.object(pa, "run", return_value=(None, "trace broke")), \ + contextlib.redirect_stdout(out): + rc = pa.main(["--repo", "/x", "--json"]) + self.assertEqual(rc, 2) + self.assertEqual(json.loads(out.getvalue()), {"refused": True, "error": "trace broke"}) + + + +class ReviewRound1Tests(_AdapterRunFixture, unittest.TestCase): + """Round-1 review of #154 (.review ledger F1, F3, F4, F8, F9).""" + + def test_f3_a_truncated_junit_report_is_refused_not_a_crash(self): + for text in ("", ""): + with self.subTest(repr(text)): + for f in os.listdir(self.work): + os.remove(os.path.join(self.work, f)) + summary, err = self._run(trace=self.WHOLE, junit=text) + self.assertIsNone(summary) + self.assertIn("JUnit XML is truncated or not XML", err) + + def test_f4_an_object_without_tgtrace_fields_is_refused(self): + for text in ("{}", '{"tests": 3}', '{"tests": {}, "declared": 3}', + '{"tests": {"t": 3}}', + # element level (R2-3): each entry a [relpath, qualname] + # pair, each declared journey a string + '{"tests": {"t": [1]}}', '{"tests": {"t": [["x"]]}}', + '{"tests": {"t": [["a", 1]]}}', + '{"tests": {}, "declared": {"t": [["J1"]]}}'): + with self.subTest(text): + for f in os.listdir(self.work): + os.remove(os.path.join(self.work, f)) + summary, err = self._run(trace=text, junit=JUNIT_OK) + self.assertIsNone(summary) + self.assertIn("not a tgtrace payload", err) + + def test_f9_a_trace_out_that_cannot_be_written_is_refused_before_the_run(self): + ran = [] + fifo = os.path.join(self.tmp, "pipe") + os.mkfifo(fifo) + bad = [os.path.join(self.tmp, "no", "such", "t.json"), self.tmp, + os.path.join(self.tmp, "newdir") + os.sep, # R2-5 + fifo, "/dev/stdout", "/dev/stdot", # R2-1/R2-2/R2-5 + "/tmp/../dev/stdout"] + for out in bad: + with self.subTest(out): + def runner(): + ran.append(out) + return types.SimpleNamespace(returncode=0, stderr="") + summary, err = pa.run(self.repo, registry_path=self.registry, + db_path=self.db, _runner=runner, + _workdir=self.work, append=lambda r: True, + trace_out=out) + self.assertIsNone(summary) + self.assertIn("--trace-out", err) + self.assertEqual(ran, [], "the suite ran before --trace-out was checked") + + def test_r3_3_a_non_string_declaration_is_not_a_broken_trace(self): + """`covers(1)` is a bad decorator, not a broken run: tgtrace writes + what `tg_journeys` holds, so an int must not refuse every verdict.""" + summary, err = self._run(trace='{"tests": {}, "declared": {"t": [1]}}', + junit=JUNIT_OK) + self.assertIsNone(err) + + def test_r3_7_a_directory_trace_out_says_directory(self): + summary, err = self._run(trace=self.WHOLE, junit=JUNIT_OK, trace_out=self.tmp) + self.assertIsNone(summary) + self.assertIn("is a directory", err) + + def test_r3_6_a_trace_out_naming_a_directory_is_refused_before_the_run(self): + ran = [] + + def runner(): + ran.append(1) + return types.SimpleNamespace(returncode=0, stderr="") + for out in (os.path.join(self.tmp, "x", "new.json") + "/.", + os.path.join(self.tmp, "nosuchdir") + "/."): + with self.subTest(out): + summary, err = pa.run(self.repo, registry_path=self.registry, + db_path=self.db, _runner=runner, + _workdir=self.work, append=lambda r: True, + trace_out=out) + self.assertIsNone(summary) + self.assertIn("--trace-out", err) + self.assertEqual(ran, []) + + def test_r3_6_a_dotted_trace_out_is_written_where_the_check_looked(self): + os.makedirs(os.path.join(self.tmp, "real")) + out = os.path.join(self.tmp, "gone", "..", "real", "t.json") + summary, err = self._run(trace=self.WHOLE, junit=JUNIT_OK, trace_out=out) + self.assertIsNone(err) + self.assertTrue(os.path.isfile(os.path.join(self.tmp, "real", "t.json"))) + + def test_r3_5_a_registry_that_is_not_an_object_is_refused_before_the_run(self): + bad = os.path.join(self.tmp, "list.json") + with open(bad, "w") as fh: + fh.write("[]") + ran = [] + + def runner(): + ran.append(1) + return types.SimpleNamespace(returncode=0, stderr="") + summary, err = pa.run(self.repo, registry_path=bad, db_path=self.db, + _runner=runner, _workdir=self.work, append=lambda r: True) + self.assertIsNone(summary) + self.assertIn("not an object", err) + self.assertEqual(ran, []) + + def test_declared_null_is_accepted_as_empty(self): + """verify has always read `declared` as `.get("declared") or {}`; + the shape check must not turn that into a refusal (R2-3 parity).""" + summary, err = self._run(trace='{"tests": {}, "declared": null}', junit=JUNIT_OK) + self.assertIsNone(err) + + @unittest.skipUnless(os.path.isdir("/dev/shm") and os.access("/dev/shm", os.W_OK), + "no writable /dev/shm") + def test_r2_1_a_trace_out_under_dev_shm_is_a_file_written_atomically(self): + out_dir = tempfile.mkdtemp(dir="/dev/shm") + self.addCleanup(shutil.rmtree, out_dir, ignore_errors=True) + out = os.path.join(out_dir, "t.json") + seen = [] + real_replace = os.replace + + def spy(src, dst): + seen.append(os.path.basename(src)) + return real_replace(src, dst) + with unittest.mock.patch("os.replace", spy): + summary, err = self._run(trace=self.WHOLE, junit=JUNIT_OK, trace_out=out) + self.assertIsNone(err) + self.assertEqual(seen, [f"t.json.tmp.{os.getpid()}"]) + with open(out) as fh: + self.assertEqual(json.load(fh), json.loads(self.WHOLE)) + + +class WriterParityTests(unittest.TestCase): + """F8/R2: testgraph's target rule and atomic writer and tgtrace's are two + copies (tgtrace must stay standalone, so it cannot import the other). This + holds them to the same behaviour rather than a comment saying they match.""" + + def _writers(self): + sys.path.insert(0, os.path.join(ROOT_DIR, "harness", "plugin")) + try: + import tgtrace + finally: + sys.path.pop(0) + return {"adapter": pa.write_json_atomic, "tgtrace": tgtrace._write} + + def _tgtrace_rule(self): + sys.path.insert(0, os.path.join(ROOT_DIR, "harness", "plugin")) + try: + import tgtrace + finally: + sys.path.pop(0) + return tgtrace._not_a_file_target + + def test_both_copies_of_the_target_rule_agree(self): + """One rule, two copies (tgtrace cannot import testgraph): same + answer on every case, and the answer is the intended one.""" + tg_rule = self._tgtrace_rule() + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + regular = os.path.join(tmp, "t.json") + open(regular, "w").close() + fifo = os.path.join(tmp, "pipe") + os.mkfifo(fifo) + cases = { + "/dev/stdout": True, "/dev/fd/1": True, "/proc/self/fd/1": True, + "/dev/stdot": True, "/dev/null": True, "/tmp/../dev/stdout": True, + fifo: True, tmp: True, + "/dev/shm/x.json": False, regular: False, + os.path.join(tmp, "absent.json"): False, + # a path that names a directory, however spelled (R3-6) + os.path.join(tmp, "new.json") + "/.": True, + os.path.join(tmp, "new.json") + "/": True, + os.path.join(tmp, "a", ".."): True, + # a dotted path to a real file location is still a file + os.path.join(tmp, "a", "..", "b.json"): False, + } + for path, expected in cases.items(): + with self.subTest(path): + self.assertEqual(pa.not_a_file_target(path), expected) + self.assertEqual(tg_rule(path), expected) + # relative paths are judged after abspath (R2-4) + cwd = os.getcwd() + self.addCleanup(os.chdir, cwd) + os.chdir("/dev") + self.assertTrue(pa.not_a_file_target("stdout")) + self.assertTrue(tg_rule("stdout")) + + def test_relative_and_dotted_dev_paths_with_stdout_redirected_to_a_file(self): + """R2-4: with stdout redirected to a file, `/dev/stdout` resolves to a + regular file, so only the abspath + prefix test catches `stdout` from + cwd /dev or `/tmp/../dev/stdout`. Run in a child whose stdout IS a file.""" + import subprocess + code = ( + "import sys; sys.path[:0] = [sys.argv[1], sys.argv[1] + '/harness/plugin']\n" + "from testgraph import pytest_adapter as pa; import tgtrace\n" + "for p in ('stdout', '/tmp/../dev/stdout', '//dev/stdout', sys.argv[2] + '/stdout'):\n" + " sys.stderr.write(f'{pa.not_a_file_target(p)} {tgtrace._not_a_file_target(p)}\\n')\n" + ) + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + link = os.path.join(tmp, "devlnk") + os.symlink("/dev", link) # a symlinked parent (R3-4) + with open(os.path.join(tmp, "out.txt"), "w") as fh: + proc = subprocess.run([sys.executable, "-c", code, ROOT_DIR, link], cwd="/dev", + stdout=fh, stderr=subprocess.PIPE, text=True) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stderr.split(), ["True"] * 8) + + @unittest.skipUnless(HAVE_PYTEST, "pytest is not installed") # R3-2 + def test_tgtrace_refuses_a_stream_before_any_test_runs(self): + """R2-2: writing a stream in place corrupted it (a second open of + /dev/stdout has its own offset). Real pytest, stdout redirected to a + file: the run is refused at configure, and says why.""" + import subprocess + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + with open(os.path.join(tmp, "test_x.py"), "w") as fh: + fh.write("def test_x():\n assert True\n") + env = dict(os.environ, TGTRACE_OUT="/dev/stdout", + PYTHONPATH=os.path.join(ROOT_DIR, "harness", "plugin")) + out = os.path.join(tmp, "out.txt") + with open(out, "w") as fh: + proc = subprocess.run([sys.executable, "-m", "pytest", "-p", "tgtrace", + "-q", "-p", "no:cacheprovider", tmp], + cwd=tmp, env=env, stdout=fh, stderr=subprocess.STDOUT) + with open(out) as fh: + text = fh.read() + self.assertNotEqual(proc.returncode, 0) + self.assertIn("not a regular file path", text) + self.assertNotIn("1 passed", text) + + def test_both_name_their_temp_file_the_same_way(self): + """F8: the harness sweep matches `.tmp.`; a killed adapter + used to leave `..tmp.`, which nothing swept.""" + for name, write in self._writers().items(): + with self.subTest(name): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + out = os.path.join(tmp, "t.json") + seen = [] + real_replace = os.replace + + def spy(src, dst): + seen.append(os.path.basename(src)) + return real_replace(src, dst) + with unittest.mock.patch("os.replace", spy): + write(out, {"a": 1}) + self.assertEqual(seen, [f"t.json.tmp.{os.getpid()}"]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_verify.py b/tests/test_verify.py index da95f61..a458fcb 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -352,24 +352,15 @@ def __getattr__(self, n): work = os.path.join(tmp, "work") os.makedirs(work) - class FixedDir: - def __enter__(self): - return work - - def __exit__(self, *exc): - return False - def runner(): with open(os.path.join(work, "trace.json"), "w") as fh: fh.write('{"tests": {}, "declared": {}}') return types.SimpleNamespace(returncode=0) with unittest.mock.patch.object( - verify.dbmod, "connect", lambda p: Proxy(real(p))), \ - unittest.mock.patch.object(verify.tempfile, "TemporaryDirectory", - lambda: FixedDir()): + verify.dbmod, "connect", lambda p: Proxy(real(p))): summary, err = verify.run(repo, "HEAD~1", "HEAD", registry_path=registry, - db_path=db, _runner=runner, + db_path=db, _runner=runner, _workdir=work, append=lambda row: True) self.assertIsNone(err) self.assertIsNotNone(summary["marker_expression"]) @@ -378,7 +369,7 @@ def runner(): self.assertEqual(len(closed), len(opened)) -class UnreadableTraceTests(unittest.TestCase): +class _VerifyRunFixture: """Issue #108: a truncated trace made `json.load` raise, and the uncaught exception exited 1 — the code for "tests failed". A missing trace read as an empty one, so every selected journey was reported as having no @@ -406,24 +397,18 @@ def setUp(self): def _run(self, trace_text): work = self.work - class FixedDir: - def __enter__(self): - return work - - def __exit__(self, *exc): - return False - def runner(): if trace_text is not None: with open(os.path.join(work, "trace.json"), "w") as fh: fh.write(trace_text) return types.SimpleNamespace(returncode=0) - with unittest.mock.patch.object(verify.tempfile, "TemporaryDirectory", - lambda: FixedDir()): - return verify.run(self.repo, "HEAD~1", "HEAD", registry_path=self.registry, - db_path=self.db, _runner=runner, append=lambda r: True) + return verify.run(self.repo, "HEAD~1", "HEAD", registry_path=self.registry, + db_path=self.db, _runner=runner, _workdir=work, + append=lambda r: True) + +class UnreadableTraceTests(_VerifyRunFixture, unittest.TestCase): def test_a_truncated_trace_is_refused_not_a_crash(self): summary, err = self._run('{"tests": {"t::a": [["app/svc.py", "hand') self.assertIsNone(summary) @@ -442,22 +427,12 @@ def test_a_trace_that_is_not_an_object_is_refused(self): def test_the_refusal_carries_pytest_stderr(self): """Review H3: the real cause (an xdist refusal, a bad flag) was in pytest's stderr, which was dropped.""" - work = self.work - - class FixedDir: - def __enter__(self): - return work - - def __exit__(self, *exc): - return False - runner = lambda: types.SimpleNamespace( returncode=4, stderr="ERROR: usage: unrecognized arguments: --bogus") - with unittest.mock.patch.object(verify.tempfile, "TemporaryDirectory", - lambda: FixedDir()): - summary, err = verify.run(self.repo, "HEAD~1", "HEAD", - registry_path=self.registry, db_path=self.db, - _runner=runner, append=lambda r: True) + summary, err = verify.run(self.repo, "HEAD~1", "HEAD", + registry_path=self.registry, db_path=self.db, + _runner=runner, _workdir=self.work, + append=lambda r: True) self.assertIn("unrecognized arguments: --bogus", err) def test_a_missing_trace_is_refused_not_uncovered(self): @@ -471,6 +446,17 @@ def test_main_exits_refused(self): rc = verify.main(["--repo", self.repo, "--base", "HEAD~1"]) self.assertEqual(rc, verify.EXIT_REFUSED) + def test_a_json_refusal_is_json(self): + """#154: `--json` printed the refusal as prose, so a JSON consumer + could not parse the reason.""" + out = io.StringIO() + with unittest.mock.patch.object(verify, "run", return_value=(None, "trace broke")), \ + contextlib.redirect_stdout(out): + rc = verify.main(["--repo", self.repo, "--base", "HEAD~1", "--json"]) + self.assertEqual(rc, verify.EXIT_REFUSED) + self.assertEqual(json.loads(out.getvalue()), + {"refused": True, "error": "trace broke"}) + class AtomicTraceWriteTests(unittest.TestCase): """#108: the trace was written in place, so a run killed mid-write left a @@ -481,30 +467,72 @@ def test_a_write_that_dies_leaves_no_partial_file(self): tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) out = os.path.join(tmp, "trace.json") - - seen_target = [] - - def dying_dump(obj, fh, **kw): - fh.write('{"tests": {') - # a SIGKILL here runs no cleanup: the target must not exist yet - seen_target.append(os.path.exists(out)) - raise KeyboardInterrupt - - with unittest.mock.patch.object(tgtrace, "_out_path", out), \ - unittest.mock.patch.object(tgtrace, "_stop", lambda: None), \ - unittest.mock.patch.object(tgtrace.json, "dump", dying_dump): - with self.assertRaises(KeyboardInterrupt): - tgtrace.pytest_unconfigure(None) + # json.dump streams, so the temp file is part-written when it reaches + # the unserializable value: a real mid-dump death, with no patch on + # the json module the rest of the process shares (#154). + with self.assertRaises(TypeError): + tgtrace._write(out, {"tests": {}, "z": object()}) self.assertFalse(os.path.exists(out)) self.assertEqual(os.listdir(tmp), []) - self.assertEqual(seen_target, [False]) + + def test_a_stream_target_is_refused_at_configure(self): + """R2-2: a stream was written in place, and a second open of + /dev/stdout has its own offset, so pytest's output overwrote the + JSON. Now refused before any test runs; a FIFO is left alone.""" + from harness.plugin import tgtrace + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + fifo = os.path.join(tmp, "pipe") + os.mkfifo(fifo) + config = types.SimpleNamespace(option=types.SimpleNamespace(numprocesses=None)) + for target in (fifo, "/dev/stdout"): + with self.subTest(target), \ + unittest.mock.patch.object(tgtrace, "_out_path", target), \ + unittest.mock.patch.object(tgtrace, "_start", lambda: None), \ + unittest.mock.patch.dict(os.environ, {}, clear=False): + os.environ.pop("PYTEST_XDIST_WORKER", None) + with self.assertRaises(RuntimeError) as cm: + tgtrace.pytest_configure(config) + self.assertIn("not a regular file path", str(cm.exception)) + self.assertEqual(sorted(os.listdir(tmp)), ["pipe"]) + + def test_r3_1_unconfigure_after_a_refusal_touches_nothing(self): + """pytest calls pytest_unconfigure even when pytest_configure raised. + It used to run _stop + _write anyway: on the setprofile backend (3.11) + that replaced the refused FIFO with a regular file; on sys.monitoring + it raised `tool 3 is not in use`, exit 1. Both backends, both + refusals (stream target, xdist).""" + from harness.plugin import tgtrace + import stat + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + fifo = os.path.join(tmp, "pipe") + os.mkfifo(fifo) + plain = os.path.join(tmp, "t.json") + for backend in (True, False): + for target, xdist in ((fifo, None), (plain, 2)): + with self.subTest(monitoring=backend, target=target), \ + unittest.mock.patch.object(tgtrace, "_HAVE_MONITORING", backend), \ + unittest.mock.patch.object(tgtrace, "_out_path", target), \ + unittest.mock.patch.object(tgtrace, "_started", False): + os.environ.pop("PYTEST_XDIST_WORKER", None) + config = types.SimpleNamespace( + option=types.SimpleNamespace(numprocesses=xdist)) + with self.assertRaises(RuntimeError): + tgtrace.pytest_configure(config) + tgtrace.pytest_unconfigure(config) # must not raise or write + self.assertTrue(stat.S_ISFIFO(os.stat(fifo).st_mode)) + self.assertFalse(os.path.exists(plain)) def test_a_whole_write_lands(self): from harness.plugin import tgtrace tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) out = os.path.join(tmp, "trace.json") + # `_started`: unconfigure writes only after configure started tracing + # (R3-1); this simulates a run that did. with unittest.mock.patch.object(tgtrace, "_out_path", out), \ + unittest.mock.patch.object(tgtrace, "_started", True), \ unittest.mock.patch.object(tgtrace, "_stop", lambda: None): tgtrace.pytest_unconfigure(None) with open(out) as fh: @@ -512,48 +540,6 @@ def test_a_whole_write_lands(self): self.assertEqual(os.listdir(tmp), ["trace.json"]) -class AdapterTruncatedTraceTests(unittest.TestCase): - def test_the_adapter_refuses_a_truncated_trace(self): - from testgraph import pytest_adapter as pa - tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) - db = _db_on_disk(tmp, build_fixture()) - registry = _registry_file(tmp, { - "J1": {"name": "ok", "entries": [{"name": "handler_a", "file": "app/svc.py"}]}, - }) - repo, _ = _git_repo(tmp, {"app/svc.py": 22}) - work = os.path.join(tmp, "work") - os.makedirs(work) - - class FixedDir: - def __enter__(self): - return work - - def __exit__(self, *exc): - return False - - def runner(): - with open(os.path.join(work, "junit.xml"), "w") as fh: - fh.write("") - with open(os.path.join(work, "trace.json"), "w") as fh: - fh.write(self.trace_text) - return types.SimpleNamespace(returncode=0, stderr="") - - with unittest.mock.patch.object(pa.tempfile, "TemporaryDirectory", - lambda: FixedDir()): - summary, err = pa.run(repo, registry_path=registry, db_path=db, - _runner=runner, append=lambda r: True) - self.assertIsNone(summary) - self.assertIn(self.expect, err) - - trace_text, expect = '{"tests": {', "truncated or not JSON" - - -class AdapterNonObjectTraceTests(AdapterTruncatedTraceTests): - """Review H1, adapter side: `[]` crashed at `trace.get("tests")`.""" - trace_text, expect = "[]", "not a tgtrace payload" - - class SummaryShapeTests(unittest.TestCase): """Every exit from `run` carries the same keys. @@ -687,5 +673,52 @@ class C: self.assertEqual(res.declared_journeys(C), ("J1",)) + +class ReviewRound1VerifyTests(_VerifyRunFixture, unittest.TestCase): + """Round-1 review of #154 (.review ledger C1, F3, F9).""" + + def test_c1_a_missing_junit_report_is_no_verdicts_not_a_pass(self): + summary, err = self._run('{"tests": {}, "declared": {}}') + self.assertIsNone(err) + self.assertNotEqual(verify.exit_code(summary), verify.EXIT_OK) + self.assertEqual(summary["collected"], 0) + + def test_f3_a_truncated_junit_report_refuses_verify(self): + work = self.work + + def runner(): + with open(os.path.join(work, "trace.json"), "w") as fh: + fh.write('{"tests": {}, "declared": {}}') + with open(os.path.join(work, "junit.xml"), "w") as fh: + fh.write("") + return types.SimpleNamespace(returncode=0, stderr="") + + summary, err = verify.run(self.repo, "HEAD~1", "HEAD", + registry_path=self.registry, db_path=self.db, + _runner=runner, _workdir=work, append=lambda r: True) + self.assertIsNone(summary) + self.assertIn("JUnit XML is truncated or not XML", err) + + def test_r2_6_a_registry_that_is_not_an_object_is_refused(self): + bad = os.path.join(self.tmp, "list.json") + with open(bad, "w") as fh: + fh.write("[]") + summary, err = verify.run(self.repo, "HEAD~1", "HEAD", registry_path=bad, + db_path=self.db, append=lambda r: True) + self.assertIsNone(summary) + self.assertIn("not an object", err) + + def test_f9_an_unreadable_registry_is_a_json_refusal(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + rc = verify.main(["--repo", self.repo, "--base", "HEAD~1", + "--registry", os.path.join(self.tmp, "nope.json"), + "--db", self.db, "--json"]) + self.assertEqual(rc, verify.EXIT_REFUSED) + body = json.loads(out.getvalue()) + self.assertTrue(body["refused"]) + self.assertIn("nope.json", body["error"]) + + if __name__ == "__main__": unittest.main()