Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,30 @@ below is on `main` or in the branch that introduced this file.

### Fixed

- **A refusal under `--json` is JSON** (#154). `verify --json` and
`pytest_adapter --json` printed a refusal as prose, so a JSON consumer could not
parse the reason. They now print `{"refused": true, "error": "..."}`, still exit 2.
- **`verify` and the pytest adapter refuse a broken run in the same order and
words** (#154). Both now run pytest through one helper: a missing trace (with
pytest's stderr tail), then a truncated one, then one that is not an object. The
adapter's missing-JUnit refusal now carries the stderr tail too.
- **Trace files survive crashes** (#154). tgtrace flushes and fsyncs before its
atomic rename, so an OS crash can no longer leave a zero-length trace;
`pytest_adapter --trace-out` is written the same way instead of with a plain
open-and-dump.
- **Stream targets are refused up front** (#154). `TGTRACE_OUT`, `--trace-out` and
`harness/trace.py --out` must name a file: `/dev/stdout`, pipes and other device
paths are refused before any test runs, with a message saying why. Writing a
stream in place was tried and dropped, because a second open of `/dev/stdout`
has its own file offset and pytest's output overwrote the JSON. `/dev/shm`
counts as a file location.
- **More broken runs are refused instead of crashing** (#154). A truncated JUnit
report, a trace whose `tests`/`declared` entries are not tgtrace's shape, and a
registry that is unreadable or not a JSON object now refuse with exit 2; they
used to raise and exit 1, the code for "tests failed". `--trace-out` is checked
before the suite runs, not after.
- **`harness/trace.py`** reads an unreadable or foreign trace as no trace instead of
raising, and removes only the `<out>.tmp.<pid>` files left by dead processes.
- **`verify` works from an installed wheel.** The wheel shipped `testgraph/` only, so
the README's headline `verify` command failed after `pip install testgraph` with
"cannot find harness/plugin/tgtrace.py". The wheel now carries the trace plugin as
Expand Down
68 changes: 61 additions & 7 deletions harness/plugin/tgtrace.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
)

_current = None # set() while a test body is running, else None
_started = False # tracing began; unconfigure does nothing otherwise
_results = {}
_declared = {} # nodeid -> [journey id, ...] the test CLAIMS to cover

Expand Down Expand Up @@ -141,7 +142,9 @@ def _stop_profile():


def _start():
global _started
_start_monitoring() if _HAVE_MONITORING else _start_profile()
_started = True


def _stop():
Expand All @@ -167,30 +170,81 @@ def pytest_configure(config):
"overwrite one another's trace and the result would silently be "
"empty. Re-run without -n/--numprocesses."
)
# Refused here, before any test runs, rather than failing at unconfigure
# after the whole suite (#154 review R2-2).
if _not_a_file_target(_out_path):
raise RuntimeError(
f"TGTRACE_OUT={_out_path} is not a regular file path: the trace is "
f"written to a temp file and renamed, so streams and devices "
f"(/dev/stdout, pipes) are refused. Point it at a file."
)
_start()


def pytest_unconfigure(config):
if not _out_path:
# pytest calls this even when pytest_configure RAISED (an xdist or
# stream-target refusal). Stopping a tracer that never started raised on
# sys.monitoring, and writing the empty payload replaced the refused
# target on setprofile (#154 review R3-1).
global _started
if not _out_path or not _started:
return
_started = False
_stop()
payload = {
"root": _root,
"backend": "sys.monitoring" if _HAVE_MONITORING else "sys.setprofile",
"tests": {tid: sorted(map(list, syms)) for tid, syms in _results.items()},
"declared": {tid: list(j) for tid, j in sorted(_declared.items())},
}
directory = os.path.dirname(os.path.abspath(_out_path))
_write(_out_path, payload)


# Device and process paths are streams however they resolve: `/dev/stdout`
# redirected to a file resolves to a REGULAR file. `/dev/shm` is ordinary
# tmpfs storage, not a stream (#154 review R2-1).
_DEVICE_PREFIXES = ("/dev/", "/proc/")
_FILE_UNDER_DEV = ("/dev/shm/",)


def _not_a_file_target(path):
"""True when `path` cannot hold a trace file: a stream or device path.

A copy of `testgraph.pytest_adapter.not_a_file_target`, because this
plugin imports nothing from testgraph; WriterParityTests in
tests/test_pytest_adapter.py holds the two to identical answers."""
if path.endswith(os.sep) or os.path.basename(path) in (".", ".."):
return True # names a directory, however it is spelled (R3-6)
p = os.path.abspath(path)
# Resolve the DIRECTORY, not the leaf: `/dev/stdout` itself resolves into
# whatever stdout is redirected to, but a symlinked parent (`d -> /dev`)
# or a POSIX `//dev` must still read as /dev (#154 review R3-4).
p = os.path.join(os.path.realpath(os.path.dirname(p)), os.path.basename(p))
if p.startswith(_DEVICE_PREFIXES) and not p.startswith(_FILE_UNDER_DEV):
return True
return os.path.exists(p) and not os.path.isfile(p)


def _write(out_path, payload):
"""Write the payload so `out_path` is either absent or whole.

Beside the target, then rename (#108): a run killed mid-dump used to leave
a truncated file where a reader expects a whole one. os.replace is atomic
on the same filesystem; the flush + fsync before it (#154) keeps an OS
crash from leaving a zero-length target, which ext4/xfs can otherwise do
when the rename reaches disk before the data. Stream targets were refused
in `pytest_configure`."""
out_path = os.path.abspath(out_path) # one spelling for check and write (R3-6)
directory = os.path.dirname(out_path)
if directory:
os.makedirs(directory, exist_ok=True)
# Write beside the target and rename (#108): a run killed mid-dump used to
# leave a truncated file where a reader expects a whole one. os.replace is
# atomic on the same filesystem, so the target is either absent or complete.
tmp = f"{_out_path}.tmp.{os.getpid()}"
tmp = f"{out_path}.tmp.{os.getpid()}"
try:
with open(tmp, "w") as f:
json.dump(payload, f, indent=1, sort_keys=True)
os.replace(tmp, _out_path)
f.flush()
os.fsync(f.fileno())
os.replace(tmp, out_path)
except BaseException:
try:
os.remove(tmp)
Expand Down
77 changes: 68 additions & 9 deletions harness/trace.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,54 @@
import argparse
import json
import os
import re
import subprocess
import sys


HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
if ROOT not in sys.path:
sys.path.insert(0, ROOT)
from testgraph import pytest_adapter as pa # noqa: E402
# ONLY this directory goes on the target's PYTHONPATH. `harness/` itself holds
# `trace.py`, which would shadow the stdlib `trace` module for every import in
# the traced suite.
PLUGIN_DIR = os.path.join(HERE, "plugin")


def _pid_alive(pid):
try:
os.kill(pid, 0)
except (ProcessLookupError, OverflowError):
return False
except PermissionError:
return True # exists, owned by someone else
return True


def _sweep_stale_temps(out):
"""Remove `<out>.tmp.<pid>` files whose writer is dead.

A run SIGKILLed mid-dump leaves tgtrace's temp behind (#154) and nothing
else removes it. Only `.tmp.<digits>` names, and only dead pids: a broader
glob took unrelated files (`a.json.tmp.json`) and a CONCURRENT run's
in-progress temp, whose rename then failed (#154 review F6)."""
name = re.compile(re.escape(os.path.basename(out)) + r"\.tmp\.(\d+)$")
directory = os.path.dirname(out)
try:
siblings = os.listdir(directory)
except OSError:
return
for sib in siblings:
m = name.match(sib)
if m and not _pid_alive(int(m.group(1))):
try:
os.remove(os.path.join(directory, sib))
except OSError:
pass


def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None):
"""Run the suite under the plugin. Returns (returncode, payload_or_None).

Expand All @@ -47,9 +84,18 @@ def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None):
— the exact honeyslate case — would load the PREVIOUS run's trace and report
it as fresh."""
repo = os.path.abspath(repo)
if pa.not_a_file_target(out):
# This runner deletes `out` first and reads it back after, so a stream
# can be neither: deleting a FIFO removes it from under its reader,
# and /dev/stdout cannot be unlinked at all (#154 review F5). The rule
# is the adapter's, imported, so it cannot drift into a third copy
# (R2-4, R2-8).
raise ValueError(f"--out {out} is not a regular file path; this "
f"runner reads the trace back, so it needs one")
out = os.path.abspath(out)
if os.path.exists(out):
os.remove(out)
_sweep_stale_temps(out)
env = dict(env if env is not None else os.environ)
env["TGTRACE_OUT"] = out
env["TGTRACE_ROOT"] = os.path.join(repo, root)
Expand All @@ -69,8 +115,16 @@ def run(repo, python, tests, root, out, extra_args=(), timeout=1800, env=None):
return None, None
payload = None
if os.path.exists(out):
with open(out) as f:
payload = json.load(f)
# tgtrace writes atomically, but an OS crash or a hand-edited file can
# still leave something unreadable. That is the no-trace case the
# caller already handles, not a traceback (#154).
try:
with open(out) as f:
payload = json.load(f)
except (OSError, ValueError):
payload = None
if not isinstance(payload, dict) or pa.payload_shape_error(payload):
payload = None
return rc, payload


Expand All @@ -85,16 +139,21 @@ def main(argv=None):
ap.add_argument("pytest_args", nargs="*", help="extra args passed to pytest")
args = ap.parse_args(argv)

rc, payload = run(
args.repo, args.python, args.tests, args.root, args.out,
extra_args=args.pytest_args, timeout=args.timeout,
)
try:
rc, payload = run(
args.repo, args.python, args.tests, args.root, args.out,
extra_args=args.pytest_args, timeout=args.timeout,
)
except ValueError as exc:
print(str(exc), file=sys.stderr)
return 2
if payload is None:
why = (
f"timed out after {args.timeout}s"
if rc is None
else f"exited {rc} before the plugin could report "
f"(a collection error, or the plugin was not loaded)"
else f"exited {rc} with no readable trace "
f"(a collection error, the plugin was not loaded, or the file is "
f"truncated or not a tgtrace object)"
)
print(f"no trace written to {args.out} — pytest {why}", file=sys.stderr)
return 1
Expand All @@ -112,7 +171,7 @@ def main(argv=None):
)
return 1
print(f"traced {len(tests)} test(s), {len(symbols)} distinct symbol(s)")
print(f" backend: {payload['backend']} root: {payload['root']}")
print(f" backend: {payload.get('backend', '?')} root: {payload.get('root', '?')}")
print(f" wrote {args.out}")
empty = [t for t, s in tests.items() if not s]
if empty:
Expand Down
Loading
Loading