From c9f5462fbeb78b01ea7b1b9a4af682ae3a54155f Mon Sep 17 00:00:00 2001 From: Eric Minish Date: Sun, 15 Nov 2026 16:43:43 -0500 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20Vitest=20adapter=20=E2=80=94=20per-?= =?UTF-8?q?test=20V8=20trace,=20same=20journey=20attribution=20(#122)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit testgraph.vitest_adapter runs a Vitest suite once and attributes journeys the way pytest_adapter does. harness/plugin/tgtrace-vitest.ts (TypeScript, shipped beside tgtrace.py) takes V8 precise coverage per test and writes one atomic part file per test file; Vitest's JSON reporter gives verdicts; attribute / journey_entries / write_outcomes are reused unchanged. The setup file is injected through a generated config in a temp dir that imports the repo's own config and mergeConfigs it in, with a node_modules symlink so vitest resolves as the repo's tests see it. Nothing is written into the target repo. Refusals (no vitest install, no or truncated report, a broken part, tests run but no trace) exit 2. pytest_adapter.render labels the runner, so both adapters share it. --- CHANGELOG.md | 7 + USAGE.md | 19 ++ harness/plugin/tgtrace-vitest.ts | 118 ++++++++++++ pyproject.toml | 3 +- testgraph/pytest_adapter.py | 8 +- testgraph/vitest_adapter.py | 305 +++++++++++++++++++++++++++++++ tests/test_plugin_packaging.py | 12 +- tests/test_vitest_adapter.py | 265 +++++++++++++++++++++++++++ 8 files changed, 732 insertions(+), 5 deletions(-) create mode 100644 harness/plugin/tgtrace-vitest.ts create mode 100644 testgraph/vitest_adapter.py create mode 100644 tests/test_vitest_adapter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3adbd34..41103f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,13 @@ below is on `main` or in the branch that introduced this file. ### Added +- **`testgraph.vitest_adapter`** (#122): the pytest adapter's journey attribution for + Vitest suites. A TypeScript setup file (`tgtrace-vitest.ts`, shipped beside + `tgtrace.py`) records each test's named functions through V8 precise coverage; + Vitest's JSON reporter supplies verdicts; the join, the attribution and the ledger + rows are the pytest adapter's own. Injected through a generated config in a temp + dir, so nothing is written into the target repo. On a real Next.js repo (167 + Vitest tests) it attributes 5 journeys with no unjoined test. - **`testgraph.reconcile`** — two-sided reconciliation of the static graph against a real test run, classifying every disagreement as a **STATIC MISS** (a symbol the journey's tests executed, outside its static footprint) or a diff --git a/USAGE.md b/USAGE.md index 1b4512c..c476308 100644 --- a/USAGE.md +++ b/USAGE.md @@ -176,6 +176,25 @@ tests, and nothing but that line would say so. The trace plugin it loads ships in the wheel (`testgraph/_plugin/tgtrace.py`), so this works from `pip install testgraph` as well as from a checkout. +### Vitest suites + +The same attribution for a TypeScript repo whose tests run on Vitest (4.x): + +```bash +python3 -m testgraph.vitest_adapter --repo . --dry-run +python3 -m testgraph.vitest_adapter --repo . --json -- src/lib # vitest's own args after -- +``` + +It needs the repo's dependencies installed (`node_modules` with `vitest`, at the repo or +a workspace root). Nothing is written into the repo: a generated config in a temp dir +imports the repo's own `vitest.config.*` (or `vite.config.*`) and adds the +`tgtrace-vitest.ts` setup file, which records each test's named functions through V8 +precise coverage. Rows carry `edge_provenance=trace`, like the pytest adapter's. + +Limits: anonymous functions are not recorded (attribution matches named entry +symbols, so this costs nothing there); tests marked `.concurrent` in one file share a +coverage window; `verify` does not run Vitest suites yet. + ## Checking the Graph Against Reality `reconcile` puts CodeGraph's edges on trial, using the same run: diff --git a/harness/plugin/tgtrace-vitest.ts b/harness/plugin/tgtrace-vitest.ts new file mode 100644 index 0000000..9ac8562 --- /dev/null +++ b/harness/plugin/tgtrace-vitest.ts @@ -0,0 +1,118 @@ +/** + * Vitest setup file: record every NAMED function that actually ran, per test. + * + * The Vitest counterpart of `tgtrace.py` (#122). It is loaded into the TARGET + * repo's test workers through a generated config (see + * `testgraph/vitest_adapter.py`), so it imports nothing from testgraph and only + * `vitest` plus Node built-ins. + * + * How: V8 precise coverage through `node:inspector`. Each `takePreciseCoverage` + * returns call counts since the previous take and resets them, so a take in + * `beforeEach` clears the slate and a take in `afterEach` is exactly what the + * test body (plus its hooks) ran. Function granularity (`detailed: false`) is + * enough for attribution, which matches journey ENTRY symbols by name. + * + * Configured by environment, like tgtrace.py: + * TGTRACE_OUT_DIR directory for one part file per test file (required, else inert) + * TGTRACE_ROOT only record functions defined under this directory + * TGTRACE_SKIP colon-separated path fragments to ignore + * (default: /node_modules/ .test. .spec. /__tests__/) + * + * Part file: {"root", "backend", "file", "tests": {"": [["", ""], ...]}} + * where is `:: > ... > `, the same id the + * adapter builds from Vitest's JSON reporter. + * + * Known limits: anonymous functions are dropped (nothing to resolve them by + * without source maps); tests marked `.concurrent` in one file share one + * coverage window, so their traces can bleed into each other. + */ +import { Session } from "node:inspector/promises"; +import { closeSync, fsyncSync, mkdirSync, openSync, renameSync, rmSync, writeSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { join, relative, sep } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, afterEach, beforeEach } from "vitest"; + +const OUT_DIR = process.env.TGTRACE_OUT_DIR; +const ROOT = process.env.TGTRACE_ROOT ?? process.cwd(); +const SKIP = (process.env.TGTRACE_SKIP ?? "/node_modules/:.test.:.spec.:/__tests__/") + .split(":") + .filter(Boolean); + +type Hit = [string, string]; +type Task = { name: string; type?: string; suite?: Task; file?: { filepath?: string; name: string } }; + +function testId(task: Task): string { + const filepath = task.file?.filepath ?? task.file?.name ?? ""; + const names: string[] = [task.name]; + for (let s = task.suite; s && s.type === "suite" && s.name !== task.file?.name; s = s.suite) { + if (s.name) names.unshift(s.name); + } + return `${relative(ROOT, filepath).split(sep).join("/")}::${names.join(" > ")}`; +} + +function scriptPath(url: string): string | null { + if (!url.startsWith("file://")) return null; + try { + return fileURLToPath(url.split("?")[0]); + } catch { + return null; + } +} + +if (OUT_DIR) { + const session = new Session(); + session.connect(); + await session.post("Profiler.enable"); + await session.post("Profiler.startPreciseCoverage", { callCount: true, detailed: false }); + + const tests: Record<string, Hit[]> = {}; + let file = ""; + + beforeEach(async () => { + await session.post("Profiler.takePreciseCoverage"); // reset: drop setup-time calls + }); + + afterEach(async (ctx) => { + const { result } = await session.post("Profiler.takePreciseCoverage"); + const task = ctx.task as unknown as Task; + file = task.file?.filepath ?? file; + const hits = new Map<string, Hit>(); + for (const script of result) { + const abs = scriptPath(script.url); + if (!abs || !abs.startsWith(ROOT + sep)) continue; + const rel = relative(ROOT, abs).split(sep).join("/"); + if (SKIP.some((frag) => ("/" + rel).includes(frag))) continue; + for (const fn of script.functions) { + const count = fn.ranges[0]?.count ?? 0; + if (count > 0 && fn.functionName) hits.set(`${rel}\0${fn.functionName}`, [rel, fn.functionName]); + } + } + tests[testId(task)] = [...hits.values()].sort(); + }); + + afterAll(async () => { + await session.post("Profiler.stopPreciseCoverage"); + session.disconnect(); + mkdirSync(OUT_DIR, { recursive: true }); + const name = createHash("sha1").update(file).digest("hex").slice(0, 16); + const target = join(OUT_DIR, `${name}.json`); + const tmp = `${target}.tmp.${process.pid}`; + const payload = { root: ROOT, backend: "v8-precise-coverage", file: relative(ROOT, file), tests }; + // Whole or absent, as tgtrace.py writes: a worker killed mid-write must + // not leave a truncated part for the adapter to merge. + try { + const fd = openSync(tmp, "w"); + try { + writeSync(fd, JSON.stringify(payload, null, 1)); + fsyncSync(fd); + } finally { + closeSync(fd); + } + renameSync(tmp, target); + } catch (err) { + rmSync(tmp, { force: true }); + throw err; + } + }); +} diff --git a/pyproject.toml b/pyproject.toml index 13728e4..29220c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ packages = ["testgraph"] # must list it too: `uv build` builds the wheel FROM the sdist. [tool.hatch.build.targets.wheel.force-include] "harness/plugin/tgtrace.py" = "testgraph/_plugin/tgtrace.py" +"harness/plugin/tgtrace-vitest.ts" = "testgraph/_plugin/tgtrace-vitest.ts" [tool.hatch.build.targets.sdist] -include = ["testgraph", "journeys", "skills", "hooks", "harness/plugin/tgtrace.py", "README.md", "USAGE.md", "TECHNICAL.md", "LICENSE"] +include = ["testgraph", "journeys", "skills", "hooks", "harness/plugin/tgtrace.py", "harness/plugin/tgtrace-vitest.ts", "README.md", "USAGE.md", "TECHNICAL.md", "LICENSE"] diff --git a/testgraph/pytest_adapter.py b/testgraph/pytest_adapter.py index 771d39e..aedf061 100644 --- a/testgraph/pytest_adapter.py +++ b/testgraph/pytest_adapter.py @@ -542,9 +542,13 @@ def render(summary, registry=None): gap is the actionable half — measured on testgraph itself, J6 (the harness CLIs) has zero covering tests in a 359-test suite, and nothing but this line would say so.""" + # One renderer for every runner's adapter (vitest_adapter sets `runner` + # and `runner_exit`); pytest summaries keep their `pytest_exit` key. + runner = summary.get("runner", "pytest") + rc = summary.get("runner_exit", summary.get("pytest_exit")) lines = [ - f"testgraph pytest-adapter[{summary['repo']}] @ {summary['commit'][:9]} " - f"(pytest exit {summary['pytest_exit']}, {summary['backend']})" + f"testgraph {runner}-adapter[{summary['repo']}] @ {summary['commit'][:9]} " + f"({runner} exit {rc}, {summary['backend']})" ] covered = summary["journeys"] names = {} diff --git a/testgraph/vitest_adapter.py b/testgraph/vitest_adapter.py new file mode 100644 index 0000000..cf491ac --- /dev/null +++ b/testgraph/vitest_adapter.py @@ -0,0 +1,305 @@ +"""Run a Vitest suite once and record which journeys its tests ACTUALLY +exercised, from a per-test runtime trace (#122). + +The Vitest counterpart of `pytest_adapter`: same attribution (a test covers a +journey when it RAN one of the journey's entry symbols), same ledger rows, +same `edge_provenance="trace"`. Only the two halves of the run differ: + + - **What ran**: `harness/plugin/tgtrace-vitest.ts`, a setup file using V8 + precise coverage, writes one part file per test file: + `{"tests": {"<relfile>::<describe> > <title>": [[relpath, functionName], ...]}}`. + - **Verdicts**: Vitest's JSON reporter, keyed by the same id. + +Nothing is written into the target repo. The setup file is injected through a +generated config in a temp dir that imports the repo's own config and +`mergeConfig`s the setup file in; Vitest has no `--setupFiles` CLI flag. The +temp dir gets a `node_modules` symlink to the repo's, so the generated config +and the setup file resolve `vitest` exactly as the repo's tests do. + +Deliberately not here yet: selecting only some journeys' tests (verify's job; +next step), and anonymous functions (they need source maps, which only +`reconcile` would use). See `~/.claude/plans/testgraph-154-then-vitest-adapter.md`. +""" +import glob +import json +import os +import shutil +import subprocess +import sys +import tempfile + +from . import db as dbmod +from . import ledger +from . import pytest_adapter as pa +from . import registry as reg + +PLUGIN_NAME = "tgtrace-vitest.ts" +BACKEND = "v8-precise-coverage" +CONFIG_NAMES = tuple( + f"{stem}.config.{ext}" + for stem in ("vitest", "vite") + for ext in ("ts", "mts", "cts", "js", "mjs", "cjs") +) +# Vitest JSON reporter status -> the run-result status `results` defines. +STATUS = {"passed": "pass", "failed": "fail", "skipped": "skip", + "pending": "skip", "todo": "skip"} + + +def find_node_modules(repo): + """The nearest `node_modules` at or above `repo` that holds vitest, or None. + + Walked up because a workspace hoists dependencies to its root.""" + d = os.path.abspath(repo) + while True: + nm = os.path.join(d, "node_modules") + if os.path.isfile(os.path.join(nm, "vitest", "package.json")): + return nm + parent = os.path.dirname(d) + if parent == d: + return None + d = parent + + +def find_config(repo): + """The repo's own Vitest (else Vite) config file, or None.""" + for name in CONFIG_NAMES: + path = os.path.join(repo, name) + if os.path.isfile(path): + return path + return None + + +def config_source(repo, base_config, setup_file): + """The generated config: the repo's config plus our setup file. + + `mergeConfig` concatenates `setupFiles`, so the repo's own setup files + still run, before ours. A config exported as a function is called the way + Vitest calls it for `vitest run`.""" + base = ( + f"import base from {json.dumps(base_config)};\n" + "const resolved = typeof base === \"function\"\n" + " ? await base({ command: \"serve\", mode: \"test\" })\n" + " : base;\n" + ) if base_config else "const resolved = {};\n" + return ( + "import { defineConfig, mergeConfig } from \"vitest/config\";\n" + + base + + "export default mergeConfig(resolved ?? {}, defineConfig({\n" + f" root: {json.dumps(os.path.abspath(repo))},\n" + f" test: {{ setupFiles: [{json.dumps(setup_file)}] }},\n" + "}));\n" + ) + + +def test_id(root, file_path, ancestors, title): + """`<relfile>::<describe> > ... > <title>`, as the setup file builds it.""" + rel = os.path.relpath(file_path, root).replace(os.sep, "/") + return f"{rel}::{' > '.join([*ancestors, title])}" + + +def parse_report(report, root): + """Vitest JSON reporter output -> {join_key: {"status": ...}}. + + Keyed through `pytest_adapter._join_key`, the key `attribute` joins on, so + the generic attribution code needs no Vitest branch.""" + out = {} + for file_result in report.get("testResults", []): + for case in file_result.get("assertionResults", []): + tid = test_id(root, file_result["name"], case.get("ancestorTitles", []), + case.get("title", "")) + out[pa._join_key(tid)] = { + "status": STATUS.get(case.get("status"), "skip"), + "test_id": tid, + } + return out + + +def merge_parts(out_dir): + """Merge the per-test-file part files into one tgtrace-shaped payload. + + Returns (trace, error). A part that is not whole is a broken run, the same + refusal as a truncated tgtrace trace.""" + tests = {} + paths = sorted(glob.glob(os.path.join(out_dir, "*.json"))) + for path in paths: + try: + with open(path) as fh: + part = json.load(fh) + except (OSError, ValueError) as exc: + return None, (f"a tgtrace-vitest part is truncated or not JSON " + f"({exc}); no verdict was recorded") + if not isinstance(part, dict) or pa.payload_shape_error(part): + return None, (f"{os.path.basename(path)} is not a tgtrace-vitest part; " + f"no verdict was recorded") + tests.update(part["tests"]) + trace = {"backend": BACKEND, "tests": tests, "declared": {}} + return trace, None + + +def run(repo, registry_path=None, db_path=None, vitest_args=(), commit=None, + append=None, trace_out=None, _runner=None, _workdir=None): + """Run the suite, attribute journeys from the trace, write the rows. + + Returns (summary, error), as `pytest_adapter.run` does: a red suite is a + result, a run that cannot be read is a refusal.""" + repo = os.path.abspath(repo) + registry_path = registry_path or reg.resolve_for_repo(repo) + if registry_path is None: + return None, (f"no journey registry for {reg.repo_name(repo)} — looked in " + f"{reg.where_it_looked(repo)}") + try: + registry = reg.load(registry_path) + except (OSError, ValueError) as exc: + return None, f"cannot read {registry_path}: {exc}" + + plugins = pa.plugin_dir() + plugin = os.path.join(plugins, PLUGIN_NAME) if plugins else None + if not plugin or not os.path.isfile(plugin): + return None, (f"cannot find {PLUGIN_NAME}: it ships beside tgtrace.py " + f"(testgraph/_plugin/ installed, harness/plugin/ in a checkout)") + + db_path = db_path or os.path.join(repo, ".codegraph", "codegraph.db") + if not os.path.isfile(db_path): + return None, f"no codegraph index at {db_path} — run `codegraph init` in {repo}" + + node_modules = find_node_modules(repo) + if node_modules is None and _runner is None: + return None, (f"no node_modules with vitest at or above {repo} — install " + f"the repo's dependencies first") + + if trace_out: + bad = pa.trace_out_error(trace_out) + if bad: + return None, bad + + sha = ledger.resolve_commit(repo, commit or "HEAD") + if sha is None: + return None, (f"cannot resolve {commit or 'HEAD'!r} in {repo} — an " + f"unjoinable key would sit in `unasked` forever") + + def go(tmp): + parts = os.path.join(tmp, "parts") + os.makedirs(parts, exist_ok=True) + report_path = os.path.join(tmp, "results.json") + setup = os.path.join(tmp, PLUGIN_NAME) + shutil.copyfile(plugin, setup) + config = os.path.join(tmp, "tg.vitest.config.mts") + with open(config, "w") as fh: + fh.write(config_source(repo, find_config(repo), setup)) + if node_modules and not os.path.exists(os.path.join(tmp, "node_modules")): + os.symlink(node_modules, os.path.join(tmp, "node_modules")) + env = dict(os.environ, TGTRACE_OUT_DIR=parts, TGTRACE_ROOT=repo) + cmd = [os.path.join(node_modules or "", ".bin", "vitest"), "run", + "--config", config, "--reporter=json", f"--outputFile={report_path}", + *vitest_args] + runner = _runner or (lambda: subprocess.run( + cmd, cwd=repo, env=env, capture_output=True, text=True)) + proc = runner() + + tail = (getattr(proc, "stderr", "") or "").strip()[-400:] + if not os.path.isfile(report_path): + return proc, None, None, ( + f"vitest exited {getattr(proc, 'returncode', '?')} and wrote no " + f"JSON report; no verdict was recorded" + + (f"\n vitest stderr (tail): {tail}" if tail else "")) + try: + with open(report_path) as fh: + report = json.load(fh) + except (OSError, ValueError) as exc: + return proc, None, None, (f"vitest's JSON report is truncated or not " + f"JSON ({exc}); no verdict was recorded") + if not isinstance(report, dict): + return proc, None, None, "vitest's JSON report is not an object" + verdicts = parse_report(report, repo) + trace, err = merge_parts(parts) + if err: + return proc, None, None, err + if verdicts and not trace["tests"]: + # Tests ran, the setup file recorded nothing: it did not load. + return proc, None, None, ( + "vitest ran tests but tgtrace-vitest wrote no trace — the setup " + "file did not load; no verdict was recorded" + + (f"\n vitest stderr (tail): {tail}" if tail else "")) + return proc, trace, verdicts, None + + if _workdir is not None: + proc, trace, verdicts, err = go(_workdir) + else: + with tempfile.TemporaryDirectory() as tmp: + proc, trace, verdicts, err = go(tmp) + if err: + return None, err + if trace_out: + pa.write_json_atomic(trace_out, trace) + + conn = dbmod.connect(db_path) + try: + entries = pa.journey_entries(conn, registry) + attribution = pa.attribute(conn, trace["tests"], entries, verdicts) + written = pa.write_outcomes(reg.repo_name(repo), sha, attribution, append=append) + summary = dict(attribution) + summary.update({ + "repo": reg.repo_name(repo), + "commit": sha, + "rows_written": len(written), + "runner": "vitest", + "runner_exit": getattr(proc, "returncode", None), + "backend": BACKEND, + "journeys_registered": len(entries), + }) + return summary, None + finally: + conn.close() + + +def main(argv=None): + import argparse + + ap = argparse.ArgumentParser( + prog="testgraph.vitest_adapter", + description="run a Vitest suite under tgtrace-vitest and record which " + "journeys its tests exercised", + epilog="Put vitest's own flags after `--`.\n" + "Exit: 0 recorded, 2 refused (no readable run).", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + ap.add_argument("--repo", required=True) + ap.add_argument("--registry", default=None) + ap.add_argument("--db", default=None) + ap.add_argument("--commit", default=None) + ap.add_argument("--dry-run", action="store_true", + help="attribute and report, but write NO ledger rows") + ap.add_argument("--json", action="store_true") + ap.add_argument("--trace-out", default=None, + help="also write the merged trace here, for testgraph.reconcile") + ap.add_argument("vitest_args", nargs="*") + args = ap.parse_args(argv) + + summary, err = run( + args.repo, registry_path=args.registry, db_path=args.db, commit=args.commit, + vitest_args=tuple(args.vitest_args), trace_out=args.trace_out, + append=(lambda row: False) if args.dry_run else None, + ) + if err: + if args.json: + print(json.dumps({"refused": True, "error": err}, indent=2, sort_keys=True)) + else: + print(f"testgraph vitest-adapter: {err}", file=sys.stderr) + return 2 + summary["dry_run"] = bool(args.dry_run) + if args.json: + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + registry = None + path = args.registry or reg.resolve_for_repo(args.repo) + if path: + try: + registry = reg.load(path) + except (OSError, ValueError): + registry = None + print(pa.render(summary, registry)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_plugin_packaging.py b/tests/test_plugin_packaging.py index ee03c7d..c53e1c1 100644 --- a/tests/test_plugin_packaging.py +++ b/tests/test_plugin_packaging.py @@ -17,8 +17,16 @@ def test_wheel_force_includes_the_plugin_from_its_source_of_truth(self): with open(os.path.join(ROOT, "pyproject.toml"), "rb") as f: conf = tomllib.load(f) inc = conf["tool"]["hatch"]["build"]["targets"]["wheel"]["force-include"] - self.assertEqual(inc, {"harness/plugin/tgtrace.py": "testgraph/_plugin/tgtrace.py"}) - self.assertTrue(os.path.isfile(os.path.join(ROOT, "harness/plugin/tgtrace.py"))) + self.assertEqual(inc, { + "harness/plugin/tgtrace.py": "testgraph/_plugin/tgtrace.py", + # the Vitest setup file (#122) ships beside it; plugin_dir finds both + "harness/plugin/tgtrace-vitest.ts": "testgraph/_plugin/tgtrace-vitest.ts", + }) + for src in inc: + self.assertTrue(os.path.isfile(os.path.join(ROOT, src)), src) + sdist = conf["tool"]["hatch"]["build"]["targets"]["sdist"]["include"] + for src in inc: + self.assertIn(src, sdist) def test_an_installed_package_finds_the_packaged_plugin(self): with tempfile.TemporaryDirectory() as site: diff --git a/tests/test_vitest_adapter.py b/tests/test_vitest_adapter.py new file mode 100644 index 0000000..3aad726 --- /dev/null +++ b/tests/test_vitest_adapter.py @@ -0,0 +1,265 @@ +"""The Vitest adapter (#122): a per-test V8 trace plus Vitest's JSON report, +joined into the same journey attribution the pytest adapter produces. + +Hermetic: no Node. The runner is replaced by a function that writes what a +real run leaves in the work dir (the reporter's JSON and the setup file's +part files). The setup file itself (harness/plugin/tgtrace-vitest.ts) is +exercised against a real Vitest 4 suite outside CI; see the PR for that run. +""" +import contextlib +import io +import json +import os +import shutil +import sys +import tempfile +import types +import unittest +import unittest.mock + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT_DIR) +from testgraph import pytest_adapter as pa # noqa: E402 +from testgraph import vitest_adapter as va # noqa: E402 +from test_core import _db_on_disk, _git_repo, _registry_file, build_fixture # noqa: E402 + + +def _report(repo, cases): + """A Vitest JSON report: cases are (relfile, ancestors, title, status).""" + files = {} + for rel, ancestors, title, status in cases: + files.setdefault(rel, []).append( + {"ancestorTitles": list(ancestors), "title": title, "status": status}) + return {"testResults": [{"name": os.path.join(repo, rel), "assertionResults": a} + for rel, a in files.items()]} + + +class IdAndReportTests(unittest.TestCase): + def test_the_id_carries_the_describe_chain(self): + self.assertEqual( + va.test_id("/r", "/r/src/a.test.ts", ["outer", "inner"], "does x"), + "src/a.test.ts::outer > inner > does x") + self.assertEqual(va.test_id("/r", "/r/a.test.ts", [], "top"), "a.test.ts::top") + + def test_statuses_map_to_run_results(self): + report = _report("/r", [("a.test.ts", [], t, st) for t, st in ( + ("p", "passed"), ("f", "failed"), ("s", "skipped"), ("n", "pending"), + ("t", "todo"), ("weird", "something-new"))]) + got = {v["test_id"].split("::")[1]: v["status"] + for v in va.parse_report(report, "/r").values()} + self.assertEqual(got, {"p": "pass", "f": "fail", "s": "skip", "n": "skip", + "t": "skip", "weird": "skip"}) + + def test_report_keys_are_the_keys_attribute_joins_on(self): + report = _report("/r", [("src/a.test.ts", ["d"], "x", "passed")]) + key, = va.parse_report(report, "/r") + self.assertEqual(key, pa._join_key("src/a.test.ts::d > x")) + + +class DiscoveryTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + + def test_node_modules_is_found_at_a_workspace_root(self): + nm = os.path.join(self.tmp, "node_modules") + os.makedirs(os.path.join(nm, "vitest")) + with open(os.path.join(nm, "vitest", "package.json"), "w") as fh: + fh.write("{}") + pkg = os.path.join(self.tmp, "packages", "app") + os.makedirs(pkg) + self.assertEqual(va.find_node_modules(pkg), nm) + + def test_no_vitest_install_is_none(self): + os.makedirs(os.path.join(self.tmp, "node_modules", "react")) + self.assertIsNone(va.find_node_modules(self.tmp)) + + def test_vitest_config_wins_over_vite_config(self): + for name in ("vite.config.ts", "vitest.config.mts"): + open(os.path.join(self.tmp, name), "w").close() + self.assertEqual(va.find_config(self.tmp), + os.path.join(self.tmp, "vitest.config.mts")) + self.assertIsNone(va.find_config(os.path.join(self.tmp, "..", "nope"))) + + def test_the_generated_config_merges_into_the_repos_own(self): + src = va.config_source("/r", "/r/vitest.config.ts", "/tmp/x/tgtrace-vitest.ts") + self.assertIn('import base from "/r/vitest.config.ts"', src) + self.assertIn('typeof base === "function"', src) + self.assertIn("mergeConfig(resolved", src) + self.assertIn('setupFiles: ["/tmp/x/tgtrace-vitest.ts"]', src) + self.assertIn('root: "/r"', src) + bare = va.config_source("/r", None, "/s.ts") + self.assertNotIn("import base", bare) + self.assertIn("const resolved = {}", bare) + + +class _VitestRunFixture: + def setUp(self): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + self.tmp = tmp + self.db = _db_on_disk(tmp, build_fixture()) + self.registry = _registry_file(tmp, { + "J1": {"name": "ok", "entries": [{"name": "handler_a", "file": "app/svc.py"}]}, + }) + self.repo, _ = _git_repo(tmp, {"app/svc.py": 22}) + self.repo = os.path.abspath(self.repo) + self.work = os.path.join(tmp, "work") + os.makedirs(self.work) + + def _part(self, tests, name="p1.json", text=None): + os.makedirs(os.path.join(self.work, "parts"), exist_ok=True) + with open(os.path.join(self.work, "parts", name), "w") as fh: + fh.write(text if text is not None else json.dumps( + {"root": self.repo, "backend": va.BACKEND, "file": "x", "tests": tests})) + + def _run(self, report=None, parts=(), stderr="", report_text=None, **kw): + work = self.work + + def runner(): + for args in parts: + self._part(*args) if isinstance(args, tuple) else self._part(args) + if report_text is not None or report is not None: + with open(os.path.join(work, "results.json"), "w") as fh: + fh.write(report_text if report_text is not None else json.dumps(report)) + return types.SimpleNamespace(returncode=1 if "failed" in str(report) else 0, + stderr=stderr) + + return va.run(self.repo, registry_path=self.registry, db_path=self.db, + _runner=runner, _workdir=work, append=lambda r: True, **kw) + + +class RunTests(_VitestRunFixture, unittest.TestCase): + TID = "src/a.test.ts::svc > calls handler" + + def test_a_test_that_ran_an_entry_covers_its_journey(self): + report = _report(self.repo, [("src/a.test.ts", ["svc"], "calls handler", "passed"), + ("src/a.test.ts", [], "internal", "passed")]) + summary, err = self._run(report, parts=[{ + self.TID: [["app/svc.py", "handler_a"]], + "src/a.test.ts::internal": [["app/svc.py", "no_such_fn"]], + }]) + self.assertIsNone(err) + self.assertEqual(summary["journeys"], {"J1": "pass"}) + self.assertEqual(summary["covering_tests"], {"J1": 1}) + self.assertEqual(summary["tests_covering_no_journey"], 1) + self.assertEqual(summary["traced_without_verdict"], []) + self.assertEqual(summary["verdicts_without_trace"], []) + self.assertEqual(summary["runner"], "vitest") + self.assertEqual(summary["rows_written"], 1) + + def test_a_failing_covering_test_fails_the_journey(self): + report = _report(self.repo, [("src/a.test.ts", ["svc"], "calls handler", "failed")]) + summary, err = self._run(report, parts=[{self.TID: [["app/svc.py", "handler_a"]]}]) + self.assertIsNone(err) + self.assertEqual(summary["journeys"], {"J1": "fail"}) + self.assertEqual(summary["runner_exit"], 1) + + def test_parts_from_several_test_files_merge(self): + report = _report(self.repo, [("src/a.test.ts", ["svc"], "calls handler", "passed"), + ("src/b.test.ts", [], "also", "passed")]) + summary, err = self._run(report, parts=[ + ({self.TID: [["app/svc.py", "handler_a"]]}, "p1.json"), + ({"src/b.test.ts::also": [["app/svc.py", "handler_a"]]}, "p2.json")]) + self.assertIsNone(err) + self.assertEqual(summary["covering_tests"], {"J1": 2}) + + def test_render_names_the_runner(self): + report = _report(self.repo, [("src/a.test.ts", ["svc"], "calls handler", "passed")]) + summary, _ = self._run(report, parts=[{self.TID: [["app/svc.py", "handler_a"]]}]) + self.assertIn("testgraph vitest-adapter[", pa.render(summary)) + self.assertIn("(vitest exit 0, v8-precise-coverage)", pa.render(summary)) + + def test_trace_out_gets_the_merged_trace(self): + report = _report(self.repo, [("src/a.test.ts", ["svc"], "calls handler", "passed")]) + out = os.path.join(self.tmp, "trace.json") + summary, err = self._run(report, parts=[{self.TID: [["app/svc.py", "handler_a"]]}], + trace_out=out) + self.assertIsNone(err) + with open(out) as fh: + trace = json.load(fh) + self.assertEqual(trace["tests"], {self.TID: [["app/svc.py", "handler_a"]]}) + self.assertEqual(trace["backend"], va.BACKEND) + + +class RefusalTests(_VitestRunFixture, unittest.TestCase): + """A run that cannot be read is refused, never attributed (#108, #154).""" + + def test_no_report_is_refused_with_stderr(self): + summary, err = self._run(None, stderr="Error: Cannot find module 'vitest/config'") + self.assertIsNone(summary) + self.assertIn("wrote no JSON report", err) + self.assertIn("vitest/config", err) + + def test_a_truncated_report_is_refused(self): + summary, err = self._run(report_text='{"testResults": [') + self.assertIn("truncated or not JSON", err) + + def test_a_report_that_is_not_an_object_is_refused(self): + summary, err = self._run(report_text="[]") + self.assertIn("not an object", err) + + def test_tests_with_no_trace_mean_the_setup_file_did_not_load(self): + report = _report(self.repo, [("src/a.test.ts", [], "x", "passed")]) + summary, err = self._run(report, stderr="boom") + self.assertIsNone(summary) + self.assertIn("setup file did not load", err) + + def test_a_truncated_part_is_refused(self): + report = _report(self.repo, [("src/a.test.ts", [], "x", "passed")]) + summary, err = self._run(report, parts=[({}, "p1.json", '{"tests": {')]) + self.assertIn("truncated or not JSON", err) + + def test_a_part_with_the_wrong_shape_is_refused(self): + report = _report(self.repo, [("src/a.test.ts", [], "x", "passed")]) + summary, err = self._run(report, parts=[({"t": [1]}, "p1.json")]) + self.assertIn("not a tgtrace-vitest part", err) + + def test_an_empty_suite_is_not_a_refusal(self): + summary, err = self._run({"testResults": []}) + self.assertIsNone(err) + self.assertEqual(summary["journeys"], {}) + + def test_no_vitest_install_is_refused_before_running(self): + summary, err = va.run(self.repo, registry_path=self.registry, db_path=self.db, + _workdir=self.work) + self.assertIsNone(summary) + self.assertIn("no node_modules with vitest", err) + + def test_a_bad_trace_out_is_refused_before_running(self): + ran = [] + + def runner(): + ran.append(1) + return types.SimpleNamespace(returncode=0, stderr="") + summary, err = va.run(self.repo, registry_path=self.registry, db_path=self.db, + _runner=runner, _workdir=self.work, + trace_out="/dev/stdout") + self.assertIn("--trace-out", err) + self.assertEqual(ran, []) + + def test_main_json_refusal_is_json(self): + out = io.StringIO() + with unittest.mock.patch.object(va, "run", return_value=(None, "broke")), \ + contextlib.redirect_stdout(out): + rc = va.main(["--repo", "/x", "--json"]) + self.assertEqual(rc, 2) + self.assertEqual(json.loads(out.getvalue()), {"refused": True, "error": "broke"}) + + +class PluginShipsTests(unittest.TestCase): + def test_the_setup_file_sits_beside_tgtrace(self): + plugins = pa.plugin_dir() + self.assertTrue(os.path.isfile(os.path.join(plugins, va.PLUGIN_NAME))) + + def test_the_setup_file_imports_nothing_from_testgraph(self): + """It runs in the TARGET repo's workers, which cannot import us.""" + with open(os.path.join(pa.plugin_dir(), va.PLUGIN_NAME)) as fh: + imports = [l for l in fh if l.startswith("import ")] + self.assertTrue(imports) + for line in imports: + self.assertRegex(line, r'from "(node:[a-z/]+|vitest)";') + + +if __name__ == "__main__": + unittest.main() From 6cf5291df28f907687a9ee4254e0ca6f10b04b74 Mon Sep 17 00:00:00 2001 From: Eric Minish <eric.minish@gmail.com> Date: Sun, 15 Nov 2026 17:10:00 -0500 Subject: [PATCH 2/2] fix: Vitest adapter refuses broken or partial runs (review V1-V12) A broken or partial run is a refusal (exit 2), never a verdict. V1 a passed/failed test with no trace refuses (count + example id); a reporter "pending" test (killed worker) refuses instead of reading as skip V2 trace recorded in onTestFinished (from beforeEach), which still fires when the repo's afterEach throws; V1 backstops it V3 duplicate test ids refuse (report side and setup-file side). Chosen over worst-status/union: with one id for two tests the join cannot say which verdict belongs to which trace, so any merge is a guess V4 a file that failed outside any test (import error, throwing beforeAll) refuses by name; the reporter marks beforeAll-skipped tests "skipped" V5 realpath for cwd, TGTRACE_ROOT and the id root (Python and TS) V6 the generated config no longer forces `root` V7 all-skipped and no-match runs are empty successes; test.projects and vitest.workspace.* are refused up front with their real cause V8 --coverage.enabled=false V9 empty describe names kept in the TS id, as ancestorTitles keeps them V10 temp node_modules is a dir of per-entry symlinks so Vite's .vite-temp stays in the temp dir (--configLoader runner rejected: no __dirname) V11 report shape validated, refusal instead of a crash V12 default-export config awaited whether function result or Promise Adds a real-Vitest integration class (TG_VITEST_NODE_MODULES) covering the above; `execute` factored out of run() so it can be driven directly. --- CHANGELOG.md | 12 +- USAGE.md | 23 +- harness/plugin/tgtrace-vitest.ts | 45 +++- testgraph/vitest_adapter.py | 255 +++++++++++++----- tests/test_vitest_adapter.py | 428 ++++++++++++++++++++++++++++++- 5 files changed, 670 insertions(+), 93 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41103f7..99ac1b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,16 @@ below is on `main` or in the branch that introduced this file. `tgtrace.py`) records each test's named functions through V8 precise coverage; Vitest's JSON reporter supplies verdicts; the join, the attribution and the ledger rows are the pytest adapter's own. Injected through a generated config in a temp - dir, so nothing is written into the target repo. On a real Next.js repo (167 - Vitest tests) it attributes 5 journeys with no unjoined test. + dir, so nothing is written into the target repo (Vitest's own `node_modules/.vite` + cache aside). A broken or partial run is a refusal (exit 2), never a verdict: a + passed or failed test with no trace (a dead worker, a hook that threw), a + never-finished test, a duplicate test id, a file that failed outside any test + (import error, throwing `beforeAll`/`afterAll`), a malformed report, and a + `test.projects`/workspace config (projects do not inherit `setupFiles`). The trace + is recorded in `onTestFinished`, so a repo `afterEach` that throws no longer loses + it; the repo is resolved through `realpath`; the repo's own `root` is kept; coverage + is forced off for the run. On a real Next.js repo (167 Vitest tests) it attributes + 5 journeys with no unjoined test. - **`testgraph.reconcile`** — two-sided reconciliation of the static graph against a real test run, classifying every disagreement as a **STATIC MISS** (a symbol the journey's tests executed, outside its static footprint) or a diff --git a/USAGE.md b/USAGE.md index c476308..c1b76c4 100644 --- a/USAGE.md +++ b/USAGE.md @@ -189,11 +189,24 @@ It needs the repo's dependencies installed (`node_modules` with `vitest`, at the a workspace root). Nothing is written into the repo: a generated config in a temp dir imports the repo's own `vitest.config.*` (or `vite.config.*`) and adds the `tgtrace-vitest.ts` setup file, which records each test's named functions through V8 -precise coverage. Rows carry `edge_provenance=trace`, like the pytest adapter's. - -Limits: anonymous functions are not recorded (attribution matches named entry -symbols, so this costs nothing there); tests marked `.concurrent` in one file share a -coverage window; `verify` does not run Vitest suites yet. +precise coverage. Rows carry `edge_provenance=trace`, like the pytest adapter's. (Vitest +still writes its own cache to `node_modules/.vite`, as any `vitest run` does.) + +A run that cannot be read is refused (exit 2), never recorded: any passed or failed test +without a trace, a test that never finished (killed worker), two tests with the same id +(same title in one describe, e.g. a repeated `test.each` row), a file that failed outside +any test (import error, throwing `beforeAll`/`afterAll`; Vitest reports the tests under +a throwing `beforeAll` as skipped, so only the file's status shows it), and a report of +the wrong shape. An all-skipped run, or a `-t` filter that matches nothing, is an empty +success. + +Limits: `test.projects` and `vitest.workspace.*` configs are not supported yet (projects +do not inherit `setupFiles`, so nothing would be traced) and are refused up front; +anonymous functions are not recorded (attribution matches named entry symbols, so this +costs nothing there); tests marked `.concurrent` in one file share a coverage window, +so their traces can bleed into each other; a failing test next to tests skipped by a +throwing `beforeAll` in the same file is not distinguishable from `it.skip`; `verify` +does not run Vitest suites yet. ## Checking the Graph Against Reality diff --git a/harness/plugin/tgtrace-vitest.ts b/harness/plugin/tgtrace-vitest.ts index 9ac8562..a039999 100644 --- a/harness/plugin/tgtrace-vitest.ts +++ b/harness/plugin/tgtrace-vitest.ts @@ -18,7 +18,7 @@ * TGTRACE_SKIP colon-separated path fragments to ignore * (default: /node_modules/ .test. .spec. /__tests__/) * - * Part file: {"root", "backend", "file", "tests": {"<id>": [["<relpath>", "<name>"], ...]}} + * Part file: {"root", "backend", "file", "duplicates", "tests": {"<id>": [["<relpath>", "<name>"], ...]}} * where <id> is `<relfile>::<describe> > ... > <title>`, the same id the * adapter builds from Vitest's JSON reporter. * @@ -27,26 +27,35 @@ * coverage window, so their traces can bleed into each other. */ import { Session } from "node:inspector/promises"; -import { closeSync, fsyncSync, mkdirSync, openSync, renameSync, rmSync, writeSync } from "node:fs"; +import { closeSync, fsyncSync, mkdirSync, openSync, realpathSync, renameSync, rmSync, writeSync } from "node:fs"; import { createHash } from "node:crypto"; import { join, relative, sep } from "node:path"; import { fileURLToPath } from "node:url"; -import { afterAll, afterEach, beforeEach } from "vitest"; +import { afterAll, beforeEach, onTestFinished } from "vitest"; const OUT_DIR = process.env.TGTRACE_OUT_DIR; -const ROOT = process.env.TGTRACE_ROOT ?? process.cwd(); +// Real path: V8 script URLs and Vitest's test file paths are symlink-resolved, so +// a root given through a symlink would never prefix-match them. +const ROOT = (() => { + const root = process.env.TGTRACE_ROOT ?? process.cwd(); + try { + return realpathSync(root); + } catch { + return root; + } +})(); const SKIP = (process.env.TGTRACE_SKIP ?? "/node_modules/:.test.:.spec.:/__tests__/") .split(":") .filter(Boolean); type Hit = [string, string]; -type Task = { name: string; type?: string; suite?: Task; file?: { filepath?: string; name: string } }; +type Task = { id: string; name: string; type?: string; suite?: Task; file?: { filepath?: string; name: string } }; function testId(task: Task): string { const filepath = task.file?.filepath ?? task.file?.name ?? ""; const names: string[] = [task.name]; for (let s = task.suite; s && s.type === "suite" && s.name !== task.file?.name; s = s.suite) { - if (s.name) names.unshift(s.name); + names.unshift(s.name); // empty names stay: the JSON reporter's ancestorTitles keeps them } return `${relative(ROOT, filepath).split(sep).join("/")}::${names.join(" > ")}`; } @@ -68,14 +77,22 @@ if (OUT_DIR) { const tests: Record<string, Hit[]> = {}; let file = ""; + const owner = new Map<string, string>(); // test id -> task id, to spot two tests sharing an id + const duplicates = new Set<string>(); - beforeEach(async () => { + beforeEach(async (ctx) => { await session.post("Profiler.takePreciseCoverage"); // reset: drop setup-time calls + // Not afterEach: Vitest runs afterEach hooks in reverse order and a throwing + // one stops the chain, so ours (registered first) would be skipped and a + // failing test would lose its trace. onTestFinished runs after every + // afterEach whether or not they threw. + onTestFinished(async () => { + await record(ctx.task as unknown as Task); + }); }); - afterEach(async (ctx) => { + async function record(task: Task): Promise<void> { const { result } = await session.post("Profiler.takePreciseCoverage"); - const task = ctx.task as unknown as Task; file = task.file?.filepath ?? file; const hits = new Map<string, Hit>(); for (const script of result) { @@ -88,8 +105,12 @@ if (OUT_DIR) { if (count > 0 && fn.functionName) hits.set(`${rel}\0${fn.functionName}`, [rel, fn.functionName]); } } - tests[testId(task)] = [...hits.values()].sort(); - }); + const id = testId(task); + const prior = owner.get(id); + if (prior !== undefined && prior !== task.id) duplicates.add(id); + owner.set(id, task.id); + tests[id] = [...hits.values()].sort(); + } afterAll(async () => { await session.post("Profiler.stopPreciseCoverage"); @@ -98,7 +119,7 @@ if (OUT_DIR) { const name = createHash("sha1").update(file).digest("hex").slice(0, 16); const target = join(OUT_DIR, `${name}.json`); const tmp = `${target}.tmp.${process.pid}`; - const payload = { root: ROOT, backend: "v8-precise-coverage", file: relative(ROOT, file), tests }; + const payload = { root: ROOT, backend: "v8-precise-coverage", file: relative(ROOT, file), tests, duplicates: [...duplicates].sort() }; // Whole or absent, as tgtrace.py writes: a worker killed mid-write must // not leave a truncated part for the adapter to merge. try { diff --git a/testgraph/vitest_adapter.py b/testgraph/vitest_adapter.py index cf491ac..76694b4 100644 --- a/testgraph/vitest_adapter.py +++ b/testgraph/vitest_adapter.py @@ -13,8 +13,17 @@ Nothing is written into the target repo. The setup file is injected through a generated config in a temp dir that imports the repo's own config and `mergeConfig`s the setup file in; Vitest has no `--setupFiles` CLI flag. The -temp dir gets a `node_modules` symlink to the repo's, so the generated config -and the setup file resolve `vitest` exactly as the repo's tests do. +temp dir gets a `node_modules` of symlinks to the repo's entries, so the +generated config and the setup file resolve `vitest` exactly as the repo's +tests do, while the config bundle Vite writes (`.vite-temp`) lands in the temp +dir. Vitest's own cache (`node_modules/.vite`) is written where any +`vitest run` in the repo writes it. (`--configLoader runner` would avoid the +bundle but does not define `__dirname`, which real configs use.) + +A broken or partial run is never a verdict: a missing trace for a test that +passed or failed, a duplicate test id, a file that failed outside any test, a +report of the wrong shape, and a `projects`/workspace config (which does not +inherit `setupFiles`) are all refusals. Deliberately not here yet: selecting only some journeys' tests (verify's job; next step), and anonymous functions (they need source maps, which only @@ -42,7 +51,7 @@ ) # Vitest JSON reporter status -> the run-result status `results` defines. STATUS = {"passed": "pass", "failed": "fail", "skipped": "skip", - "pending": "skip", "todo": "skip"} + "todo": "skip"} def find_node_modules(repo): @@ -60,6 +69,20 @@ def find_node_modules(repo): d = parent +def link_node_modules(source, dest): + """A real `dest` directory holding a symlink to each entry of `source`. + + Not one symlink to the directory: Vite bundles a config into + `node_modules/.vite-temp` beside the config file, which through a + directory symlink would be the target's own node_modules. Here it lands in + `dest`, which dies with the temp dir. `.vite` and `.vite-temp` are not + linked, so no stale cache is read either.""" + os.makedirs(dest, exist_ok=True) + for entry in os.listdir(source): + if entry not in (".vite", ".vite-temp") and not os.path.lexists(os.path.join(dest, entry)): + os.symlink(os.path.join(source, entry), os.path.join(dest, entry)) + + def find_config(repo): """The repo's own Vitest (else Vite) config file, or None.""" for name in CONFIG_NAMES: @@ -69,23 +92,45 @@ def find_config(repo): return None +WORKSPACE_STEMS = tuple( + f"vitest.workspace.{ext}" for ext in ("ts", "mts", "cts", "js", "mjs", "cjs", "json")) +PROJECTS_MARKER = "tgtrace-vitest: test.projects is not supported yet" +PROJECTS_REFUSAL = ("the repo's Vitest config uses `test.projects` (or a workspace " + "file), which this adapter does not support yet: projects do " + "not inherit `setupFiles`, so no test would be traced") + + +def find_workspace(repo): + """A `vitest.workspace.*` file in the repo, or None.""" + for name in WORKSPACE_STEMS: + path = os.path.join(repo, name) + if os.path.isfile(path): + return path + return None + + def config_source(repo, base_config, setup_file): """The generated config: the repo's config plus our setup file. `mergeConfig` concatenates `setupFiles`, so the repo's own setup files still run, before ours. A config exported as a function is called the way Vitest calls it for `vitest run`.""" + # `await` takes a function's result and a Promise default export alike. + # No `root`: the run's cwd is the repo, which is already Vitest's default, + # and forcing one would override a `root` the repo sets itself. base = ( f"import base from {json.dumps(base_config)};\n" - "const resolved = typeof base === \"function\"\n" - " ? await base({ command: \"serve\", mode: \"test\" })\n" - " : base;\n" + "const resolved = await (typeof base === \"function\"\n" + " ? base({ command: \"serve\", mode: \"test\" })\n" + " : base);\n" + "if (resolved?.test?.projects || resolved?.test?.workspace) {\n" + f" throw new Error({json.dumps(PROJECTS_MARKER)});\n" + "}\n" ) if base_config else "const resolved = {};\n" return ( "import { defineConfig, mergeConfig } from \"vitest/config\";\n" + base + "export default mergeConfig(resolved ?? {}, defineConfig({\n" - f" root: {json.dumps(os.path.abspath(repo))},\n" f" test: {{ setupFiles: [{json.dumps(setup_file)}] }},\n" "}));\n" ) @@ -97,21 +142,59 @@ def test_id(root, file_path, ancestors, title): return f"{rel}::{' > '.join([*ancestors, title])}" +def _bad_report(why): + return None, f"vitest's JSON report has an unexpected shape ({why}); no verdict was recorded" + + def parse_report(report, root): - """Vitest JSON reporter output -> {join_key: {"status": ...}}. + """Vitest JSON reporter output -> ({join_key: {"status", "test_id"}}, error). Keyed through `pytest_adapter._join_key`, the key `attribute` joins on, so - the generic attribution code needs no Vitest branch.""" + the generic attribution code needs no Vitest branch. The error is a + refusal: the report is malformed, a test id appears twice (the join would + keep one and mask the other, so a failure could vanish), or a file failed + outside any test (import or collection error, a throwing beforeAll or + afterAll), which the per-test rows cannot show.""" + results = report.get("testResults") + if not isinstance(results, list): + return _bad_report("`testResults` is missing or not a list") out = {} - for file_result in report.get("testResults", []): - for case in file_result.get("assertionResults", []): - tid = test_id(root, file_result["name"], case.get("ancestorTitles", []), - case.get("title", "")) - out[pa._join_key(tid)] = { - "status": STATUS.get(case.get("status"), "skip"), - "test_id": tid, - } - return out + for file_result in results: + if not isinstance(file_result, dict) or not isinstance(file_result.get("name"), str): + return _bad_report("a `testResults` entry is not an object with a string `name`") + cases = file_result.get("assertionResults") + if not isinstance(cases, list) or not all(isinstance(c, dict) for c in cases): + return _bad_report(f"`assertionResults` of {file_result['name']} is not a list of objects") + rel = os.path.relpath(file_result["name"], root).replace(os.sep, "/") + if file_result.get("status") == "failed" and not any( + c.get("status") == "failed" for c in cases): + # Vitest reports tests under a throwing beforeAll as "skipped", so + # only the file's own status says it broke; reading those tests as + # skips would hide the failure. + message = str(file_result.get("message") or "").strip().splitlines() + return None, (f"{rel} failed outside any test (an import or collection " + f"error, or a beforeAll/afterAll that threw); no verdict " + f"was recorded" + (f"\n {message[0][:300]}" if message else "")) + for case in cases: + ancestors, title = case.get("ancestorTitles", []), case.get("title") + if (not isinstance(ancestors, list) + or not all(isinstance(a, str) for a in ancestors) + or not isinstance(title, str)): + return _bad_report(f"a test in {rel} lacks a string `title` or a " + f"list-of-strings `ancestorTitles`") + tid = test_id(root, file_result["name"], ancestors, title) + if case.get("status") == "pending": + # The reporter's "pending" is a test that never finished (a + # killed worker, a bail), not a skip: a skip is "skipped". + return None, (f"test {tid!r} never finished (status pending: a worker " + f"died or the run was cut short); no verdict was recorded") + key = pa._join_key(tid) + if key in out: + return None, (f"two tests share the id {tid!r} (the same title in the " + f"same describe, e.g. test.each with a repeated row); the " + f"verdicts cannot be told apart, so no verdict was recorded") + out[key] = {"status": STATUS.get(case.get("status"), "skip"), "test_id": tid} + return out, None def merge_parts(out_dir): @@ -131,11 +214,95 @@ def merge_parts(out_dir): if not isinstance(part, dict) or pa.payload_shape_error(part): return None, (f"{os.path.basename(path)} is not a tgtrace-vitest part; " f"no verdict was recorded") + dups = part.get("duplicates") or [] + if not isinstance(dups, list): + return None, (f"{os.path.basename(path)} is not a tgtrace-vitest part; " + f"no verdict was recorded") + if dups: + return None, (f"two tests share the id {dups[0]!r} (the same title in the " + f"same describe, e.g. test.each with a repeated row); their " + f"traces cannot be told apart, so no verdict was recorded") tests.update(part["tests"]) trace = {"backend": BACKEND, "tests": tests, "declared": {}} return trace, None +def execute(repo, plugin, node_modules, tmp, vitest_args=(), runner=None): + """One traced Vitest run in `tmp`: (proc, trace, verdicts, error). + + Any error is a refusal: nothing here may turn a broken or partial run into + a verdict.""" + # The real path, for everything compared: V8 script URLs and Vitest's test + # file paths are symlink-resolved, so a root spelled through a symlink + # would never prefix-match them and every trace would be empty. + repo = os.path.realpath(repo) + workspace = find_workspace(repo) + if workspace: + return None, None, None, f"{PROJECTS_REFUSAL} ({os.path.basename(workspace)})" + parts = os.path.join(tmp, "parts") + os.makedirs(parts, exist_ok=True) + report_path = os.path.join(tmp, "results.json") + setup = os.path.join(tmp, PLUGIN_NAME) + shutil.copyfile(plugin, setup) + config = os.path.join(tmp, "tg.vitest.config.mts") + with open(config, "w") as fh: + fh.write(config_source(repo, find_config(repo), setup)) + if node_modules: + link_node_modules(node_modules, os.path.join(tmp, "node_modules")) + env = dict(os.environ, TGTRACE_OUT_DIR=parts, TGTRACE_ROOT=repo) + cmd = [os.path.join(node_modules or "", ".bin", "vitest"), "run", + "--config", config, + "--coverage.enabled=false", + "--reporter=json", f"--outputFile={report_path}", *vitest_args] + runner = runner or (lambda: subprocess.run( + cmd, cwd=repo, env=env, capture_output=True, text=True)) + proc = runner() + + stderr = getattr(proc, "stderr", "") or "" + tail = stderr.strip()[-400:] + if not os.path.isfile(report_path): + if PROJECTS_MARKER in stderr: + return proc, None, None, PROJECTS_REFUSAL + return proc, None, None, ( + f"vitest exited {getattr(proc, 'returncode', '?')} and wrote no " + f"JSON report; no verdict was recorded" + + (f"\n vitest stderr (tail): {tail}" if tail else "")) + try: + with open(report_path) as fh: + report = json.load(fh) + except (OSError, ValueError) as exc: + return proc, None, None, (f"vitest's JSON report is truncated or not " + f"JSON ({exc}); no verdict was recorded") + if not isinstance(report, dict): + return proc, None, None, "vitest's JSON report is not an object" + verdicts, err = parse_report(report, repo) + if err: + return proc, None, None, err + trace, err = merge_parts(parts) + if err: + return proc, None, None, err + traced = {pa._join_key(t) for t in trace["tests"]} + # A test that passed or failed ran its beforeEach, so it has a trace. Skipped + # and todo tests never ran a hook and legitimately have none. A missing + # trace means a worker died or the setup file never ran, and attributing + # from the rest would silently shrink coverage. + ran = [k for k, v in verdicts.items() if v["status"] in ("pass", "fail")] + missing = [k for k in ran if k not in traced] + if missing and len(missing) == len(ran): + return proc, None, None, ( + f"vitest ran {len(ran)} test(s) (e.g. {verdicts[missing[0]]['test_id']!r}) " + f"but tgtrace-vitest traced none — the setup file did not load or did " + f"not reach the tests (a projects/workspace config does not inherit " + f"setupFiles); no verdict was recorded" + + (f"\n vitest stderr (tail): {tail}" if tail else "")) + if missing: + return proc, None, None, ( + f"{len(missing)} of {len(ran)} passed/failed tests have no trace " + f"(e.g. {verdicts[missing[0]]['test_id']!r}) — a worker died or a hook " + f"threw before the trace was written; no verdict was recorded") + return proc, trace, verdicts, None + + def run(repo, registry_path=None, db_path=None, vitest_args=(), commit=None, append=None, trace_out=None, _runner=None, _workdir=None): """Run the suite, attribute journeys from the trace, write the rows. @@ -143,6 +310,7 @@ def run(repo, registry_path=None, db_path=None, vitest_args=(), commit=None, Returns (summary, error), as `pytest_adapter.run` does: a red suite is a result, a run that cannot be read is a refusal.""" repo = os.path.abspath(repo) + real = os.path.realpath(repo) # `execute` resolves it too; node_modules is looked up from it registry_path = registry_path or reg.resolve_for_repo(repo) if registry_path is None: return None, (f"no journey registry for {reg.repo_name(repo)} — looked in " @@ -162,7 +330,7 @@ def run(repo, registry_path=None, db_path=None, vitest_args=(), commit=None, if not os.path.isfile(db_path): return None, f"no codegraph index at {db_path} — run `codegraph init` in {repo}" - node_modules = find_node_modules(repo) + node_modules = find_node_modules(real) if node_modules is None and _runner is None: return None, (f"no node_modules with vitest at or above {repo} — install " f"the repo's dependencies first") @@ -177,56 +345,11 @@ def run(repo, registry_path=None, db_path=None, vitest_args=(), commit=None, return None, (f"cannot resolve {commit or 'HEAD'!r} in {repo} — an " f"unjoinable key would sit in `unasked` forever") - def go(tmp): - parts = os.path.join(tmp, "parts") - os.makedirs(parts, exist_ok=True) - report_path = os.path.join(tmp, "results.json") - setup = os.path.join(tmp, PLUGIN_NAME) - shutil.copyfile(plugin, setup) - config = os.path.join(tmp, "tg.vitest.config.mts") - with open(config, "w") as fh: - fh.write(config_source(repo, find_config(repo), setup)) - if node_modules and not os.path.exists(os.path.join(tmp, "node_modules")): - os.symlink(node_modules, os.path.join(tmp, "node_modules")) - env = dict(os.environ, TGTRACE_OUT_DIR=parts, TGTRACE_ROOT=repo) - cmd = [os.path.join(node_modules or "", ".bin", "vitest"), "run", - "--config", config, "--reporter=json", f"--outputFile={report_path}", - *vitest_args] - runner = _runner or (lambda: subprocess.run( - cmd, cwd=repo, env=env, capture_output=True, text=True)) - proc = runner() - - tail = (getattr(proc, "stderr", "") or "").strip()[-400:] - if not os.path.isfile(report_path): - return proc, None, None, ( - f"vitest exited {getattr(proc, 'returncode', '?')} and wrote no " - f"JSON report; no verdict was recorded" - + (f"\n vitest stderr (tail): {tail}" if tail else "")) - try: - with open(report_path) as fh: - report = json.load(fh) - except (OSError, ValueError) as exc: - return proc, None, None, (f"vitest's JSON report is truncated or not " - f"JSON ({exc}); no verdict was recorded") - if not isinstance(report, dict): - return proc, None, None, "vitest's JSON report is not an object" - verdicts = parse_report(report, repo) - trace, err = merge_parts(parts) - if err: - return proc, None, None, err - if verdicts and not trace["tests"]: - # Tests ran, the setup file recorded nothing: it did not load. - return proc, None, None, ( - "vitest ran tests but tgtrace-vitest wrote no trace — the setup " - "file did not load; no verdict was recorded" - + (f"\n vitest stderr (tail): {tail}" if tail else "")) - return proc, trace, verdicts, None - if _workdir is not None: - proc, trace, verdicts, err = go(_workdir) + proc, trace, verdicts, err = execute(repo, plugin, node_modules, _workdir, vitest_args, _runner) else: with tempfile.TemporaryDirectory() as tmp: - proc, trace, verdicts, err = go(tmp) + proc, trace, verdicts, err = execute(repo, plugin, node_modules, tmp, vitest_args, _runner) if err: return None, err if trace_out: diff --git a/tests/test_vitest_adapter.py b/tests/test_vitest_adapter.py index 3aad726..c9251d1 100644 --- a/tests/test_vitest_adapter.py +++ b/tests/test_vitest_adapter.py @@ -11,6 +11,7 @@ import json import os import shutil +import subprocess import sys import tempfile import types @@ -43,18 +44,28 @@ def test_the_id_carries_the_describe_chain(self): def test_statuses_map_to_run_results(self): report = _report("/r", [("a.test.ts", [], t, st) for t, st in ( - ("p", "passed"), ("f", "failed"), ("s", "skipped"), ("n", "pending"), + ("p", "passed"), ("f", "failed"), ("s", "skipped"), ("t", "todo"), ("weird", "something-new"))]) - got = {v["test_id"].split("::")[1]: v["status"] - for v in va.parse_report(report, "/r").values()} - self.assertEqual(got, {"p": "pass", "f": "fail", "s": "skip", "n": "skip", + verdicts, err = va.parse_report(report, "/r") + self.assertIsNone(err) + got = {v["test_id"].split("::")[1]: v["status"] for v in verdicts.values()} + self.assertEqual(got, {"p": "pass", "f": "fail", "s": "skip", "t": "skip", "weird": "skip"}) def test_report_keys_are_the_keys_attribute_joins_on(self): report = _report("/r", [("src/a.test.ts", ["d"], "x", "passed")]) - key, = va.parse_report(report, "/r") + verdicts, err = va.parse_report(report, "/r") + key, = verdicts self.assertEqual(key, pa._join_key("src/a.test.ts::d > x")) + def test_an_empty_describe_name_is_kept_as_the_reporter_keeps_it(self): + """V9: ancestorTitles [""] -> an id with an empty segment, which the + setup file must build the same way.""" + report = _report("/r", [("a.test.ts", [""], "x", "passed")]) + verdicts, _ = va.parse_report(report, "/r") + key, = verdicts + self.assertEqual(verdicts[key]["test_id"], "a.test.ts:: > x") + class DiscoveryTests(unittest.TestCase): def setUp(self): @@ -87,7 +98,10 @@ def test_the_generated_config_merges_into_the_repos_own(self): self.assertIn('typeof base === "function"', src) self.assertIn("mergeConfig(resolved", src) self.assertIn('setupFiles: ["/tmp/x/tgtrace-vitest.ts"]', src) - self.assertIn('root: "/r"', src) + # V6: no forced root; cwd is the repo, and the repo's own `root` must win. + self.assertNotIn("root:", src) + # V12: a Promise default export is awaited like a function's result. + self.assertIn("await (typeof base", src) bare = va.config_source("/r", None, "/s.ts") self.assertNotIn("import base", bare) self.assertIn("const resolved = {}", bare) @@ -199,11 +213,12 @@ def test_a_report_that_is_not_an_object_is_refused(self): summary, err = self._run(report_text="[]") self.assertIn("not an object", err) - def test_tests_with_no_trace_mean_the_setup_file_did_not_load(self): + def test_tests_with_no_trace_mean_the_setup_file_did_not_reach_them(self): report = _report(self.repo, [("src/a.test.ts", [], "x", "passed")]) summary, err = self._run(report, stderr="boom") self.assertIsNone(summary) - self.assertIn("setup file did not load", err) + self.assertIn("did not load or did not reach the tests", err) + self.assertIn("does not inherit setupFiles", err) def test_a_truncated_part_is_refused(self): report = _report(self.repo, [("src/a.test.ts", [], "x", "passed")]) @@ -247,6 +262,403 @@ def test_main_json_refusal_is_json(self): self.assertEqual(json.loads(out.getvalue()), {"refused": True, "error": "broke"}) +class Round1RefusalTests(_VitestRunFixture, unittest.TestCase): + """Review round 1 (V1-V12): a broken or partial run is a refusal, never a + verdict, never attribution.""" + A = "src/a.test.ts::svc > calls handler" + B = "src/a.test.ts::other" + + def _two(self, st_b="passed"): + return _report(self.repo, [("src/a.test.ts", ["svc"], "calls handler", "passed"), + ("src/a.test.ts", [], "other", st_b)]) + + # V1 / V2 + def test_V1_a_passed_test_without_a_trace_is_refused(self): + summary, err = self._run(self._two(), parts=[{self.A: [["app/svc.py", "handler_a"]]}]) + self.assertIsNone(summary) + self.assertIn("1 of 2", err) + self.assertIn("src/a.test.ts::other", err) + + def test_V1_V2_a_failed_test_without_a_trace_is_refused(self): + """The repo's afterEach threw, our hook never ran: a verdict, no trace.""" + summary, err = self._run(self._two("failed"), + parts=[{self.A: [["app/svc.py", "handler_a"]]}]) + self.assertIsNone(summary) + self.assertIn("src/a.test.ts::other", err) + + def test_V1_skipped_and_todo_tests_need_no_trace(self): + report = _report(self.repo, [("src/a.test.ts", ["svc"], "calls handler", "passed"), + ("src/a.test.ts", [], "s", "skipped"), + ("src/a.test.ts", [], "t", "todo")]) + summary, err = self._run(report, parts=[{self.A: [["app/svc.py", "handler_a"]]}]) + self.assertIsNone(err) + self.assertEqual(summary["journeys"], {"J1": "pass"}) + + def test_a_test_that_never_finished_is_refused(self): + """A killed worker leaves its tests `pending`, which must not read as skip.""" + report = _report(self.repo, [("src/a.test.ts", ["svc"], "calls handler", "passed"), + ("src/a.test.ts", [], "other", "pending")]) + summary, err = self._run(report, parts=[{self.A: [["app/svc.py", "handler_a"]]}]) + self.assertIsNone(summary) + self.assertIn("never finished", err) + self.assertIn("src/a.test.ts::other", err) + + # V3 + def test_V3_duplicate_ids_in_the_report_are_refused(self): + report = _report(self.repo, [("src/a.test.ts", ["d"], "same", "passed"), + ("src/a.test.ts", ["d"], "same", "failed")]) + summary, err = self._run(report, parts=[{"src/a.test.ts::d > same": []}]) + self.assertIsNone(summary) + self.assertIn("share the id 'src/a.test.ts::d > same'", err) + + def test_V3_a_part_that_saw_a_duplicate_is_refused(self): + report = _report(self.repo, [("src/a.test.ts", [], "x", "passed")]) + text = json.dumps({"tests": {"src/a.test.ts::x": []}, + "duplicates": ["src/a.test.ts::x"]}) + summary, err = self._run(report, parts=[({}, "p1.json", text)]) + self.assertIsNone(summary) + self.assertIn("share the id 'src/a.test.ts::x'", err) + + # V4 + def test_V4_a_file_that_failed_to_import_is_refused_by_name(self): + report = {"testResults": [{"name": os.path.join(self.repo, "src/c.test.ts"), + "status": "failed", "message": "Cannot find module './x'", + "assertionResults": []}]} + summary, err = self._run(report) + self.assertIsNone(summary) + self.assertIn("src/c.test.ts failed outside any test", err) + self.assertIn("Cannot find module", err) + + def test_V4_a_throwing_beforeall_is_not_read_as_skips(self): + """Vitest reports the file `failed` and its tests `skipped`.""" + report = _report(self.repo, [("src/b.test.ts", ["bb"], "t1", "skipped")]) + report["testResults"][0]["status"] = "failed" + summary, err = self._run(report) + self.assertIsNone(summary) + self.assertIn("src/b.test.ts failed outside any test", err) + + def test_V4_a_file_with_a_failing_test_is_a_result(self): + report = self._two("failed") + report["testResults"][0]["status"] = "failed" + summary, err = self._run(report, parts=[{self.A: [["app/svc.py", "handler_a"]], + self.B: []}]) + self.assertIsNone(err) + self.assertEqual(summary["journeys"], {"J1": "pass"}) + + # V5 + def test_V5_a_symlinked_repo_is_compared_by_its_real_path(self): + link = os.path.join(self.tmp, "link") + os.symlink(self.repo, link) + report = _report(os.path.realpath(self.repo), + [("src/a.test.ts", ["svc"], "calls handler", "passed")]) + work = self.work + + def runner(): + self._part({self.A: [["app/svc.py", "handler_a"]]}) + with open(os.path.join(work, "results.json"), "w") as fh: + json.dump(report, fh) + return types.SimpleNamespace(returncode=0, stderr="") + summary, err = va.run(link, registry_path=self.registry, db_path=self.db, + _runner=runner, _workdir=work, append=lambda r: True) + self.assertIsNone(err) + self.assertEqual(summary["journeys"], {"J1": "pass"}) + self.assertEqual(summary["traced_without_verdict"], []) + + def _captured_command(self, repo): + calls = [] + + def fake_run(cmd, **kw): + calls.append((cmd, kw)) + return types.SimpleNamespace(returncode=0, stderr="") + with unittest.mock.patch.object(va.subprocess, "run", fake_run): + va.execute(repo, os.path.join(pa.plugin_dir(), va.PLUGIN_NAME), None, self.work) + return calls[0] + + def test_V5_the_run_gets_the_real_path_as_cwd_and_trace_root(self): + link = os.path.join(self.tmp, "link2") + os.symlink(self.repo, link) + _, kw = self._captured_command(link) + self.assertEqual(kw["cwd"], os.path.realpath(self.repo)) + self.assertEqual(kw["env"]["TGTRACE_ROOT"], os.path.realpath(self.repo)) + + # V8 / V10 + def test_V8_the_command_line_disables_coverage(self): + cmd, _ = self._captured_command(self.repo) + self.assertIn("--coverage.enabled=false", cmd) + + def test_V10_the_temp_node_modules_is_a_real_dir_of_links(self): + """A directory symlink would send Vite's `.vite-temp` into the target.""" + src = os.path.join(self.tmp, "nm") + os.makedirs(os.path.join(src, "vitest")) + os.makedirs(os.path.join(src, ".vite-temp")) + os.makedirs(os.path.join(src, ".bin")) + dest = os.path.join(self.tmp, "t", "node_modules") + va.link_node_modules(src, dest) + self.assertFalse(os.path.islink(dest)) + self.assertTrue(os.path.islink(os.path.join(dest, "vitest"))) + self.assertTrue(os.path.islink(os.path.join(dest, ".bin"))) + self.assertFalse(os.path.lexists(os.path.join(dest, ".vite-temp"))) + + # V7 + def test_V7_an_all_skipped_run_is_not_a_refusal(self): + report = _report(self.repo, [("src/a.test.ts", [], "s", "skipped"), + ("src/a.test.ts", [], "t", "todo")]) + summary, err = self._run(report) + self.assertIsNone(err) + self.assertEqual(summary["journeys"], {}) + + def test_V7_a_filter_that_matches_nothing_is_not_a_refusal(self): + summary, err = self._run({"testResults": [{ + "name": os.path.join(self.repo, "src/a.test.ts"), "status": "passed", + "assertionResults": []}]}) + self.assertIsNone(err) + + def test_V7_a_projects_config_is_refused_up_front(self): + summary, err = self._run(None, stderr=f"Error: {va.PROJECTS_MARKER}\n at ...") + self.assertIsNone(summary) + self.assertIn("test.projects", err) + self.assertIn("does not support yet", err) + + def test_V7_a_workspace_file_is_refused_before_running(self): + open(os.path.join(self.repo, "vitest.workspace.ts"), "w").close() + ran = [] + + def runner(): + ran.append(1) + return types.SimpleNamespace(returncode=0, stderr="") + summary, err = va.run(self.repo, registry_path=self.registry, db_path=self.db, + _runner=runner, _workdir=self.work) + self.assertIsNone(summary) + self.assertIn("workspace", err) + self.assertEqual(ran, []) + + def test_V7_the_generated_config_refuses_a_projects_config(self): + src = va.config_source("/r", "/r/vitest.config.ts", "/s.ts") + self.assertIn("resolved?.test?.projects", src) + self.assertIn(va.PROJECTS_MARKER, src) + + # V11 + def test_V11_a_malformed_report_is_a_refusal_not_a_crash(self): + bad = [ + {}, + {"testResults": None}, + {"testResults": [None]}, + {"testResults": [{"assertionResults": []}]}, + {"testResults": [{"name": "/x/a.ts", "assertionResults": None}]}, + {"testResults": [{"name": "/x/a.ts", "assertionResults": [None]}]}, + {"testResults": [{"name": "/x/a.ts", "assertionResults": [ + {"title": None, "ancestorTitles": [], "status": "passed"}]}]}, + {"testResults": [{"name": "/x/a.ts", "assertionResults": [ + {"title": "t", "ancestorTitles": None, "status": "passed"}]}]}, + {"testResults": [{"name": "/x/a.ts", "assertionResults": [ + {"title": "t", "ancestorTitles": [1], "status": "passed"}]}]}, + ] + for report in bad: + with self.subTest(report=report): + summary, err = self._run(report) + self.assertIsNone(summary) + self.assertIn("unexpected shape", err) + + +VITEST_NODE_MODULES = os.environ.get( + "TG_VITEST_NODE_MODULES", + "/home/ericm/personal_projects/signedintake/main/node_modules") +HAVE_VITEST = bool(shutil.which("node")) and os.path.isfile( + os.path.join(VITEST_NODE_MODULES, "vitest", "package.json")) + +LIB = "export function handlerA() { return 1; }\n" +CONFIG = ('import { defineConfig } from "vitest/config";\n' + 'export default defineConfig({ test: { include: ["src/**/*.test.ts"] } });\n') + + +@unittest.skipUnless(HAVE_VITEST, "needs node and a vitest install " + "(TG_VITEST_NODE_MODULES, default: signedintake's node_modules)") +class RealVitestTests(unittest.TestCase): + """The setup file and the adapter against a real Vitest run (#122, review + round 1). Drives `execute`, the traced run `run()` is built from; the + codegraph join is covered by the hermetic tests.""" + + def _project(self, files, config=CONFIG): + d = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, d, ignore_errors=True) + # A real node_modules of symlinks to each entry, not one symlink to the + # directory: Vitest/Vite write caches into node_modules, which must + # land here and never in the shared install. + nm = os.path.join(d, "node_modules") + os.mkdir(nm) + for entry in os.listdir(VITEST_NODE_MODULES): + if entry not in (".vite", ".vite-temp", ".cache"): + os.symlink(os.path.join(VITEST_NODE_MODULES, entry), os.path.join(nm, entry)) + os.makedirs(os.path.join(d, "src")) + with open(os.path.join(d, "src", "lib.ts"), "w") as fh: + fh.write(LIB) + if config: + with open(os.path.join(d, "vitest.config.ts"), "w") as fh: + fh.write(config) + for name, text in files.items(): + with open(os.path.join(d, "src", name), "w") as fh: + fh.write(text) + return d + + def _exec(self, d, repo=None): + work = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, work, ignore_errors=True) + plugin = os.path.join(pa.plugin_dir(), va.PLUGIN_NAME) + return va.execute(repo or d, plugin, os.path.join(d, "node_modules"), work) + + @staticmethod + def _keys(trace, verdicts): + return ({pa._join_key(t) for t in trace["tests"]}, set(verdicts)) + + def test_V2_a_throwing_repo_afterEach_does_not_cost_the_trace(self): + d = self._project({"a.test.ts": ( + 'import { it, expect, afterEach } from "vitest";\n' + 'import { handlerA } from "./lib";\n' + 'afterEach(() => { throw new Error("repo afterEach"); });\n' + 'it("passes", () => { handlerA(); });\n' + 'it("fails", () => { handlerA(); expect(1).toBe(2); });\n')}) + proc, trace, verdicts, err = self._exec(d) + self.assertIsNone(err) + self.assertEqual({v["status"] for v in verdicts.values()}, {"fail"}) + self.assertEqual(trace["tests"], { + "src/a.test.ts::passes": [["src/lib.ts", "handlerA"]], + "src/a.test.ts::fails": [["src/lib.ts", "handlerA"]]}) + + def test_V1_a_part_that_never_landed_is_a_refusal(self): + d = self._project({ + "a.test.ts": 'import { it } from "vitest";\nit("x", () => {});\n', + "b.test.ts": 'import { it } from "vitest";\nit("y", () => {});\n'}) + real_run = subprocess.run + + def lose_a_part(cmd, **kw): + proc = real_run(cmd, **kw) + parts = kw["env"]["TGTRACE_OUT_DIR"] + os.remove(os.path.join(parts, sorted(os.listdir(parts))[0])) + return proc + with unittest.mock.patch.object(va.subprocess, "run", lose_a_part): + proc, trace, verdicts, err = self._exec(d) + self.assertIsNone(trace) + self.assertIn("1 of 2 passed/failed tests have no trace", err) + + def test_a_killed_worker_is_a_refusal_not_a_skip(self): + d = self._project({"k.test.ts": ( + 'import { it } from "vitest";\n' + 'it("dies", () => { process.kill(process.pid, "SIGKILL"); });\n')}) + proc, trace, verdicts, err = self._exec(d) + self.assertIsNone(trace) + self.assertIsNotNone(err) + + def test_V3_duplicate_titles_are_a_refusal(self): + d = self._project({"a.test.ts": ( + 'import { it, expect, describe } from "vitest";\n' + 'describe("dup", () => { it("same", () => {});' + ' it("same", () => { expect(1).toBe(2); }); });\n')}) + proc, trace, verdicts, err = self._exec(d) + self.assertIsNone(trace) + self.assertIn("share the id 'src/a.test.ts::dup > same'", err) + + def test_V3_the_setup_file_flags_a_duplicate_in_its_own_part(self): + d = self._project({"a.test.ts": ( + 'import { it, describe } from "vitest";\n' + 'describe("dup", () => { it("same", () => {}); it("same", () => {}); });\n')}) + seen = [] + real_merge = va.merge_parts + + def spy(out_dir): + for name in os.listdir(out_dir): + with open(os.path.join(out_dir, name)) as fh: + seen.append(json.load(fh)) + return real_merge(out_dir) + # parse_report would refuse the duplicate first; bypass it to see the part. + with unittest.mock.patch.object(va, "merge_parts", spy), \ + unittest.mock.patch.object(va, "parse_report", lambda r, root: ({}, None)): + self._exec(d) + self.assertEqual([p["duplicates"] for p in seen], [["src/a.test.ts::dup > same"]]) + + def test_V3_a_retried_test_is_not_a_duplicate(self): + d = self._project({"a.test.ts": ( + 'import { it, expect } from "vitest";\n' + 'let n = 0;\n' + 'it("flaky", { retry: 2 }, () => { n++; expect(n).toBe(2); });\n')}) + proc, trace, verdicts, err = self._exec(d) + self.assertIsNone(err) + self.assertEqual({v["status"] for v in verdicts.values()}, {"pass"}) + + def test_V9_an_empty_describe_name_joins(self): + d = self._project({"a.test.ts": ( + 'import { it, describe } from "vitest";\n' + 'import { handlerA } from "./lib";\n' + 'describe("", () => { it("x", () => { handlerA(); }); });\n')}) + proc, trace, verdicts, err = self._exec(d) + self.assertIsNone(err) + traced, judged = self._keys(trace, verdicts) + self.assertEqual(traced, judged) + self.assertEqual(list(trace["tests"]), ["src/a.test.ts:: > x"]) + + def test_V5_a_repo_spelled_through_a_symlink_still_traces(self): + d = self._project({"a.test.ts": ( + 'import { it } from "vitest";\n' + 'import { handlerA } from "./lib";\n' + 'it("x", () => { handlerA(); });\n')}) + link = os.path.join(tempfile.mkdtemp(), "link") + self.addCleanup(shutil.rmtree, os.path.dirname(link), ignore_errors=True) + os.symlink(d, link) + proc, trace, verdicts, err = self._exec(d, repo=link) + self.assertIsNone(err) + self.assertEqual(trace["tests"], {"src/a.test.ts::x": [["src/lib.ts", "handlerA"]]}) + traced, judged = self._keys(trace, verdicts) + self.assertEqual(traced, judged) + + def test_V4_import_failures_and_throwing_beforeAll_are_refusals(self): + d = self._project({"c.test.ts": 'import "./nonexistent";\nimport { it } from "vitest";\nit("n", () => {});\n'}) + proc, trace, verdicts, err = self._exec(d) + self.assertIn("src/c.test.ts failed outside any test", err) + d = self._project({"b.test.ts": ( + 'import { it, describe, beforeAll } from "vitest";\n' + 'describe("bb", () => { beforeAll(() => { throw new Error("boom"); });' + ' it("t1", () => {}); });\n')}) + proc, trace, verdicts, err = self._exec(d) + self.assertIn("src/b.test.ts failed outside any test", err) + + def test_V7_an_all_skipped_run_is_an_empty_success(self): + d = self._project({"d.test.ts": ( + 'import { it } from "vitest";\nit.skip("s", () => {}); it.todo("t");\n')}) + proc, trace, verdicts, err = self._exec(d) + self.assertIsNone(err) + self.assertEqual({v["status"] for v in verdicts.values()}, {"skip"}) + + def test_V7_a_projects_config_is_refused_with_its_cause(self): + d = self._project({"a.test.ts": 'import { it } from "vitest";\nit("x", () => {});\n'}, + config='import { defineConfig } from "vitest/config";\n' + 'export default defineConfig({ test: { projects: ' + '[{ test: { name: "u", include: ["src/**/*.test.ts"] } }] } });\n') + proc, trace, verdicts, err = self._exec(d) + self.assertIn("test.projects", err) + + def test_V12_a_promise_default_export_config_is_awaited(self): + d = self._project({"a.test.ts": 'import { it } from "vitest";\nit("x", () => {});\n'}, + config='export default (async () => ({ test: ' + '{ include: ["src/**/*.test.ts"] } }))();\n') + proc, trace, verdicts, err = self._exec(d) + self.assertIsNone(err) + self.assertEqual(list(trace["tests"]), ["src/a.test.ts::x"]) + + def test_V10_no_bundled_config_lands_in_the_targets_node_modules(self): + d = self._project({"a.test.ts": 'import { it } from "vitest";\nit("x", () => {});\n'}) + self._exec(d) + self.assertFalse(os.path.exists(os.path.join(d, "node_modules", ".vite-temp"))) + + def test_a_config_using_dirname_loads(self): + """`--configLoader runner` would break this; the default bundler defines it.""" + d = self._project({"a.test.ts": 'import { it } from "vitest";\nit("x", () => {});\n'}, + config='import { defineConfig } from "vitest/config";\n' + 'export default defineConfig({ resolve: { alias: ' + '{ "@": __dirname + "/src" } }, test: { include: ' + '["src/**/*.test.ts"] } });\n') + proc, trace, verdicts, err = self._exec(d) + self.assertIsNone(err) + + class PluginShipsTests(unittest.TestCase): def test_the_setup_file_sits_beside_tgtrace(self): plugins = pa.plugin_dir()