diff --git a/CHANGELOG.md b/CHANGELOG.md index 99ac1b7..c7ac810 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,11 @@ below is on `main` or in the branch that introduced this file. ### Added +- **`verify --runner vitest`** (#122): runs only the selected journeys' Vitest tests, + chosen by an `@tg:` tag in a test or describe title (Vitest's `-t`, with whole-tag + boundaries so `@tg:J1` never selects `@tg:J10`). Each tagged test's own V8 trace must + reach the journey's entry symbols, exactly as a `covers(...)` pytest test's must; a + Vitest exit other than 0 with no failing journey is incomplete (exit 3). - **`testgraph.vitest_adapter`** (#122): the pytest adapter's journey attribution for Vitest suites. A TypeScript setup file (`tgtrace-vitest.ts`, shipped beside `tgtrace.py`) records each test's named functions through V8 precise coverage; diff --git a/USAGE.md b/USAGE.md index c1b76c4..ea1b3f4 100644 --- a/USAGE.md +++ b/USAGE.md @@ -205,8 +205,30 @@ do not inherit `setupFiles`, so nothing would be traced) and are refused up fron anonymous functions are not recorded (attribution matches named entry symbols, so this costs nothing there); tests marked `.concurrent` in one file share a coverage window, so their traces can bleed into each other; a failing test next to tests skipped by a -throwing `beforeAll` in the same file is not distinguishable from `it.skip`; `verify` -does not run Vitest suites yet. +throwing `beforeAll` in the same file is not distinguishable from `it.skip`. + +`verify` runs only the selected journeys' Vitest tests with `--runner vitest`. A test +declares its journey with `@tg:` in its title or in a describe title, the same +string a Playwright tag uses: + +```ts +describe("checkout @tg:J4", () => { + it("charges the card", async () => { /* ... */ }); +}); +it("submits a task @tg:J1 @tg:J7", () => { /* ... */ }); +``` + +```bash +python3 -m testgraph.verify --repo . --base origin/main --runner vitest +``` + +Selection is Vitest's `-t`, so untagged tests and tests of other journeys do not run. +Each tagged test is validated as a `covers(...)` test is: its own trace must reach the +journey's entry symbols, or the declaration is reported `unvalidated` and the journey +is not credited. A journey whose tagged tests were all skipped is incomplete, not +uncovered. Narrow the run with a file path after `--`, not with your own `-t`: verify +owns the name filter and refuses a second one. The exit codes are pytest's, plus one rule: a Vitest exit other than 0 +with no failing journey is incomplete (3). ## Checking the Graph Against Reality diff --git a/testgraph/verify.py b/testgraph/verify.py index 958b00f..fd98855 100644 --- a/testgraph/verify.py +++ b/testgraph/verify.py @@ -42,6 +42,7 @@ from . import registry as reg from . import results as res from . import select as sel +from . import vitest_adapter as va #: Rows written here are DECLARED and validated, not inferred. Phase 4's bare #: `"trace"` stays valid and means the older unqualified inference (D2 @@ -58,6 +59,23 @@ #: into an integer. EXIT_INCOMPLETE = 3 +RUNNERS = ("pytest", "vitest") +#: Vitest's name filter, which `verify --runner vitest` owns (review W3). +_VITEST_NAME_FLAGS = ("-t", "--t", "--testNamePattern", "--test-name-pattern") + + +def _vitest_name_flag(args): + """The first arg Vitest's CLI parser (mri) would read as its `t` option, or + None: the spelled-out flags, plus a short cluster holding `t` (`-ut x`, + `-tx`), which mri splits into one option per letter (round 2, R2-2).""" + for a in args: + head = a.split("=", 1)[0] + if head in _VITEST_NAME_FLAGS or ( + len(head) > 2 and head[0] == "-" and head[1:].isalpha() + and head[1:].isascii() and "t" in head[1:]): + return a + return None + def marker_expression(journeys, template=None): """`tg_J1 or tg_J2` — a pytest `-m` expression for these journeys. @@ -150,15 +168,21 @@ def _summary(repo, head, selection, **extra): "incomplete": [], "rows_written": 0, "pytest_exit": None, + "runner": "pytest", } base.update(extra) return base def run(repo, base, head="HEAD", registry_path=None, db_path=None, - pytest_args=(), append=None, _runner=None, _workdir=None): + pytest_args=(), append=None, runner="pytest", _runner=None, _workdir=None): """Select, run the declared tests, validate the declarations, report. + `runner="vitest"` runs the Vitest tests tagged `@tg:` in their + title or a describe title instead (#122); `pytest_args` then go to Vitest. + Validation is the same: each tagged test's own trace must reach the + journey's entry symbols. + Returns (summary, error). A red suite is not an error — it is the signal.""" registry_path = registry_path or reg.resolve_for_repo(repo) if registry_path is None: @@ -174,9 +198,25 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, db_path = db_path or os.path.join(repo, ".codegraph", "codegraph.db") if not os.path.isfile(db_path): return None, f"no codegraph index at {db_path} — run `codegraph init` in {repo}" + if runner not in RUNNERS: + return None, f"unknown runner {runner!r} — one of {', '.join(RUNNERS)}" + if runner == "vitest": + own = _vitest_name_flag(pytest_args) + if own: + # A second -t either replaces the journey filter or crashes Vitest + # with no report; neither is a run this can vouch for (review W3). + return None, (f"verify --runner vitest selects tests by name itself " + f"(-t on the journeys' @tg: tags); drop {own!r} and " + f"narrow the run with a file path instead") + # Every return carries the runner, so a quiet vitest run's --json says so (W2). + quiet = {"runner": runner, **({"runner_exit": None} if runner == "vitest" else {})} plugins = pa.plugin_dir() if plugins is None: return None, pa.PLUGIN_MISSING + if runner == "vitest": + vplugin, node_modules, err = va.locate(repo, _runner) + if err: + return None, err # Strict (the default): an unresolvable journey BLOCKs and the run is # refused. `strict_registry=False` is reserved for history analysis @@ -188,7 +228,7 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, if selection.get("status") == "BLOCKED" or selection.get("blocking"): # The index is not trustworthy. Running tests off it would produce a # verdict about a journey set chosen from a graph the guard refused. - return _summary(repo, head, selection, refused=True), None + return _summary(repo, head, selection, refused=True, **quiet), None # `select` returns journey dicts keyed `id` (with name/rank/confidence); # the plain-string form is what a caller passing a hand-built selection uses. @@ -198,26 +238,57 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, try: if not journeys: - return _summary(repo, head, selection), None - - expr = marker_expression(journeys) - proc, trace, verdicts, err = pa.run_traced( - repo, plugins, ("-m", expr, *pytest_args), - _runner=_runner, _workdir=_workdir) - if err: - return None, err + return _summary(repo, head, selection, **quiet), None + + if runner == "vitest": + expr = va.name_pattern(journeys) + proc, trace, verdicts, err = va.run_traced( + repo, vplugin, node_modules, ("-t", expr, *pytest_args), + _runner=_runner, _workdir=_workdir) + if err: + return None, err + # Vitest reports the tests `-t` filtered out as skipped; only the + # tagged ones it selected are declarations. + trace["declared"] = va.declared(verdicts, list(registry.get("journeys", {})), + journeys) + collected = len(trace["declared"]) + exits = {"runner_exit": getattr(proc, "returncode", None)} + else: + expr = marker_expression(journeys) + proc, trace, verdicts, err = pa.run_traced( + repo, plugins, ("-m", expr, *pytest_args), + _runner=_runner, _workdir=_workdir) + if err: + return None, err + collected = len(verdicts or {}) + exits = {"pytest_exit": getattr(proc, "returncode", None)} # No JUnit XML reads as no verdicts: every selected journey then lands # in `incomplete` or `uncovered`, never in a pass. verdicts = verdicts or {} credited, unvalidated, per_journey = validate(conn, registry, trace, verdicts) - uncovered = [j for j in journeys if j not in per_journey] + skipped_only = set() + if runner == "vitest": + # A tagged test that never ran its body (it.skip, todo, describe.skip) + # has no trace, so validate() passes it over and the journey would + # read "no test carries the tag", which is false (review W4). It is + # a skip; a journey with an unvalidated tag keeps its own message. + claimed = {j for _, j in unvalidated} + skipped_only = { + j for j in journeys + if j not in per_journey and j not in claimed and any( + j in js and verdicts.get(pa._join_key(t), {}).get("status") == "skip" + for t, js in trace["declared"].items())} + uncovered = [j for j in journeys if j not in per_journey and j not in skipped_only] # Selected, has declared tests, but not a pass: the tests ran (or were # collected) and no passing verdict came out. Distinct from `uncovered` # (nothing declared) and from a credited fail (reported as a failure). + # One pass in selection order, as every other list here (R2-7). incomplete = [] for j in journeys: - if j in per_journey and credited.get(j) != "pass" and credited.get(j) != "fail": + if j in skipped_only: + incomplete.append((j, "skip")) + elif j in per_journey and credited.get(j) != "pass" and credited.get(j) != "fail": incomplete.append((j, "skip" if credited.get(j) == "skip" else "no verdict")) written = [] @@ -236,15 +307,16 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, repo, head, selection, journeys=journeys, marker_expression=expr, - collected=len(verdicts), + collected=collected, credited=credited, unvalidated=unvalidated, per_journey={k: sorted(v) for k, v in per_journey.items()}, uncovered=uncovered, incomplete=incomplete, rows_written=len(written), - pytest_exit=getattr(proc, "returncode", None), + runner=runner, commit=sha, + **exits, ), None finally: # Close deterministically (audit M6): a long-lived MCP server calls this per @@ -294,6 +366,10 @@ def exit_code(summary): return EXIT_INCOMPLETE if summary.get("pytest_exit") not in (None, 0, 5): return EXIT_INCOMPLETE + # Vitest's own exit (#122): non-zero with nothing credited `fail` means the + # run went wrong somewhere the report did not name. + if summary.get("runner_exit") not in (None, 0): + return EXIT_INCOMPLETE return EXIT_OK @@ -330,16 +406,37 @@ def render(summary, registry=None): return "\n".join(lines) lines.append(f" journeys selected: {', '.join(summary['journeys'])}") - lines.append(f" pytest -m '{summary['marker_expression']}' " - f"-> {summary['collected']} test(s), exit {summary['pytest_exit']}") + vitest = summary.get("runner") == "vitest" + if vitest: + lines.append(f" vitest -t '{summary['marker_expression']}' " + f"-> {summary['collected']} tagged test(s), exit " + f"{summary.get('runner_exit')}") + else: + lines.append(f" pytest -m '{summary['marker_expression']}' " + f"-> {summary['collected']} test(s), exit {summary['pytest_exit']}") for jid in sorted(summary["credited"], key=reg.journey_sort_key): lines.append(f" {jid} {names.get(jid, ''):<38.38} " f"{summary['credited'][jid]:<5} " f"{len(summary['per_journey'].get(jid, ()))} declared test(s)") - if summary["uncovered"]: + uncovered = summary["uncovered"] + if vitest: + # A journey whose tagged tests ran but never reached its entry is in + # `uncovered` (it is not a pass), but "no test carries the tag" would be + # false for it (round 2, R2-1). + claimed = {j for _, j in summary["unvalidated"]} + ran = [j for j in uncovered if j in claimed] + uncovered = [j for j in uncovered if j not in claimed] + if ran: + lines.append( + f" ! {', '.join(ran)}: tagged tests ran, but none reached the " + f"journey's entry symbols (listed below). This run says NOTHING " + f"about those journeys; it is not a pass.") + if uncovered: lines.append( - f" ! NO JOURNEY-LEVEL TEST: {', '.join(summary['uncovered'])} — " - f"selected, and nothing declared `covers(...)` for them. This run " + f" ! NO JOURNEY-LEVEL TEST: {', '.join(uncovered)} — " + f"selected, and " + f"{'no test in this run carries `@tg:` in its title or a describe title' if vitest else 'nothing declared `covers(...)`'}" + f" for them. This run " f"says NOTHING about those journeys; it is not a pass." ) if summary.get("incomplete"): @@ -375,7 +472,7 @@ def main(argv=None): "3 incomplete — at " "least one selected journey had no passing declared test, or " "the selection was NONE but UNKNOWN.\n" - "Put pytest's own flags after `--`.", + "Put the runner's own flags after `--`.", formatter_class=argparse.RawDescriptionHelpFormatter, ) ap.add_argument("--repo", required=True) @@ -386,7 +483,10 @@ def main(argv=None): ap.add_argument("--dry-run", action="store_true", help="run and report, but write NO ledger rows") ap.add_argument("--json", action="store_true") - ap.add_argument("pytest_args", nargs="*") + ap.add_argument("--runner", choices=RUNNERS, default="pytest", + help="pytest: tests marked covers(...); vitest: tests with " + "@tg: in their title or a describe title") + ap.add_argument("pytest_args", nargs="*", metavar="runner_args") args = ap.parse_args(argv) summary, err = run( @@ -394,6 +494,7 @@ def main(argv=None): registry_path=args.registry, db_path=args.db, pytest_args=tuple(args.pytest_args), append=(lambda row: False) if args.dry_run else None, + runner=args.runner, ) if err: if args.json: diff --git a/testgraph/vitest_adapter.py b/testgraph/vitest_adapter.py index 76694b4..eb95381 100644 --- a/testgraph/vitest_adapter.py +++ b/testgraph/vitest_adapter.py @@ -25,13 +25,14 @@ report of the wrong shape, and a `projects`/workspace config (which does not inherit `setupFiles`) are all refusals. -Deliberately not here yet: selecting only some journeys' tests (verify's job; -next step), and anonymous functions (they need source maps, which only +Selecting only some journeys' tests is `verify --runner vitest`'s job (the +`@tg:` tag helpers below). Deliberately not here yet: anonymous functions (they need source maps, which only `reconcile` would use). See `~/.claude/plans/testgraph-154-then-vitest-adapter.md`. """ import glob import json import os +import re import shutil import subprocess import sys @@ -41,6 +42,7 @@ from . import ledger from . import pytest_adapter as pa from . import registry as reg +from . import results as res PLUGIN_NAME = "tgtrace-vitest.ts" BACKEND = "v8-precise-coverage" @@ -189,11 +191,20 @@ def parse_report(report, root): return None, (f"test {tid!r} never finished (status pending: a worker " f"died or the run was cut short); no verdict was recorded") key = pa._join_key(tid) + status = STATUS.get(case.get("status"), "skip") if key in out: + # Two skips sharing an id cannot mask anything: neither ran, and + # both read the same. `verify` filters with `-t`, so every + # deselected `test.each` repeat lands here (review W1). + if status == "skip" and out[key]["status"] == "skip": + continue return None, (f"two tests share the id {tid!r} (the same title in the " f"same describe, e.g. test.each with a repeated row); the " f"verdicts cannot be told apart, so no verdict was recorded") - out[key] = {"status": STATUS.get(case.get("status"), "skip"), "test_id": tid} + # `full_name` is the string Vitest matches `-t` against: the + # describe chain and the title, never the file (round 2, F1). + out[key] = {"status": status, "test_id": tid, + "full_name": " ".join([*ancestors, title])} return out, None @@ -303,6 +314,74 @@ def execute(repo, plugin, node_modules, tmp, vitest_args=(), runner=None): return proc, trace, verdicts, None +def locate(repo, _runner=None): + """(plugin, node_modules, error): what a traced run needs, or why it can't.""" + plugins = pa.plugin_dir() + plugin = os.path.join(plugins, PLUGIN_NAME) if plugins else None + if not plugin or not os.path.isfile(plugin): + return None, None, (f"cannot find {PLUGIN_NAME}: it ships beside tgtrace.py " + f"(testgraph/_plugin/ installed, harness/plugin/ in a checkout)") + # From the real path: `execute` resolves the repo the same way. + node_modules = find_node_modules(os.path.realpath(repo)) + if node_modules is None and _runner is None: + return None, None, (f"no node_modules with vitest at or above {repo} — install " + f"the repo's dependencies first") + return plugin, node_modules, None + + +def run_traced(repo, plugin, node_modules, vitest_args=(), _runner=None, _workdir=None): + """`execute` in a temp dir that dies with the run: (proc, trace, verdicts, error). + The one traced-run path for this adapter and `verify --runner vitest`.""" + if _workdir is not None: + return execute(repo, plugin, node_modules, _workdir, vitest_args, _runner) + with tempfile.TemporaryDirectory() as tmp: + return execute(repo, plugin, node_modules, tmp, vitest_args, _runner) + + +#: A test declares a journey with this tag in its own title or a describe title, +#: the same string Playwright's tag uses (results.DEFAULT_TAG_CONVENTIONS). +TAG_TEMPLATE = res.DEFAULT_TAG_CONVENTIONS["playwright"] + + +def name_pattern(journeys): + """`(^|\\s)(@tg:J1|@tg:J2)(\\s|$)`: a Vitest `-t` for exactly these journeys. + + Vitest matches `-t` against the test's full name: every describe title + and the test title joined by single spaces, without the file path. Both boundaries are load-bearing: + without the trailing one `@tg:J1` also selects `@tg:J10`, without the + leading one `x@tg:J1` would select J1.""" + alts = "|".join(re.escape(res.format_tag(TAG_TEMPLATE, j)) for j in journeys) + return f"(^|\\s)({alts})(\\s|$)" + + +#: JavaScript's `\\s`, which is what Vitest's `-t` (a `new RegExp`) means by +#: whitespace. Python's `\\s` and str.split() differ from it (U+001C-U+001F and +#: U+0085 are whitespace to Python only, U+FEFF to JavaScript only), and any +#: difference is a test `-t` ran but nothing declared, or the reverse (W5). +_JS_SPACE = "[\t\n\v\f\r \u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]" + + +def declared(verdicts, journey_ids, selected): + """{test_id: [journey, ...]} for the tests `name_pattern(selected)` ran. + + A test declares every registry journey whose tag is a whole token of the + name Vitest matches `-t` against (describe chain and title), with the + same boundaries `name_pattern` uses. Tags the registry does not define are + ignored, as an unknown `covers(...)` id would be. A test whose tags name no + selected journey was filtered out by `-t` and declares nothing here.""" + wanted = set(selected) + tags = [(j, re.compile(f"(?:^|{_JS_SPACE}){re.escape(res.format_tag(TAG_TEMPLATE, j))}" + f"(?={_JS_SPACE}|\\Z)")) + for j in journey_ids] + out = {} + for v in verdicts.values(): + name = v["full_name"] # no fallback: a test id would hide a leading tag + js = [j for j, rx in tags if rx.search(name)] + if wanted.intersection(js): + out[v["test_id"]] = js + return out + + def run(repo, registry_path=None, db_path=None, vitest_args=(), commit=None, append=None, trace_out=None, _runner=None, _workdir=None): """Run the suite, attribute journeys from the trace, write the rows. @@ -310,7 +389,6 @@ def run(repo, registry_path=None, db_path=None, vitest_args=(), commit=None, Returns (summary, error), as `pytest_adapter.run` does: a red suite is a result, a run that cannot be read is a refusal.""" repo = os.path.abspath(repo) - real = os.path.realpath(repo) # `execute` resolves it too; node_modules is looked up from it registry_path = registry_path or reg.resolve_for_repo(repo) if registry_path is None: return None, (f"no journey registry for {reg.repo_name(repo)} — looked in " @@ -320,20 +398,13 @@ def run(repo, registry_path=None, db_path=None, vitest_args=(), commit=None, except (OSError, ValueError) as exc: return None, f"cannot read {registry_path}: {exc}" - plugins = pa.plugin_dir() - plugin = os.path.join(plugins, PLUGIN_NAME) if plugins else None - if not plugin or not os.path.isfile(plugin): - return None, (f"cannot find {PLUGIN_NAME}: it ships beside tgtrace.py " - f"(testgraph/_plugin/ installed, harness/plugin/ in a checkout)") - db_path = db_path or os.path.join(repo, ".codegraph", "codegraph.db") if not os.path.isfile(db_path): return None, f"no codegraph index at {db_path} — run `codegraph init` in {repo}" - node_modules = find_node_modules(real) - if node_modules is None and _runner is None: - return None, (f"no node_modules with vitest at or above {repo} — install " - f"the repo's dependencies first") + plugin, node_modules, err = locate(repo, _runner) + if err: + return None, err if trace_out: bad = pa.trace_out_error(trace_out) @@ -345,11 +416,8 @@ def run(repo, registry_path=None, db_path=None, vitest_args=(), commit=None, return None, (f"cannot resolve {commit or 'HEAD'!r} in {repo} — an " f"unjoinable key would sit in `unasked` forever") - if _workdir is not None: - proc, trace, verdicts, err = execute(repo, plugin, node_modules, _workdir, vitest_args, _runner) - else: - with tempfile.TemporaryDirectory() as tmp: - proc, trace, verdicts, err = execute(repo, plugin, node_modules, tmp, vitest_args, _runner) + proc, trace, verdicts, err = run_traced(repo, plugin, node_modules, vitest_args, + _runner=_runner, _workdir=_workdir) if err: return None, err if trace_out: diff --git a/tests/test_verify_vitest.py b/tests/test_verify_vitest.py new file mode 100644 index 0000000..01f5857 --- /dev/null +++ b/tests/test_verify_vitest.py @@ -0,0 +1,512 @@ +"""`verify --runner vitest` (#122): run only the selected journeys' Vitest +tests, and validate each declaration against its own V8 trace. + +A Vitest test declares its journey with `@tg:J1` in its title or in a describe +title (the same string Playwright's tag uses). Selection is Vitest's `-t`, +which matches the space-joined full name. Validation is `verify.validate`, +unchanged: a tagged test whose trace never reaches the journey's entry symbols +is `unvalidated`, never a pass. + +The hermetic classes fake the Vitest run (report + part files). The real class +runs actual Vitest and skips where Node or a Vitest install is missing. +""" +import io +import contextlib +import json +import os +import re +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import types +import unittest +import unittest.mock + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT_DIR) +from testgraph import verify # noqa: E402 +from testgraph import vitest_adapter as va # noqa: E402 +from test_core import _db_on_disk, _git_repo, _registry_file, build_fixture # noqa: E402 +from test_vitest_adapter import HAVE_VITEST, VITEST_NODE_MODULES, _report # noqa: E402 + +JOURNEYS = { + "J1": {"name": "one", "entries": [{"name": "handler_a", "file": "app/svc.py"}]}, + "J10": {"name": "ten", "entries": [{"name": "leaf", "file": "app/leaf.py"}]}, +} + + +def _verdicts(cases): + verdicts, err = va.parse_report(_report("/r", cases), "/r") + assert err is None, err + return verdicts + + +class TagTests(unittest.TestCase): + """The `-t` pattern and the declaration parser must agree on what a tag is.""" + + def test_the_pattern_matches_whole_tags_only(self): + # Vitest compiles `-t` with `new RegExp`; this syntax is the same in Python. + pat = re.compile(va.name_pattern(["J1"])) + for name in ("src/a.test.ts pays @tg:J1", + "src/a.test.ts flow @tg:J1 pays", + "src/a.test.ts @tg:J1"): + self.assertTrue(pat.search(name), name) + for name in ("src/a.test.ts flow @tg:J10", + "src/a.test.ts flow x@tg:J1", + "src/a.test.ts flow @tg:J1x"): + self.assertFalse(pat.search(name), name) + + def test_the_pattern_is_a_union_of_the_journeys(self): + pat = re.compile(va.name_pattern(["J1", "J10"])) + self.assertTrue(pat.search("f a @tg:J1")) + self.assertTrue(pat.search("f a @tg:J10")) + self.assertFalse(pat.search("f a @tg:J2")) + + def test_declared_reads_test_and_describe_titles(self): + v = _verdicts([ + ("src/a.test.ts", [], "pays @tg:J1", "passed"), + ("src/a.test.ts", ["checkout @tg:J1"], "pays", "passed"), + ("src/a.test.ts", [], "both @tg:J1 @tg:J10", "passed"), + ("src/a.test.ts", [], "only ten @tg:J10", "skipped"), + ("src/a.test.ts", [], "plain", "skipped"), + ("src/a.test.ts", [], "typo @tg:J1x", "skipped"), + ]) + self.assertEqual(va.declared(v, ["J1", "J10"], ["J1"]), { + "src/a.test.ts::pays @tg:J1": ["J1"], + "src/a.test.ts::checkout @tg:J1 > pays": ["J1"], + "src/a.test.ts::both @tg:J1 @tg:J10": ["J1", "J10"], + }) + + def test_declared_ignores_tags_the_registry_does_not_define(self): + v = _verdicts([("src/a.test.ts", [], "x @tg:J1 @tg:J99", "passed")]) + self.assertEqual(va.declared(v, ["J1"], ["J1"]), + {"src/a.test.ts::x @tg:J1 @tg:J99": ["J1"]}) + + def test_W5_declared_splits_on_exactly_javascripts_whitespace(self): + """`-t` is a JS regex, so `\\s` is JS's set: U+FEFF counts, U+001F and + U+0085 do not. Python's str.split() disagrees on all three.""" + v = _verdicts([ + ("src/a.test.ts", [], "bom\ufeff@tg:J1", "passed"), + ("src/a.test.ts", [], "unit\x1f@tg:J1", "skipped"), + ("src/a.test.ts", [], "nel\x85@tg:J1", "skipped"), + ("src/a.test.ts", [], "tab\t@tg:J1", "passed"), + ("src/a.test.ts", [], "nl @tg:J1\n", "passed"), + ]) + self.assertEqual(sorted(va.declared(v, ["J1"], ["J1"])), [ + "src/a.test.ts::bom\ufeff@tg:J1", + "src/a.test.ts::nl @tg:J1\n", + "src/a.test.ts::tab\t@tg:J1", + ]) + + def test_W5_the_trailing_boundary_is_javascripts_too(self): + v = _verdicts([ + ("src/a.test.ts", [], "@tg:J1\ufeffx", "passed"), + ("src/a.test.ts", [], "@tg:J1\x1fx", "skipped"), + ]) + self.assertEqual(list(va.declared(v, ["J1"], ["J1"])), + ["src/a.test.ts::@tg:J1\ufeffx"]) + + def test_F1_the_file_path_is_not_part_of_the_name(self): + """Vitest's `-t` sees the describe chain and the title, never the file + (`-t '^src'` selects nothing), so a path token must not declare.""" + v = _verdicts([("src/x @tg:J1 y.test.ts", [], "plain", "passed"), + ("src/x @tg:J1 y.test.ts", [], "t @tg:J2", "passed")]) + self.assertEqual(va.declared(v, ["J1", "J2"], ["J1", "J2"]), + {"src/x @tg:J1 y.test.ts::t @tg:J2": ["J2"]}) + + def test_R2_4_a_verdict_without_a_full_name_fails_loudly(self): + with self.assertRaises(KeyError): + va.declared({"k": {"status": "pass", "test_id": "f::@tg:J1 > t"}}, ["J1"], ["J1"]) + + def test_W1_two_skipped_tests_sharing_an_id_are_not_a_refusal(self): + verdicts, err = va.parse_report(_report("/r", [ + ("src/o.test.ts", [], "dup", "skipped"), + ("src/o.test.ts", [], "dup", "skipped")]), "/r") + self.assertIsNone(err) + self.assertEqual([v["status"] for v in verdicts.values()], ["skip"]) + + def test_W1_a_duplicate_that_ran_is_still_a_refusal(self): + for pair in (("skipped", "passed"), ("passed", "skipped"), ("failed", "failed")): + with self.subTest(pair): + verdicts, err = va.parse_report(_report("/r", [ + ("src/o.test.ts", [], "dup", pair[0]), + ("src/o.test.ts", [], "dup", pair[1])]), "/r") + self.assertIsNone(verdicts) + self.assertIn("share the id", err) + + +class _VitestVerifyFixture: + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.db = _db_on_disk(self.tmp, build_fixture()) + self.registry = _registry_file(self.tmp, JOURNEYS) + self.repo, run = _git_repo(self.tmp, {"app/svc.py": 22}) + self.repo = os.path.realpath(self.repo) + full = os.path.join(self.repo, "app", "svc.py") + with open(full) as fh: + lines = fh.readlines() + lines[11] = "changed = 1\n" # inside handler_a -> J1 only + with open(full, "w") as fh: + fh.writelines(lines) + run("git", "add", "-A") + run("git", "commit", "-qm", "touch") + self.work = os.path.join(self.tmp, "work") + os.makedirs(self.work) + + def _run(self, cases, tests, returncode=None, write_report=True): + """cases: (ancestors, title, status); tests: {test_id: [[file, name]]}.""" + work, repo = self.work, self.repo + + def runner(): + os.makedirs(os.path.join(work, "parts"), exist_ok=True) + with open(os.path.join(work, "parts", "p.json"), "w") as fh: + json.dump({"root": repo, "backend": va.BACKEND, "file": "x", + "tests": tests}, fh) + if write_report: + with open(os.path.join(work, "results.json"), "w") as fh: + json.dump(_report(repo, [("src/a.test.ts", *c) for c in cases]), fh) + failed = any(c[-1] == "failed" for c in cases) + return types.SimpleNamespace( + returncode=returncode if returncode is not None else int(failed), + stderr="") + + return verify.run(self.repo, "HEAD~1", "HEAD", registry_path=self.registry, + db_path=self.db, runner="vitest", _runner=runner, + _workdir=work, append=lambda r: True) + + +SVC = [["app/svc.py", "handler_a"]] +LEAF = [["app/leaf.py", "leaf"]] + + +class VitestVerifyTests(_VitestVerifyFixture, unittest.TestCase): + + def test_a_tagged_test_that_reaches_the_entry_and_passes_is_credited(self): + summary, err = self._run([((), "pays @tg:J1", "passed")], + {"src/a.test.ts::pays @tg:J1": SVC}) + self.assertIsNone(err) + self.assertEqual(summary["journeys"], ["J1"]) + self.assertEqual(summary["credited"], {"J1": "pass"}) + self.assertEqual(summary["runner"], "vitest") + self.assertEqual(summary["marker_expression"], va.name_pattern(["J1"])) + self.assertEqual(summary["collected"], 1) + self.assertEqual(summary["rows_written"], 1) + self.assertEqual(verify.exit_code(summary), verify.EXIT_OK) + + def test_a_tag_on_the_describe_counts(self): + summary, err = self._run([(("checkout @tg:J1",), "pays", "passed")], + {"src/a.test.ts::checkout @tg:J1 > pays": SVC}) + self.assertIsNone(err) + self.assertEqual(verify.exit_code(summary), verify.EXIT_OK) + + def test_a_failing_tagged_test_fails_the_journey(self): + summary, err = self._run([((), "pays @tg:J1", "failed")], + {"src/a.test.ts::pays @tg:J1": SVC}) + self.assertIsNone(err) + self.assertEqual(summary["credited"], {"J1": "fail"}) + self.assertEqual(verify.exit_code(summary), verify.EXIT_TESTS_FAILED) + + def test_a_tag_the_trace_does_not_support_is_unvalidated_not_a_pass(self): + summary, err = self._run([((), "claims @tg:J1", "passed")], + {"src/a.test.ts::claims @tg:J1": LEAF}) + self.assertIsNone(err) + self.assertEqual(summary["unvalidated"], [("src/a.test.ts::claims @tg:J1", "J1")]) + self.assertEqual(summary["credited"], {}) + self.assertEqual(summary["uncovered"], ["J1"]) + self.assertEqual(summary["incomplete"], []) + self.assertEqual(verify.exit_code(summary), verify.EXIT_INCOMPLETE) + + def test_no_tagged_test_is_uncovered(self): + summary, err = self._run([((), "plain", "skipped")], {}) + self.assertIsNone(err) + self.assertEqual(summary["uncovered"], ["J1"]) + self.assertEqual(summary["collected"], 0) + self.assertEqual(verify.exit_code(summary), verify.EXIT_INCOMPLETE) + + def test_a_J10_tag_does_not_cover_J1(self): + summary, err = self._run([((), "ten @tg:J10", "skipped")], {}) + self.assertIsNone(err) + self.assertEqual(summary["uncovered"], ["J1"]) + + def test_W4_a_tagged_test_that_was_skipped_is_incomplete_not_uncovered(self): + summary, err = self._run([((), "pays @tg:J1", "skipped")], {}) + self.assertIsNone(err) + self.assertEqual(summary["uncovered"], []) + self.assertEqual(summary["incomplete"], [("J1", "skip")]) + self.assertEqual(verify.exit_code(summary), verify.EXIT_INCOMPLETE) + self.assertNotIn("NO JOURNEY-LEVEL TEST", verify.render(summary)) + + def test_W4_an_unvalidated_tag_keeps_the_journey_uncovered_beside_a_skip(self): + summary, err = self._run([((), "claims @tg:J1", "passed"), + ((), "later @tg:J1", "skipped")], + {"src/a.test.ts::claims @tg:J1": LEAF}) + self.assertIsNone(err) + self.assertEqual(summary["uncovered"], ["J1"]) + self.assertEqual(summary["incomplete"], []) + + def test_R2_1_an_unvalidated_journey_is_not_called_untagged(self): + summary, _ = self._run([((), "claims @tg:J1", "passed")], + {"src/a.test.ts::claims @tg:J1": LEAF}) + text = verify.render(summary) + self.assertNotIn("no test in this run carries", text) + self.assertIn("J1: tagged tests ran, but none reached the journey's entry", text) + self.assertEqual(verify.exit_code(summary), verify.EXIT_INCOMPLETE) + + def test_R2_7_incomplete_follows_selection_order(self): + real = verify.sel.select + + def two(*a, **kw): + out = real(*a, **kw) + out["journeys"] = ["J10", "J1"] + return out + with unittest.mock.patch.object(verify.sel, "select", two): + summary, err = self._run([((), "ten @tg:J10", "skipped"), + ((), "one @tg:J1", "skipped")], + {"src/a.test.ts::one @tg:J1": SVC}) + self.assertIsNone(err) + self.assertEqual(summary["incomplete"], [("J10", "skip"), ("J1", "skip")]) + + def test_W4_the_uncovered_message_speaks_of_this_run(self): + summary, _ = self._run([((), "plain", "skipped")], {}) + self.assertIn("no test in this run carries `@tg:`", verify.render(summary)) + + def test_W1_an_unrelated_skipped_duplicate_does_not_block_verify(self): + summary, err = self._run([((), "pays @tg:J1", "passed"), + ((), "dup", "skipped"), ((), "dup", "skipped")], + {"src/a.test.ts::pays @tg:J1": SVC}) + self.assertIsNone(err) + self.assertEqual(verify.exit_code(summary), verify.EXIT_OK) + + def test_W2_a_quiet_vitest_run_says_vitest(self): + def runner(): + raise AssertionError("nothing is selected, so vitest must not run") + summary, err = verify.run(self.repo, "HEAD", "HEAD", registry_path=self.registry, + db_path=self.db, runner="vitest", _runner=runner, + _workdir=self.work, append=lambda r: True) + self.assertIsNone(err) + self.assertEqual(summary["journeys"], []) + self.assertEqual(summary["runner"], "vitest") + self.assertIn("runner_exit", summary) + self.assertIsNone(summary["runner_exit"]) + + def test_W3_a_name_filter_of_your_own_is_refused_up_front(self): + def runner(): + raise AssertionError("refused before vitest runs") + for args in (("-t", "smoke"), ("-t=smoke",), ("--testNamePattern", "smoke"), + ("--testNamePattern=smoke",), ("--test-name-pattern=smoke",), + # R2-2: mri also reads these as the `t` option + ("--t", "smoke"), ("--t=smoke",), ("-ut", "smoke"), ("-tx",)): + with self.subTest(args): + summary, err = verify.run(self.repo, "HEAD~1", "HEAD", + registry_path=self.registry, db_path=self.db, + runner="vitest", pytest_args=args, + _runner=runner, _workdir=self.work, + append=lambda r: True) + self.assertIsNone(summary) + self.assertIn("selects tests by name itself", err) + + def test_W3_other_vitest_flags_pass_through(self): + for args in ((), ("--testTimeout=5000",), ("--testTimeout", "5000"), ("-u",), + ("--typecheck",), ("--teardownTimeout=1",), ("src/a.test.ts",), + ("-r", "src"), ("-T", "x")): + with self.subTest(args): + self.assertIsNone(verify._vitest_name_flag(args)) + + def test_a_nonzero_vitest_exit_downgrades_a_pass(self): + summary, err = self._run([((), "pays @tg:J1", "passed")], + {"src/a.test.ts::pays @tg:J1": SVC}, returncode=1) + self.assertIsNone(err) + self.assertEqual(summary["runner_exit"], 1) + self.assertEqual(verify.exit_code(summary), verify.EXIT_INCOMPLETE) + + def test_a_broken_run_is_a_refusal(self): + summary, err = self._run([((), "pays @tg:J1", "passed")], {}, write_report=False) + self.assertIsNone(summary) + self.assertIn("wrote no JSON report", err) + + def test_the_run_selects_with_dash_t(self): + calls = [] + real_run = subprocess.run + + def fake_run(cmd, **kw): + # `va.subprocess` is the shared module: let git through. + if not str(cmd[0]).endswith("vitest"): + return real_run(cmd, **kw) + calls.append(cmd) + return types.SimpleNamespace(returncode=0, stderr="") + nm = os.path.join(self.tmp, "nm") + os.makedirs(nm) + with unittest.mock.patch.object(va.subprocess, "run", fake_run), \ + unittest.mock.patch.object(va, "find_node_modules", lambda repo: nm): + verify.run(self.repo, "HEAD~1", "HEAD", registry_path=self.registry, + db_path=self.db, runner="vitest", _workdir=self.work, + append=lambda r: True) + cmd = calls[0] + self.assertEqual(cmd[cmd.index("-t") + 1], va.name_pattern(["J1"])) + + def test_no_vitest_install_is_a_refusal(self): + with unittest.mock.patch.object(va, "find_node_modules", lambda repo: None): + summary, err = verify.run(self.repo, "HEAD~1", "HEAD", + registry_path=self.registry, db_path=self.db, + runner="vitest", append=lambda r: True) + self.assertIsNone(summary) + self.assertIn("no node_modules with vitest", err) + + def test_render_names_the_runner_and_the_tag(self): + summary, _ = self._run([((), "plain", "skipped")], {}) + text = verify.render(summary) + self.assertIn(f"vitest -t '{va.name_pattern(['J1'])}'", text) + self.assertIn("@tg:", text) + self.assertNotIn("covers(", text) + + def test_pytest_render_is_unchanged(self): + summary = verify._summary(self.repo, "HEAD", {"base": "a", "head": "b"}, + journeys=["J1"], marker_expression="tg_J1", + uncovered=["J1"], pytest_exit=5) + text = verify.render(summary) + self.assertIn("pytest -m 'tg_J1'", text) + self.assertIn("covers(...)", text) + + def test_the_cli_takes_runner_vitest(self): + seen = {} + + def fake_run(*a, **kw): + seen.update(kw) + return None, "stop here" + out = io.StringIO() + with unittest.mock.patch.object(verify, "run", fake_run), \ + contextlib.redirect_stdout(out): + code = verify.main(["--repo", self.repo, "--base", "HEAD~1", + "--runner", "vitest", "--", "--bail=1"]) + self.assertEqual(code, verify.EXIT_REFUSED) + self.assertEqual(seen["runner"], "vitest") + self.assertEqual(seen["pytest_args"], ("--bail=1",)) + + +LIB = "export function handlerA() {\n return 1;\n}\n" +CONFIG = ('import { defineConfig } from "vitest/config";\n' + 'export default defineConfig({ test: { include: ["src/**/*.test.ts"] } });\n') +HEADER = ('import { it, describe, expect } from "vitest";\n' + 'import { handlerA } from "./lib";\n') + + +@unittest.skipUnless(HAVE_VITEST, "needs node and a vitest install " + "(TG_VITEST_NODE_MODULES, default: signedintake's node_modules)") +class RealVitestVerifyTests(unittest.TestCase): + """verify end to end with real Vitest: one journey (J1 -> handlerA in + src/lib.ts), one commit touching it, a test file per case.""" + + @classmethod + def setUpClass(cls): + cls.tmp = os.path.realpath(tempfile.mkdtemp(prefix="tg-vv-")) + src = build_fixture() + src.execute("INSERT INTO nodes VALUES (?,?,?,?,?,?,?)", + ("function:handlerA", "function", "handlerA", "handlerA", + "src/lib.ts", 1, 3)) + src.commit() # backup() waits forever on an open write transaction + cls.db = _db_on_disk(cls.tmp, src) + src.close() + cls.registry = _registry_file(cls.tmp, { + "J1": {"name": "one", "entries": [{"name": "handlerA", "file": "src/lib.ts"}]}, + "J10": {"name": "ten", "entries": [{"name": "leaf", "file": "app/leaf.py"}]}, + }) + cls.repo = os.path.join(cls.tmp, "repo") + os.makedirs(os.path.join(cls.repo, "src")) + nm = os.path.join(cls.repo, "node_modules") + os.mkdir(nm) + for entry in os.listdir(VITEST_NODE_MODULES): + if entry not in (".vite", ".vite-temp", ".cache"): + os.symlink(os.path.join(VITEST_NODE_MODULES, entry), os.path.join(nm, entry)) + for rel, text in (("src/lib.ts", LIB), ("vitest.config.ts", CONFIG), + (".gitignore", "node_modules/\n")): + with open(os.path.join(cls.repo, rel), "w") as fh: + fh.write(text) + + def git(*args): + subprocess.run(["git", "-C", cls.repo, *args], check=True, capture_output=True) + git("init", "-q") + git("config", "user.email", "t@t") + git("config", "user.name", "t") + git("add", "-A") + git("commit", "-qm", "base") + with open(os.path.join(cls.repo, "src", "lib.ts"), "w") as fh: + fh.write(LIB.replace("return 1;", "return 1; // touched")) + git("add", "-A") + git("commit", "-qm", "touch handlerA") + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp, ignore_errors=True) + + def _verify(self, body, name=None): + for old in os.listdir(os.path.join(self.repo, "src")): + if old.endswith(".test.ts"): + os.remove(os.path.join(self.repo, "src", old)) + self._n = getattr(self, "_n", 0) + 1 + name = name or f"case{self._n}.test.ts" + with open(os.path.join(self.repo, "src", name), "w") as fh: + fh.write(HEADER + body) + summary, err = verify.run(self.repo, "HEAD~1", "HEAD", + registry_path=self.registry, db_path=self.db, + runner="vitest", append=lambda r: True) + self.assertIsNone(err) + self.assertEqual(summary["journeys"], ["J1"]) + return summary + + def test_only_the_selected_journeys_tests_run(self): + # The untagged test would fail if it ran. + s = self._verify('it("pays @tg:J1", () => { expect(handlerA()).toBe(1); });\n' + 'it("other", () => { expect(1).toBe(2); });\n') + self.assertEqual(s["credited"], {"J1": "pass"}) + self.assertEqual(s["collected"], 1) + self.assertEqual(verify.exit_code(s), verify.EXIT_OK) + + def test_a_describe_tag_selects_its_tests(self): + s = self._verify('describe("flow @tg:J1", () => {\n' + ' it("pays", () => { expect(handlerA()).toBe(1); });\n});\n') + self.assertEqual(verify.exit_code(s), verify.EXIT_OK) + + def test_a_failing_tagged_test_exits_1(self): + s = self._verify('it("pays @tg:J1", () => { expect(handlerA()).toBe(2); });\n') + self.assertEqual(verify.exit_code(s), verify.EXIT_TESTS_FAILED) + + def test_a_tag_that_never_reaches_the_entry_is_unvalidated(self): + s = self._verify('it("claims @tg:J1", () => { expect(1).toBe(1); });\n') + self.assertEqual(len(s["unvalidated"]), 1) + self.assertEqual(verify.exit_code(s), verify.EXIT_INCOMPLETE) + + def test_W1_an_untagged_duplicate_elsewhere_does_not_block(self): + s = self._verify('it("pays @tg:J1", () => { expect(handlerA()).toBe(1); });\n' + 'it.each([1, 1])("dup", () => {});\n') + self.assertEqual(verify.exit_code(s), verify.EXIT_OK) + + def test_W4_a_skipped_tagged_test_is_incomplete(self): + s = self._verify('it.skip("pays @tg:J1", () => { expect(handlerA()).toBe(1); });\n') + self.assertEqual(s["incomplete"], [("J1", "skip")]) + self.assertEqual(s["uncovered"], []) + + def test_W5_selection_and_declaration_agree_on_odd_whitespace(self): + # U+FEFF is JS whitespace: -t runs it, so it must be declared and credited. + s = self._verify('it("pays\\uFEFF@tg:J1", () => { expect(handlerA()).toBe(1); });\n') + self.assertEqual((s["collected"], s["credited"]), (1, {"J1": "pass"})) + # U+001F is not: -t skips it, so it must not be counted as collected. + s = self._verify('it("pays\\u001F@tg:J1", () => { expect(handlerA()).toBe(1); });\n') + self.assertEqual((s["collected"], s["uncovered"]), (0, ["J1"])) + + def test_F1_a_tag_in_the_file_path_neither_selects_nor_declares(self): + s = self._verify('it("plain", () => { expect(handlerA()).toBe(1); });\n', + name="x @tg:J1 y.test.ts") + self.assertEqual((s["collected"], s["uncovered"], s["credited"]), (0, ["J1"], {})) + self.assertEqual(verify.exit_code(s), verify.EXIT_INCOMPLETE) + + def test_a_J10_tag_is_not_run_for_J1(self): + s = self._verify('it("ten @tg:J10", () => { expect(1).toBe(2); });\n') + self.assertEqual(s["uncovered"], ["J1"]) + self.assertEqual(verify.exit_code(s), verify.EXIT_INCOMPLETE) + + +if __name__ == "__main__": + unittest.main()