From 2ff8764512a46a78e46b8ad5c11db2cd1a6f9bf9 Mon Sep 17 00:00:00 2001 From: Eric Minish Date: Sun, 15 Nov 2026 15:05:49 -0500 Subject: [PATCH] feat: verify --runner playwright runs the selected journeys' tagged browser tests (#122) The registry's `e2e` block names where the Playwright suite lives ({"runner": "playwright", "cwd": ..., "config": ...}). verify selects with a case-sensitive `--grep '/(@tg:J1|...)(\s|$)/'`, reads the JSON report, and credits each selected journey. Verdicts are DECLARED, not trace-validated (edge_provenance=e2e-declared; backend tracing is #157), and the summary says so. - fail (1): a tagged test failed, timed out or crashed, also when -x or maxFailures stopped the run after it. - incomplete (3): flaky (passed only on a retry), skip-only, a test.fail() test that failed as declared, no tagged test (uncovered; --pass-with-no-tests keeps Playwright's "No tests found" from reading as an error), or a non-zero runner exit with nothing failed. - refused (2): a test outside the selected journeys failed (setup/environment), other global report errors, no or unreadable report, a malformed e2e block, a cwd or config that resolves outside the repo, or a user --grep, --reporter or --config after `--` (verify owns them). `--e2e` from the checkpoint became `--runner playwright`, sharing the runner_exit rule and quiet-run summary keys with the Vitest path. --- CHANGELOG.md | 7 + TECHNICAL.md | 3 + USAGE.md | 46 ++++ journeys/honeyslate.json | 1 + testgraph/e2e.py | 276 +++++++++++++++++++++++ testgraph/verify.py | 93 +++++++- tests/test_e2e.py | 472 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 889 insertions(+), 9 deletions(-) create mode 100644 testgraph/e2e.py create mode 100644 tests/test_e2e.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c7ac810..edc21d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,13 @@ below is on `main` or in the branch that introduced this file. ### Added +- **`verify --runner playwright`** (#122): runs the Playwright tests tagged `@tg:` + for the selected journeys, from the registry's `e2e` block (`cwd`, `config`). + Verdicts are declared, not trace-validated (`edge_provenance=e2e-declared`; backend + tracing is #157). A flaky pass or an all-skipped journey is incomplete (exit 3). A + failing untagged setup test, global report errors or a missing report are a + refusal (exit 2). Checked on honeyslate's real-backend J1 test: green credits J1, + and a broken `create_task` exits 1. - **`verify --runner vitest`** (#122): runs only the selected journeys' Vitest tests, chosen by an `@tg:` tag in a test or describe title (Vitest's `-t`, with whole-tag boundaries so `@tg:J1` never selects `@tg:J10`). Each tagged test's own V8 trace must diff --git a/TECHNICAL.md b/TECHNICAL.md index 2a19e9f..a0bd618 100644 --- a/TECHNICAL.md +++ b/TECHNICAL.md @@ -80,6 +80,9 @@ leaving edges wrong) — only a full `codegraph index` did. Three checks: line is skipped rather than fatal. - `testgraph/record.py` — the `outcome` writer and the `--summary` / `--export-kb` readers. Refuses a journey id the registry does not know. +- `testgraph/e2e.py` — `verify --runner playwright` (#122): the `--grep` for the + selected journeys' `@tg:` tags, Playwright's JSON report into per-journey + verdicts, `e2e-declared` rows. No trace yet, so nothing validates a pass (#157). - `journeys/honeyslate.json` — 8 journeys (submit, browse, edit, reschedule, comments, auth, gcal sync, scheduler) + `spot_checks` floors for the guard. - `harness/plugin/tgtrace.py` — pytest plugin, loaded into the TARGET's diff --git a/USAGE.md b/USAGE.md index ea1b3f4..2778560 100644 --- a/USAGE.md +++ b/USAGE.md @@ -309,6 +309,52 @@ Exit codes, and the third one is the point: A selected journey with nothing to run is not a pass, and sharing an exit code with success is how that would be read as one. +### Browser journeys (Playwright) + +`--runner playwright` runs the Playwright tests tagged for the selected journeys. +The registry says where they live: + +```json +"e2e": {"runner": "playwright", "cwd": "frontend", "config": "playwright.real.config.js"} +``` + +A test declares its journey with Playwright's own tag, `@tg:`, either as an +option or in its title: + +```js +test("a submitter adds a task", { tag: "@tg:J1" }, async ({ page }) => { /* ... */ }); +``` + +```bash +python3 -m testgraph.verify --repo . --base origin/main --runner playwright +``` + +Selection is `--grep '/(@tg:J1|@tg:J2)(\s|$)/'`. The trailing boundary keeps +`@tg:J1` from selecting `@tg:J10`. No leading boundary is needed, because Playwright +reads any `@…` run in a title as a tag, `x@tg:J1` included. The `/…/` form makes the +match case-sensitive; a bare `--grep` would also run `@tg:j1`. + +verify owns `--grep`, `--reporter` and `--config`, and refuses them after `--`. Narrow +the run with a spec path or `--project` instead. The registry's `cwd` and `config` +must stay inside the repo. + +**These verdicts are declared, not trace-validated.** Nothing traces the backend +during a browser run yet, so a tagged test that mocks the app's own API passes just +the same. Rows carry `edge_provenance=e2e-declared`, and the summary says so (backend +tracing is #157). Point the config at a real backend, not the mocked smoke suite. + +Verdicts per selected journey: +- **fail (exit 1):** any tagged test failed, timed out or crashed. +- **fail (exit 1)** also covers a run that `-x` or `maxFailures` stopped early after + a tagged test failed. +- **incomplete (exit 3):** a tagged test passed only on a retry (flaky), every + tagged test was skipped, or a `test.fail()` test failed as it declared. +- **uncovered (exit 3):** no test carries the journey's tag. +- **refused (exit 2):** a test outside the selected journeys failed, tagged or not + (the login or seed setup project, meaning a broken environment, not a verdict + about any journey). Also refused: any other global error in the report, no + report at all, or a malformed `e2e` block. + ## What to Do When Something Breaks - **"STATUS: BLOCKED — index not trustworthy"** — the underlying code map is diff --git a/journeys/honeyslate.json b/journeys/honeyslate.json index e1ae7f1..fcc1ca5 100644 --- a/journeys/honeyslate.json +++ b/journeys/honeyslate.json @@ -7,6 +7,7 @@ "frontend: src/routes/** (+layout.js, +layout.svelte, gantt/+page.svelte, login/+page.svelte), service-worker.js, playwright.config.js, svelte.config.js, vite.config.js — 8 files. TECHNICAL.md states no journey has a frontend entry point, so a frontend change is visible only where it reaches a backend entry." ], "approved": true, + "e2e": {"runner": "playwright", "cwd": "frontend", "config": "playwright.real.config.js"}, "journeys": { "J1": { "name": "submit task", diff --git a/testgraph/e2e.py b/testgraph/e2e.py new file mode 100644 index 0000000..d3097a0 --- /dev/null +++ b/testgraph/e2e.py @@ -0,0 +1,276 @@ +"""Run the Playwright tests declared for the journeys a diff endangers (#122). + +The e2e counterpart of `verify.run`. A registry opts in with + + "e2e": {"runner": "playwright", "cwd": "frontend", + "config": "playwright.real.config.js"} + +and a test declares its journey with Playwright's tag, in the project's one +convention (`results.DEFAULT_TAG_CONVENTIONS["playwright"]`): + + test("a submitter adds a task", { tag: "@tg:J1" }, async ({ page }) => ...) + +What this does NOT yet know is whether a passing test actually reached the +journey's entry symbols. Nothing traces the backend during a browser run, so a +test that mocks the app's own API passes just the same. Every verdict here is +therefore DECLARED, not trace-validated, and says so: provenance +`e2e-declared`, and the rendered summary names it. Backend tracing is #157. + +Verdicts, per selected journey, from Playwright's JSON report: + - any tagged test unexpected (failed, timed out, crashed) -> fail (exit 1) + - any tagged test flaky (passed only on a retry) -> incomplete + - a tagged test passed -> pass + - every tagged test skipped -> incomplete + - no tagged test at all -> uncovered (exit 3) +An untagged test that fails (the login/seed setup project) is a broken +environment, not a verdict about any journey: the run is refused (exit 2), as is +a report with global errors or no report at all. +""" +import json +import os +import re +import subprocess +import tempfile + +from . import ledger +from . import results as res + +PROVENANCE = "e2e-declared" +RUNNERS = ("playwright",) + + +def tag(journey): + return res.format_tag(res.DEFAULT_TAG_CONVENTIONS["playwright"], journey) + + +def grep_expression(journeys): + """`(@tg:J1|@tg:J2)(\\s|$)` — Playwright `--grep` for exactly these journeys. + + The trailing boundary is load-bearing: a bare `@tg:J1` also matches + `@tg:J10`. Playwright matches the regex against the title plus its tags.""" + alts = "|".join(re.escape(tag(j)) for j in journeys) + return f"({alts})(\\s|$)" + + +def grep_argument(journeys): + """`grep_expression` as a `/.../` literal. Playwright compiles a bare + `--grep` with flags "gi" (forceRegExp, lib/util.js), so `@tg:j1` and + `@TG:J1` would run while the report's tags cannot be read as J1, and a + failure could vanish (review P5). A literal with no flags is case-sensitive.""" + return f"/{grep_expression(journeys)}/" + + +#: Flags whose value verify sets itself: a second one would replace the journey +#: filter, the JSON reporter or the config (review P4). +OWNED_FLAGS = ("--grep", "-g", "--grep-invert", "--reporter", "--config", "-c") + + +def owned_flag(args): + """The first arg that would override what verify sets, or None. Short flags + can be clustered or carry their value (`-xg smoke`, `-gsmoke`).""" + for a in args: + if a.split("=", 1)[0] in OWNED_FLAGS: + return a + if not a.startswith("--") and re.match(r"-[A-Za-z]*[gc]", a): + return a + return None + + +def check_config(repo, e2e): + """(cwd, config, error) for the registry's `e2e` block. Every bad shape is a + refusal: a traceback would exit 1, the "tests failed" code (review P1), and + a cwd or config outside the repo would run someone else's tests (P3).""" + if not isinstance(e2e, dict): + return None, None, (f"registry e2e must be an object like " + f'{{"runner": "playwright", "cwd": "frontend"}}, got {e2e!r}') + if e2e.get("runner") not in RUNNERS: + return None, None, (f"registry e2e.runner must be one of {RUNNERS}, " + f"got {e2e.get('runner')!r}") + cwd = e2e.get("cwd") + if not isinstance(cwd, str) or not cwd or os.path.isabs(cwd): + return None, None, (f"registry e2e.cwd must be a path relative to the repo " + f'("." for its root), got {cwd!r}') + root = os.path.realpath(repo) + + def inside(path): + return os.path.commonpath([root, os.path.realpath(path)]) == root + full = os.path.normpath(os.path.join(repo, cwd)) + if not inside(full): + return None, None, f"registry e2e.cwd {cwd!r} resolves outside the repo" + if not os.path.isdir(full): + return None, None, f"registry e2e.cwd does not exist: {full}" + config = e2e.get("config") + if config is not None: + if (not isinstance(config, str) or not config or config.startswith("-") + or os.path.isabs(config)): + return None, None, (f"registry e2e.config must be a path relative to " + f"e2e.cwd, got {config!r}") + if not inside(os.path.join(full, config)): + return None, None, f"registry e2e.config {config!r} resolves outside the repo" + return full, config, None + + +def _message(err): + return str(err.get("message", err) if isinstance(err, dict) else err) + + +def _specs(suite): + for spec in suite.get("specs", []): + yield spec + for child in suite.get("suites", []): + yield from _specs(child) + + +def parse_report(report, journeys): + """Playwright's JSON report -> (per_journey, untagged_failures, unknown_tags). + + `per_journey`: {jid: [(title, status), ...]} for the selected journeys, + where status is Playwright's test status (`expected`, `unexpected`, + `flaky`, `skipped`) refined so an `expected` whose last attempt did not + pass (a `test.fail()` expectation) is not read as a pass.""" + prefix = tag("")[1:] # "tg:" — JSON tags carry no leading '@' + wanted = set(journeys) + per_journey = {j: [] for j in journeys} + untagged_failures, unknown = [], set() + for suite in report.get("suites", []): + for spec in _specs(suite): + ids = [t[len(prefix):] for t in spec.get("tags", []) if t.startswith(prefix)] + for test in spec.get("tests", []): + status = test.get("status") + results = test.get("results") or [] + if status == "expected" and (not results + or results[-1].get("status") != "passed"): + status = "expected-not-passed" + if not wanted.intersection(ids): + # Untagged, or tagged only for journeys not selected (a + # setup project, a mis-cased tag): it is not a verdict + # about any selected journey, and a failure here means the + # environment is broken (review P5). Never dropped. + if status in ("unexpected", "flaky"): + untagged_failures.append(spec.get("title", "?")) + unknown.update(j for j in ids if j not in wanted) + continue + for jid in ids: + if jid in wanted: + per_journey[jid].append((spec.get("title", "?"), status)) + else: + unknown.add(jid) + return per_journey, untagged_failures, sorted(unknown) + + +def judge(per_journey): + """(credited, incomplete, uncovered) in verify's summary vocabulary.""" + credited, incomplete, uncovered = {}, [], [] + for jid, tests in per_journey.items(): + statuses = {s for _title, s in tests} + if not tests: + uncovered.append(jid) + elif "unexpected" in statuses: + credited[jid] = "fail" + elif "flaky" in statuses: + incomplete.append((jid, "flaky")) + elif "expected" in statuses: + credited[jid] = "pass" + elif statuses == {"expected-not-passed"} or ( + "expected-not-passed" in statuses and "skipped" not in statuses): + # test.fail(): it ran, and failed as declared. Not a pass, and not + # a skip either (review P7). + incomplete.append((jid, "expected-fail")) + else: + credited[jid] = "skip" + incomplete.append((jid, "skip")) + return credited, incomplete, uncovered + + +def run(repo, journeys, config, extra_args=(), _runner=None): + """Run Playwright for `journeys`. Returns (fields, error) — `fields` are the + summary keys verify merges in; `error` is a refusal message.""" + cwd, cfg, err = check_config(repo, config) + if err: + return None, err + own = owned_flag(extra_args) + if own: + return None, owned_refusal(own) + expr = grep_argument(journeys) + with tempfile.TemporaryDirectory() as tmp: + report_path = os.path.join(tmp, "report.json") + # --pass-with-no-tests: a selection nobody tagged is `uncovered`, not + # Playwright's "No tests found" run error (review P2). + cmd = ["npx", "--no-install", "playwright", "test", "--grep", expr, + "--reporter=json", "--pass-with-no-tests"] + if cfg: + cmd[4:4] = ["-c", cfg] + cmd += list(extra_args) + # _FILE wins over _NAME in Playwright, so an inherited one would send + # the report elsewhere (and overwrite the user's file): set both (P4). + env = dict(os.environ, PLAYWRIGHT_JSON_OUTPUT_NAME=report_path, + PLAYWRIGHT_JSON_OUTPUT_FILE=report_path) + runner = _runner or (lambda c, d, e: subprocess.run( + c, cwd=d, env=e, capture_output=True, text=True)) + try: + proc = runner(cmd, cwd, env) + except OSError as exc: + return None, f"could not start Playwright ({exc}) — is Node installed?" + tail = (getattr(proc, "stderr", "") or "").strip()[-400:] + if not os.path.isfile(report_path): + return None, (f"Playwright exited {getattr(proc, 'returncode', '?')} " + f"and wrote no JSON report — no verdict was recorded" + + (f"\n stderr (tail): {tail}" if tail else "")) + try: + with open(report_path) as fh: + report = json.load(fh) + except (OSError, ValueError) as exc: + return None, f"the Playwright report is unreadable ({exc}) — no verdict was recorded" + if not isinstance(report, dict): + return None, "the Playwright report is not a JSON object — no verdict was recorded" + errors = report.get("errors") or [] + # maxFailures / -x stop the run with a global error, but a tagged test + # that already failed is red whatever came after it (review P6). Any + # other run error, or a stop with no tagged failure, is a refusal. + stopped = bool(errors) and all("maximum allowed failures" in _message(e) + for e in errors) + if errors and not stopped: + return None, f"Playwright reported a run error: {_message(errors[0])[:300]}" + per_journey, untagged_failures, unknown = parse_report(report, journeys) + if untagged_failures: + return None, (f"an untagged Playwright test failed, or one tagged only for " + f"journeys not selected ({', '.join(untagged_failures[:3])}) — " + f"setup or environment is broken, so no journey verdict was " + f"recorded") + credited, incomplete, uncovered = judge(per_journey) + if stopped and "fail" not in credited.values(): + return None, f"Playwright reported a run error: {_message(errors[0])[:300]}" + return { + "runner": "playwright", + "marker_expression": expr, + "collected": sum(len(t) for t in per_journey.values()), + "credited": credited, + "per_journey": {j: [title for title, _s in t] for j, t in per_journey.items() if t}, + "uncovered": uncovered, + "incomplete": incomplete, + "unknown_tags": unknown, + "runner_exit": getattr(proc, "returncode", None), + }, None + + +def owned_refusal(flag): + return (f"verify --runner playwright owns --grep, --reporter and --config " + f"(it selects the journeys' @tg: tags and reads the JSON report); " + f"drop {flag!r} — narrow the run with a spec path or --project instead") + + +def write_rows(repo_name, sha, fields, append=None): + append = append or ledger.append + written = 0 + for jid, verdict in sorted(fields["credited"].items()): + if verdict not in ("pass", "fail"): + continue + row = ledger.outcome_row( + repo_name, sha, jid, verdict, + note=(f"{len(fields['per_journey'].get(jid, ()))} tagged Playwright " + f"test(s) — declared, not trace-validated"), + edge_provenance=PROVENANCE, + ) + if append(row): + written += 1 + return written diff --git a/testgraph/verify.py b/testgraph/verify.py index fd98855..9943667 100644 --- a/testgraph/verify.py +++ b/testgraph/verify.py @@ -37,6 +37,7 @@ import sys from . import db as dbmod +from . import e2e from . import ledger from . import pytest_adapter as pa from . import registry as reg @@ -59,7 +60,13 @@ #: into an integer. EXIT_INCOMPLETE = 3 -RUNNERS = ("pytest", "vitest") +RUNNERS = ("pytest", "vitest", "playwright") +#: What an uncovered journey lacked, per runner, in the rendered summary. +_UNCOVERED_WHY = { + "pytest": "nothing declared `covers(...)`", + "vitest": "no test in this run carries `@tg:` in its title or a describe title", + "playwright": "no Playwright test in this run carries a `@tg:` tag", +} #: Vitest's name filter, which `verify --runner vitest` owns (review W3). _VITEST_NAME_FLAGS = ("-t", "--t", "--testNamePattern", "--test-name-pattern") @@ -181,9 +188,13 @@ def run(repo, base, head="HEAD", registry_path=None, db_path=None, `runner="vitest"` runs the Vitest tests tagged `@tg:` in their title or a describe title instead (#122); `pytest_args` then go to Vitest. Validation is the same: each tagged test's own trace must reach the - journey's entry symbols. + journey's entry symbols. `runner="playwright"` is `run_e2e`: the registry's + e2e block, declared verdicts with no trace yet. Returns (summary, error). A red suite is not an error — it is the signal.""" + if runner == "playwright": + return run_e2e(repo, base, head, registry_path=registry_path, db_path=db_path, + runner_args=pytest_args, append=append, _runner=_runner) registry_path = registry_path or reg.resolve_for_repo(repo) if registry_path is None: return None, (f"no journey registry for {reg.repo_name(repo)} — looked " @@ -328,6 +339,53 @@ def _unknown_because(selection): return selection.get("unknown_because") or sel.none_is_unknown(selection) +def run_e2e(repo, base, head="HEAD", registry_path=None, db_path=None, + runner_args=(), append=None, _runner=None): + """`run`, for the registry's e2e runner (Playwright) instead of pytest (#122). + + Same selection and the same refusals; the journeys' tagged browser tests run + instead of `covers(...)` pytest tests. No trace validates them yet, so every + verdict is declared (see testgraph/e2e.py) and the summary says so.""" + registry_path = registry_path or reg.resolve_for_repo(repo) + if registry_path is None: + return None, (f"no journey registry for {reg.repo_name(repo)} — looked " + f"in {reg.where_it_looked(repo)}") + try: + registry = reg.load(registry_path) + except (OSError, ValueError) as exc: + # As in `run`: a traceback would exit 1, the "tests failed" code (#154 F9). + return None, f"cannot read registry {registry_path}: {exc}" + if not registry.get("e2e"): + return None, (f"{registry_path} has no `e2e` block — add " + f'{{"runner": "playwright", "cwd": ..., "config": ...}}') + # Checked before selection, so a bad block is refused on every run, not + # only on the ones that select something (review P12). + _cwd, _cfg, err = e2e.check_config(repo, registry["e2e"]) + if err: + return None, err + own = e2e.owned_flag(runner_args) + if own: + return None, e2e.owned_refusal(own) + db_path = db_path or os.path.join(repo, ".codegraph", "codegraph.db") + if not os.path.isfile(db_path): + return None, f"no codegraph index at {db_path} — run `codegraph init` in {repo}" + selection = sel.select(repo, base, head, db_path, registry_path) + if selection.get("status") == "BLOCKED" or selection.get("blocking"): + return _summary(repo, head, selection, refused=True, runner="playwright", + runner_exit=None), None + journeys = [j["id"] if isinstance(j, dict) else j + for j in selection.get("journeys", [])] + if not journeys: + return _summary(repo, head, selection, runner="playwright", runner_exit=None), None + fields, err = e2e.run(repo, journeys, registry["e2e"], runner_args, _runner) + if err: + return None, err + sha = ledger.resolve_commit(repo, head) + written = e2e.write_rows(reg.repo_name(repo), sha, fields, append) + return _summary(repo, head, selection, journeys=journeys, rows_written=written, + commit=sha, **fields), None + + def exit_code(summary): """OK / tests-failed / refused / incomplete, as distinct answers. @@ -366,8 +424,9 @@ def exit_code(summary): return EXIT_INCOMPLETE if summary.get("pytest_exit") not in (None, 0, 5): return EXIT_INCOMPLETE - # Vitest's own exit (#122): non-zero with nothing credited `fail` means the - # run went wrong somewhere the report did not name. + # The runner's own exit (#122, Vitest and Playwright): non-zero with + # nothing credited `fail` means the run went wrong somewhere the report + # did not name. if summary.get("runner_exit") not in (None, 0): return EXIT_INCOMPLETE return EXIT_OK @@ -406,11 +465,19 @@ def render(summary, registry=None): return "\n".join(lines) lines.append(f" journeys selected: {', '.join(summary['journeys'])}") - vitest = summary.get("runner") == "vitest" + runner = summary.get("runner") + vitest, e2e_run = runner == "vitest", runner == "playwright" if vitest: lines.append(f" vitest -t '{summary['marker_expression']}' " f"-> {summary['collected']} tagged test(s), exit " f"{summary.get('runner_exit')}") + elif e2e_run: + lines.append(f" playwright --grep '{summary['marker_expression']}' " + f"-> {summary['collected']} tagged test(s), exit " + f"{summary.get('runner_exit')}") + lines.append(" DECLARED, not trace-validated: a tagged test that passes is " + "credited without proof it reached the journey's backend " + "(backend tracing is #157)") else: lines.append(f" pytest -m '{summary['marker_expression']}' " f"-> {summary['collected']} test(s), exit {summary['pytest_exit']}") @@ -435,12 +502,14 @@ def render(summary, registry=None): lines.append( f" ! NO JOURNEY-LEVEL TEST: {', '.join(uncovered)} — " f"selected, and " - f"{'no test in this run carries `@tg:` in its title or a describe title' if vitest else 'nothing declared `covers(...)`'}" + f"{_UNCOVERED_WHY.get(runner, _UNCOVERED_WHY['pytest'])}" f" for them. This run " f"says NOTHING about those journeys; it is not a pass." ) if summary.get("incomplete"): - why = {"skip": "skipped", "no verdict": "no verdict"} + why = {"skip": "skipped", "no verdict": "no verdict", + "flaky": "flaky — passed only on a retry", + "expected-fail": "failed as test.fail() expected"} lines.append( " ! ran, but not a pass: " + ", ".join(f"{j} ({why.get(r, r)})" for j, r in summary["incomplete"]) @@ -455,8 +524,12 @@ def render(summary, registry=None): ) for nodeid, jid in summary["unvalidated"][:8]: lines.append(f" {jid} {nodeid}") + if summary.get("unknown_tags"): + lines.append(f" ! tests tagged with journeys this registry does not " + f"select or define: {', '.join(summary['unknown_tags'])}") lines.append(f" {summary['rows_written']} ledger row(s) written, tagged " - f"edge_provenance={JOURNEY_PROVENANCE}") + f"edge_provenance=" + f"{e2e.PROVENANCE if e2e_run else JOURNEY_PROVENANCE}") return "\n".join(lines) @@ -485,7 +558,9 @@ def main(argv=None): ap.add_argument("--json", action="store_true") ap.add_argument("--runner", choices=RUNNERS, default="pytest", help="pytest: tests marked covers(...); vitest: tests with " - "@tg: in their title or a describe title") + "@tg: in their title or a describe title; " + "playwright: the registry's e2e block, tests tagged " + "@tg:") ap.add_argument("pytest_args", nargs="*", metavar="runner_args") args = ap.parse_args(argv) diff --git a/tests/test_e2e.py b/tests/test_e2e.py new file mode 100644 index 0000000..44a541b --- /dev/null +++ b/tests/test_e2e.py @@ -0,0 +1,472 @@ +"""`verify --runner playwright`: Playwright tests declared for the selected journeys (#122). + +The runner is faked with recorded Playwright JSON reports, in the shape a real +run produced against honeyslate (specs carry `tags` without the '@'; an untagged +setup project sits beside the journey tests). CI has no browser; the real run is +documented in TECHNICAL.md. +""" +import contextlib +import io +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import types +import unittest +import unittest.mock + +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +sys.path.insert(0, ROOT_DIR) +from testgraph import e2e # noqa: E402 +from testgraph import verify # noqa: E402 +from test_core import _db_on_disk, _git_repo, build_fixture # noqa: E402 + + +def spec(title, tags, status, last="passed"): + results = [] if status == "skipped" else [{"status": last}] + if status == "flaky": + results = [{"status": "failed"}, {"status": "passed"}] + return {"title": title, "tags": tags, + "tests": [{"projectName": "journeys", "status": status, + "results": results}]} + + +def report(*specs, errors=()): + return {"suites": [{"title": "j.spec.js", "specs": list(specs), "suites": []}], + "errors": list(errors), "stats": {}} + + +SETUP_OK = spec("seed and log in", [], "expected") + + +class GrepTests(unittest.TestCase): + def test_one_journey_does_not_select_its_longer_namesake(self): + expr = e2e.grep_expression(["J1"]) + self.assertTrue(re.search(expr, "adds a task @tg:J1")) + self.assertTrue(re.search(expr, "adds a task @tg:J1 @slow")) + self.assertFalse(re.search(expr, "other journey @tg:J10")) + + def test_several_journeys_are_a_union(self): + expr = e2e.grep_expression(["J1", "J2"]) + self.assertTrue(re.search(expr, "x @tg:J2")) + self.assertFalse(re.search(expr, "x @tg:J3")) + + def test_journey_ids_are_matched_literally(self): + expr = e2e.grep_expression(["J1.2"]) + self.assertTrue(re.search(expr, "x @tg:J1.2")) + self.assertFalse(re.search(expr, "x @tg:J1x2")) + + def test_the_tag_follows_the_project_convention(self): + self.assertEqual(e2e.tag("J1"), "@tg:J1") + + def test_P5_the_grep_argument_is_case_sensitive(self): + """Playwright compiles a bare --grep with flags "gi"; a /.../ literal + with no flags is compiled as written (forceRegExp, lib/util.js).""" + self.assertEqual(e2e.grep_argument(["J1"]), "/" + e2e.grep_expression(["J1"]) + "/") + + +class OwnedFlagTests(unittest.TestCase): + def test_P4_flags_verify_owns_are_found(self): + for args in (("--grep", "x"), ("--grep=x",), ("-g", "x"), ("-gx",), ("-xg", "x"), + ("--grep-invert", "x"), ("--grep-invert=x",), ("--reporter=list",), + ("--reporter", "list"), ("-c", "x"), ("--config=x",), ("-cx",)): + with self.subTest(args): + self.assertEqual(e2e.owned_flag(args), args[0]) + + def test_P4_other_flags_pass(self): + for args in ((), ("--workers=1",), ("-j", "2"), ("-j2",), ("-x",), ("--project", "journeys"), + ("--headed",), ("--shard=1/2",), ("tests/a.spec.js",), ("-u",), + ("--max-failures=1",), ("--retries", "1")): + with self.subTest(args): + self.assertIsNone(e2e.owned_flag(args)) + + +class JudgeTests(unittest.TestCase): + def _judge(self, *specs, journeys=("J1",)): + per, untagged, unknown = e2e.parse_report(report(SETUP_OK, *specs), list(journeys)) + return e2e.judge(per) + (untagged, unknown) + + def test_a_passing_tagged_test_is_a_pass(self): + credited, incomplete, uncovered, _, _ = self._judge(spec("a", ["tg:J1"], "expected")) + self.assertEqual((credited, incomplete, uncovered), ({"J1": "pass"}, [], [])) + + def test_one_failure_fails_the_journey(self): + credited, *_ = self._judge(spec("a", ["tg:J1"], "expected"), + spec("b", ["tg:J1"], "unexpected", "failed")) + self.assertEqual(credited, {"J1": "fail"}) + + def test_a_pass_only_on_retry_is_not_a_pass(self): + credited, incomplete, *_ = self._judge(spec("a", ["tg:J1"], "flaky")) + self.assertEqual(credited, {}) + self.assertEqual(incomplete, [("J1", "flaky")]) + + def test_skipped_only_is_not_a_pass(self): + credited, incomplete, *_ = self._judge(spec("a", ["tg:J1"], "skipped")) + self.assertEqual(credited, {"J1": "skip"}) + self.assertEqual(incomplete, [("J1", "skip")]) + + def test_an_expected_failure_is_not_a_pass(self): + """`test.fail()` reports `expected` with a failed attempt.""" + credited, incomplete, *_ = self._judge(spec("a", ["tg:J1"], "expected", "failed")) + self.assertNotEqual(credited.get("J1"), "pass") + # P7: not "skip" — the test ran and failed as test.fail() expected. + self.assertEqual(credited, {}) + self.assertEqual(incomplete, [("J1", "expected-fail")]) + + def test_P5_a_failure_tagged_only_for_other_journeys_is_a_broken_environment(self): + for tags in (["tg:J2"], ["tg:j1"], ["TG:J1"]): + with self.subTest(tags): + _, untagged, _ = e2e.parse_report( + report(spec("setup", tags, "unexpected", "failed"), + spec("a", ["tg:J1"], "skipped")), ["J1"]) + self.assertEqual(untagged, ["setup"]) + + def test_no_tagged_test_is_uncovered(self): + credited, incomplete, uncovered, *_ = self._judge(spec("a", ["tg:J2"], "expected")) + self.assertEqual(uncovered, ["J1"]) + + def test_a_failing_untagged_test_is_reported(self): + per, untagged, _ = e2e.parse_report( + report(spec("seed and log in", [], "unexpected", "failed"), + spec("a", ["tg:J1"], "skipped")), ["J1"]) + self.assertEqual(untagged, ["seed and log in"]) + + def test_other_tags_are_ignored_and_unknown_journeys_named(self): + _, _, uncovered, _, unknown = self._judge( + spec("a", ["tg:J1", "slow"], "expected"), spec("b", ["tg:J9"], "expected")) + self.assertEqual(uncovered, []) + self.assertEqual(unknown, ["J9"]) + + +class RunE2ETests(unittest.TestCase): + """verify.run_e2e over a real git diff that selects J1, with a fake runner.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + src = build_fixture() + self.db = _db_on_disk(self.tmp, src) + src.close() + self.repo, run = _git_repo(self.tmp, {"app/svc.py": 22}) + os.makedirs(os.path.join(self.repo, "frontend")) + full = os.path.join(self.repo, "app", "svc.py") + with open(full) as fh: + lines = fh.readlines() + lines[11] = "changed = 1\n" # inside handler_a -> J1 + with open(full, "w") as fh: + fh.writelines(lines) + run("git", "add", "-A") + run("git", "commit", "-qm", "touch") + self.registry = self._registry({"runner": "playwright", "cwd": "frontend", + "config": "playwright.real.config.js"}) + self.rows = [] + + def _registry(self, e2e_block): + path = os.path.join(self.tmp, "reg.json") + body = {"codegraph_schema_version": 8, "approved": True, "spot_checks": {}, + "journeys": {"J1": {"name": "submit", "entries": [ + {"name": "handler_a", "file": "app/svc.py"}]}}} + if e2e_block is not None: + body["e2e"] = e2e_block + with open(path, "w") as fh: + json.dump(body, fh) + return path + + def _run(self, rep, returncode=0, write=True, stderr=""): + seen = {} + + def runner(cmd, cwd, env): + seen.update(cmd=cmd, cwd=cwd) + if write: + with open(env["PLAYWRIGHT_JSON_OUTPUT_NAME"], "w") as fh: + fh.write(rep if isinstance(rep, str) else json.dumps(rep)) + return types.SimpleNamespace(returncode=returncode, stderr=stderr) + + summary, err = verify.run_e2e(self.repo, "HEAD~1", "HEAD", + registry_path=self.registry, db_path=self.db, + append=lambda row: self.rows.append(row) or True, + _runner=runner) + return summary, err, seen + + def test_a_pass_is_exit_0_and_labelled_declared(self): + summary, err, seen = self._run(report(SETUP_OK, spec("a", ["tg:J1"], "expected"))) + self.assertIsNone(err, err) + self.assertEqual(summary["credited"], {"J1": "pass"}) + self.assertEqual(verify.exit_code(summary), verify.EXIT_OK) + out = verify.render(summary) + self.assertIn("DECLARED, not trace-validated", out) + self.assertIn("edge_provenance=e2e-declared", out) + self.assertEqual([r["edge_provenance"] for r in self.rows], ["e2e-declared"]) + self.assertEqual(seen["cwd"], os.path.join(self.repo, "frontend")) + self.assertEqual(seen["cmd"][:7], ["npx", "--no-install", "playwright", "test", + "-c", "playwright.real.config.js", "--grep"]) + + def test_a_failure_is_exit_1(self): + summary, err, _ = self._run( + report(SETUP_OK, spec("a", ["tg:J1"], "unexpected", "failed")), returncode=1) + self.assertIsNone(err, err) + self.assertEqual(verify.exit_code(summary), verify.EXIT_TESTS_FAILED) + + def test_no_tagged_test_is_exit_3(self): + summary, err, _ = self._run(report(SETUP_OK)) + self.assertIsNone(err, err) + self.assertEqual(summary["uncovered"], ["J1"]) + self.assertEqual(verify.exit_code(summary), verify.EXIT_INCOMPLETE) + self.assertIn("no Playwright test in this run carries a `@tg:` tag", verify.render(summary)) + + def test_a_flaky_pass_is_exit_3(self): + summary, err, _ = self._run(report(SETUP_OK, spec("a", ["tg:J1"], "flaky"))) + self.assertEqual(verify.exit_code(summary), verify.EXIT_INCOMPLETE) + self.assertIn("passed only on a retry", verify.render(summary)) + self.assertEqual(self.rows, []) + + def test_a_skipped_only_journey_writes_no_ledger_row(self): + summary, err, _ = self._run(report(SETUP_OK, spec("a", ["tg:J1"], "skipped"))) + self.assertIsNone(err, err) + self.assertEqual(verify.exit_code(summary), verify.EXIT_INCOMPLETE) + self.assertEqual(self.rows, []) + + def test_a_nonzero_runner_exit_with_nothing_failed_is_not_a_pass(self): + summary, err, _ = self._run(report(SETUP_OK, spec("a", ["tg:J1"], "expected")), + returncode=1) + self.assertEqual(verify.exit_code(summary), verify.EXIT_INCOMPLETE) + + def test_a_broken_setup_is_refused(self): + summary, err, _ = self._run(report(spec("seed and log in", [], "unexpected", "failed"), + spec("a", ["tg:J1"], "skipped")), returncode=1) + self.assertIsNone(summary) + self.assertIn("untagged Playwright test failed", err) + + def test_no_report_is_refused_with_stderr(self): + summary, err, _ = self._run(None, returncode=1, write=False, + stderr="Error: browserType.launch: Executable doesn't exist") + self.assertIsNone(summary) + self.assertIn("wrote no JSON report", err) + self.assertIn("Executable doesn't exist", err) + + def test_a_run_error_or_unreadable_report_is_refused(self): + for rep, want in ((report(errors=[{"message": "config: no tests"}]), "run error"), + ("{", "unreadable"), ("[]", "not a JSON object")): + with self.subTest(want): + summary, err, _ = self._run(rep) + self.assertIsNone(summary) + self.assertIn(want, err) + + def test_P1_a_malformed_e2e_block_is_refused_not_a_crash(self): + for block in ("playwright", 5, True, ["x"], + {"runner": "playwright", "cwd": None}, + {"runner": "playwright", "cwd": 5}, + {"runner": "playwright"}, + {"runner": "playwright", "cwd": ""}, + {"runner": "playwright", "cwd": "frontend", "config": 5}, + {"runner": "playwright", "cwd": "frontend", "config": ["a"]}): + with self.subTest(block): + self.registry = self._registry(block) + summary, err, _ = self._run(report()) + self.assertIsNone(summary) + self.assertIn("registry e2e", err) + + def test_P3_a_cwd_or_config_outside_the_repo_is_refused(self): + out = os.path.join(self.tmp, "outside") + os.makedirs(out) + os.symlink(out, os.path.join(self.repo, "frontend", "escape")) + for block in ({"runner": "playwright", "cwd": "../.."}, + {"runner": "playwright", "cwd": "frontend/../.."}, + {"runner": "playwright", "cwd": self.tmp}, + {"runner": "playwright", "cwd": "frontend/escape"}, + {"runner": "playwright", "cwd": "frontend", "config": "-x"}, + {"runner": "playwright", "cwd": "frontend", "config": "/etc/x.js"}, + {"runner": "playwright", "cwd": "frontend", "config": "../../x.js"}, + {"runner": "playwright", "cwd": "frontend", "config": "escape/x.js"}): + with self.subTest(block): + self.registry = self._registry(block) + summary, err, seen = self._run(report()) + self.assertIsNone(summary) + self.assertIn("registry e2e", err) + self.assertEqual(seen, {}) # nothing ran + + def test_P3_the_repo_root_itself_is_a_valid_cwd(self): + self.registry = self._registry({"runner": "playwright", "cwd": "."}) + summary, err, seen = self._run(report(SETUP_OK)) + self.assertIsNone(err, err) + self.assertEqual(os.path.realpath(seen["cwd"]), os.path.realpath(self.repo)) + + def test_P12_a_bad_block_is_refused_even_when_nothing_is_selected(self): + self.registry = self._registry({"runner": "playwright", "cwd": "../.."}) + summary, err = verify.run_e2e(self.repo, "HEAD", "HEAD", registry_path=self.registry, + db_path=self.db, append=lambda r: True, + _runner=lambda *a: None) + self.assertIsNone(summary) + self.assertIn("registry e2e", err) + + def test_P2_no_matching_test_is_not_an_error(self): + _, _, seen = self._run(report(SETUP_OK)) + self.assertIn("--pass-with-no-tests", seen["cmd"]) + + def test_P5_the_command_greps_case_sensitively(self): + _, _, seen = self._run(report(SETUP_OK)) + self.assertEqual(seen["cmd"][seen["cmd"].index("--grep") + 1], e2e.grep_argument(["J1"])) + + def test_P4_a_flag_verify_owns_is_refused_before_the_run(self): + seen = {} + summary, err = verify.run_e2e(self.repo, "HEAD~1", "HEAD", registry_path=self.registry, + db_path=self.db, runner_args=("-g", "smoke"), + append=lambda r: True, + _runner=lambda *a: seen.update(ran=True)) + self.assertIsNone(summary) + self.assertIn("owns", err) + self.assertEqual(seen, {}) + + def test_P4_e2e_run_refuses_an_owned_flag_itself(self): + fields, err = e2e.run(self.repo, ["J1"], {"runner": "playwright", "cwd": "frontend"}, + extra_args=("--reporter=list",), + _runner=lambda *a: self.fail("must not run")) + self.assertIsNone(fields) + self.assertIn("owns", err) + + def test_P4_an_owned_flag_is_refused_even_when_nothing_is_selected(self): + summary, err = verify.run_e2e(self.repo, "HEAD", "HEAD", registry_path=self.registry, + db_path=self.db, runner_args=("--grep", "x"), + append=lambda r: True, _runner=lambda *a: None) + self.assertIsNone(summary) + self.assertIn("owns", err) + + def test_P4_an_inherited_json_output_file_cannot_redirect_the_report(self): + with unittest.mock.patch.dict(os.environ, { + "PLAYWRIGHT_JSON_OUTPUT_FILE": os.path.join(self.tmp, "theirs.json")}): + captured = {} + + def runner(cmd, cwd, env): + captured.update(env) + with open(env["PLAYWRIGHT_JSON_OUTPUT_FILE"], "w") as fh: + json.dump(report(SETUP_OK, spec("a", ["tg:J1"], "expected")), fh) + return types.SimpleNamespace(returncode=0, stderr="") + summary, err = verify.run_e2e(self.repo, "HEAD~1", "HEAD", + registry_path=self.registry, db_path=self.db, + append=lambda r: True, _runner=runner) + self.assertIsNone(err, err) + self.assertEqual(captured["PLAYWRIGHT_JSON_OUTPUT_FILE"], + captured["PLAYWRIGHT_JSON_OUTPUT_NAME"]) + self.assertFalse(os.path.exists(os.path.join(self.tmp, "theirs.json"))) + + def test_P6_a_run_stopped_by_max_failures_still_reports_the_failure(self): + summary, err, _ = self._run(report( + SETUP_OK, spec("a", ["tg:J1"], "unexpected", "failed"), + errors=[{"message": "Testing stopped early after 1 maximum allowed failures."}]), + returncode=1) + self.assertIsNone(err, err) + self.assertEqual(summary["credited"], {"J1": "fail"}) + self.assertEqual(verify.exit_code(summary), verify.EXIT_TESTS_FAILED) + + def test_P6_a_stopped_run_with_no_tagged_failure_is_still_refused(self): + summary, err, _ = self._run(report( + SETUP_OK, spec("a", ["tg:J1"], "skipped"), + errors=[{"message": "Testing stopped early after 1 maximum allowed failures."}]), + returncode=1) + self.assertIsNone(summary) + self.assertIn("run error", err) + + def test_P7_an_expected_failure_renders_as_such(self): + summary, err, _ = self._run(report(SETUP_OK, spec("a", ["tg:J1"], "expected", "failed"))) + self.assertIsNone(err, err) + self.assertEqual(summary["incomplete"], [("J1", "expected-fail")]) + text = verify.render(summary) + self.assertIn("failed as test.fail() expected", text) + self.assertNotIn("(skipped)", text) + + def test_a_registry_without_e2e_is_refused(self): + self.registry = self._registry(None) + summary, err, _ = self._run(report()) + self.assertIsNone(summary) + self.assertIn("no `e2e` block", err) + + def test_an_unknown_runner_is_refused(self): + self.registry = self._registry({"runner": "cypress", "cwd": "frontend"}) + summary, err, _ = self._run(report()) + self.assertIsNone(summary) + self.assertIn("e2e.runner", err) + + def test_runner_playwright_routes_to_e2e(self): + with unittest.mock.patch.object(verify, "run_e2e", + return_value=(None, "e2e path")) as m, \ + contextlib.redirect_stdout(io.StringIO()): + rc = verify.main(["--repo", self.repo, "--base", "HEAD~1", "--runner", "playwright", + "--", "--workers=1"]) + self.assertEqual(rc, verify.EXIT_REFUSED) + self.assertEqual(m.call_args.kwargs["runner_args"], ("--workers=1",)) + + +PW_NODE_MODULES = os.environ.get( + "TG_PLAYWRIGHT_NODE_MODULES", + "/home/ericm/personal_projects/honeyslate/e2e-j1-real/frontend/node_modules") +HAVE_PLAYWRIGHT = bool(shutil.which("node")) and os.path.isfile( + os.path.join(PW_NODE_MODULES, "@playwright", "test", "package.json")) +PW_CONFIG = ('const { defineConfig } = require("@playwright/test");\n' + 'module.exports = defineConfig({ testDir: ".", workers: 1 });\n') +PW_HEAD = 'const { test, expect } = require("@playwright/test");\n' + + +@unittest.skipUnless(HAVE_PLAYWRIGHT, "needs node and a Playwright install " + "(TG_PLAYWRIGHT_NODE_MODULES)") +class RealPlaywrightVerifyTests(unittest.TestCase): + """verify --runner playwright against real Playwright 1.x: tests with no + `page`, so no browser starts. One commit selects J1.""" + + setUp = RunE2ETests.setUp + _registry = RunE2ETests._registry + + def _real(self, body, args=()): + fe = os.path.join(self.repo, "frontend") + if not os.path.lexists(os.path.join(fe, "node_modules")): + os.symlink(PW_NODE_MODULES, os.path.join(fe, "node_modules")) + with open(os.path.join(fe, "pw.config.js"), "w") as fh: + fh.write(PW_CONFIG) + with open(os.path.join(fe, "a.spec.js"), "w") as fh: + fh.write(PW_HEAD + body) + self.registry = self._registry({"runner": "playwright", "cwd": "frontend", + "config": "pw.config.js"}) + summary, err = verify.run_e2e(self.repo, "HEAD~1", "HEAD", + registry_path=self.registry, db_path=self.db, + runner_args=args, append=lambda r: True) + self.assertIsNone(err, err) + self.assertEqual(summary["journeys"], ["J1"]) + return summary + + def test_a_tagged_pass_is_exit_0_and_untagged_tests_do_not_run(self): + s = self._real('test("adds @tg:J1", () => {});\n' + 'test("other", () => { throw new Error("must not run"); });\n') + self.assertEqual(s["credited"], {"J1": "pass"}) + self.assertEqual((s["runner_exit"], verify.exit_code(s)), (0, verify.EXIT_OK)) + + def test_P2_no_tagged_test_is_uncovered_not_refused(self): + s = self._real('test("someone else @tg:J9", () => {});\n') + self.assertEqual(s["uncovered"], ["J1"]) + self.assertEqual(verify.exit_code(s), verify.EXIT_INCOMPLETE) + + def test_P5_a_lowercase_tag_is_not_run(self): + s = self._real('test("lower @tg:j1", () => { throw new Error("ran"); });\n') + self.assertEqual((s["uncovered"], s["runner_exit"]), (["J1"], 0)) + + def test_P6_max_failures_still_fails_the_journey(self): + s = self._real('test("a @tg:J1", () => { throw new Error("red"); });\n' + 'test("b @tg:J1", () => { throw new Error("red"); });\n', args=("-x",)) + self.assertEqual(s["credited"], {"J1": "fail"}) + self.assertEqual(verify.exit_code(s), verify.EXIT_TESTS_FAILED) + + def test_P7_test_fail_is_an_expected_failure(self): + s = self._real('test.fail("xf @tg:J1", () => { throw new Error("known"); });\n') + self.assertEqual(s["incomplete"], [("J1", "expected-fail")]) + + def test_P4_an_inherited_json_output_file_is_ignored(self): + theirs = os.path.join(self.tmp, "theirs.json") + with unittest.mock.patch.dict(os.environ, {"PLAYWRIGHT_JSON_OUTPUT_FILE": theirs}): + s = self._real('test("adds @tg:J1", () => {});\n') + self.assertEqual(s["credited"], {"J1": "pass"}) + self.assertFalse(os.path.exists(theirs)) + + +if __name__ == "__main__": + unittest.main()