diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2ad40e605d..18096ee0ec 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2238,50 +2238,45 @@ jobs: # --------------------------------------------------------------------------- # Parity tests (Perry output vs Node.js) # --------------------------------------------------------------------------- + # parity: the full Perry-vs-node parity sweep, SHARDED. Full tier only. + # + # Sharded 2026-08-16: the unsharded job was killed by GitHub's 6-hour job + # cap (run 31935729773, 11:44 -> 17:45) — the release gate could not + # complete even in principle. Each shard runs `run_parity_tests.sh + # --shard N/M` (round-robin partition, same mechanism as gap-suite) plus + # `parity_known_failures.py`, which is shard-safe by design ("not in this + # shard is never flagged"). The AGGREGATE gates — the threshold minimums + # and the per-module matrix trend, whose baselines describe the whole + # suite — run once in `parity-aggregate` below over the merged report + # (scripts/parity_report_merge.py, which FAILS on a missing shard rather + # than shrinking the suite). + # + # No `continue-on-error`: since the tiered restructure a red here fails + # `full-suite-gate`, which is exactly what release-packages.yml's + # await-tests keys on. If a standing failure must not block a release, + # triage it into test-parity/known_failures.json — do not soften the job. + # --------------------------------------------------------------------------- parity: # Sweep/full-tier only. Chained behind `check` so a sweep's fan-out does # not take every runner slot the moment a merge lands -- PR gates share # the same 20-slot pool. needs: [plan, check] if: fromJSON(needs.plan.outputs.plan).jobs.parity - # Release-publish decoupling: aspirational extended suite. Per maintainer - # decision it no longer BLOCKS package publishing — release-packages.yml's - # await-tests gate keys on this workflow's run conclusion, and job-level - # `continue-on-error: true` keeps a red result here from failing that - # conclusion. The job still runs on every tag + shows its own pass/fail as - # an informational signal (and core jobs — cargo-test/lint/api-docs-drift/ - # compiler-output-regression — still gate publish). - # Was macos-14 — moved to ubuntu-latest in v0.5.392. Parity tests - # just compare Perry's stdout against `node --experimental-strip-types`'s - # stdout per test file; both run cleanly on Linux. `gtimeout` on - # macOS is `timeout` on Linux (the run_parity_tests.sh wrapper - # detects either). Node 22+ is installed via setup-node@v4 below. - # 10× billing weight cut. - # - # v0.5.1018: gated to tag pushes only (`github.event_name == 'push'`). - # The pull_request trigger above still fires the workflow on PRs for - # the lint / cargo-test / api-docs-drift gates, but parity now only - # runs on release tags. Direct main commits + PR cycle no longer pay - # the ~20 min parity bill; release tagging still catches regressions - # before publish (release-packages.yml await-tests gate waits on - # this job by name for tag events). - # - # Opt-in: apply the `run-extended-tests` label to a PR, or dispatch - # the workflow manually with `run_extended_tests=true`, to run this - # job on demand. PR authors and maintainers can both apply labels. + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON(needs.plan.outputs.plan).parity.shards }} runs-on: ubuntu-latest + timeout-minutes: 150 steps: - uses: actions/checkout@v7 - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable - uses: ./.github/actions/setup-llvm22 - - uses: Swatinem/rust-cache@v2 with: shared-key: "${{ runner.os }}-perry" save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Setup Node.js uses: actions/setup-node@v7 with: @@ -2290,13 +2285,89 @@ jobs: # correctness input, not an incidental toolchain detail — never pin it # inline here. See CLAUDE.md ("TypeScript Parity Status"). node-version-file: .node-version - + # #8198: the npm-fixture parity/gap tests import root devDependencies. + - name: Install the npm packages the oracle imports + run: npm ci --ignore-scripts --no-audit --no-fund - name: Build compiler run: cargo build --release + - name: Run parity tests (shard ${{ matrix.shard }}/${{ fromJSON(needs.plan.outputs.plan).parity.total }}) + run: ./run_parity_tests.sh --shard ${{ matrix.shard }}/${{ fromJSON(needs.plan.outputs.plan).parity.total }} + # Bidirectional since #7582: red on a failure that is not allowed here, + # AND red on an allowlist entry whose test ran on this platform and + # PASSED. Shard-safe: an entry whose test is not in this shard is never + # flagged, so each shard adjudicates exactly its own slice. + - name: Check for new and stale failures + run: >- + python3 scripts/parity_known_failures.py + --report test-parity/reports/latest.json + --known test-parity/known_failures.json + - name: Upload shard report + if: always() + uses: actions/upload-artifact@v7 + with: + name: parity-shard-${{ matrix.shard }} + path: | + test-parity/reports/latest.json + test-parity/output/node/test_parity_*.txt + test-parity/output/perry/test_parity_*.txt + if-no-files-found: error + # Capture per-test compile stderr written by run_parity_tests.sh into + # `test-parity/output/*.compile_error.log` so the long-tail + # macOS-14-only compile failures (tracked as `ci-env` in + # known_failures.json) can finally be diagnosed by reading the actual + # error message rather than inferring from the test family. + - name: Upload compile-error logs + if: always() + uses: actions/upload-artifact@v7 + with: + name: parity-compile-errors-${{ runner.os }}-shard-${{ matrix.shard }} + path: test-parity/output/*.compile_error.log + if-no-files-found: ignore - - name: Run parity tests - run: ./run_parity_tests.sh - + # --------------------------------------------------------------------------- + # parity-aggregate: fan-in for the sharded parity sweep. Merges the shard + # reports into one whole-suite report and runs the gates whose baselines + # only make sense on the aggregate: the global + per-category threshold + # minimums (a 62%-floor category with two tests in a shard would flap) and + # the per-module matrix trend. The merge REFUSES a missing shard (--expect), + # so a lost artifact is a red run, not a smaller green suite. + # --------------------------------------------------------------------------- + parity-aggregate: + needs: [plan, parity] + # `!cancelled()`: run even when a shard FAILED its verdict (dark debt in + # the known-failures gate) — the merged report is exactly what a triage + # needs, and the threshold/trend verdicts are meaningful regardless. A + # missing shard ARTIFACT still fails the merge (--expect). Skipped only + # when the plan turned parity off. + if: ${{ !cancelled() && fromJSON(needs.plan.outputs.plan).jobs.parity }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - name: Download shard reports + uses: actions/download-artifact@v8 + with: + pattern: parity-shard-* + path: parity-shards + - name: Merge shard reports + env: + EXPECT: ${{ fromJSON(needs.plan.outputs.plan).parity.total }} + run: | + set -euo pipefail + python3 scripts/parity_report_merge.py --self-test + mkdir -p test-parity/reports test-parity/output/node test-parity/output/perry + # Reconstruct the per-test output captures the matrix trend reads. + # NOTE: upload-artifact strips the common parent, so each artifact + # extracts as parity-shard-N/{reports,output}/... (no test-parity/ + # prefix) — verified against run 31964093732's artifacts. + for d in parity-shards/parity-shard-*/; do + if [ -d "$d/output/node" ]; then cp "$d"/output/node/*.txt test-parity/output/node/ 2>/dev/null || true; fi + if [ -d "$d/output/perry" ]; then cp "$d"/output/perry/*.txt test-parity/output/perry/ 2>/dev/null || true; fi + done + python3 scripts/parity_report_merge.py \ + --expect "$EXPECT" \ + --output test-parity/reports/latest.json \ + parity-shards/parity-shard-*/reports/latest.json - name: Check parity threshold run: | set +e @@ -2308,18 +2379,6 @@ jobs: set -e cat test-parity/reports/parity_threshold_latest.md >> "$GITHUB_STEP_SUMMARY" exit "$status" - - # Bidirectional since #7582: red on a failure that is not allowed here, - # AND red on an allowlist entry whose test ran on this platform and - # PASSED. The second direction is the one that matters — the file was a - # pure suppression list, so a fixed test kept its entry forever and the - # entry silently absorbed that test's next regression (#7580). - - name: Check for new and stale failures - run: >- - python3 scripts/parity_known_failures.py - --report test-parity/reports/latest.json - --known test-parity/known_failures.json - - name: Generate parity matrix trend run: | python3 scripts/parity_matrix_trend.py \ @@ -2327,8 +2386,7 @@ jobs: --output-json test-parity/reports/parity_matrix_latest.json \ --output-md test-parity/reports/parity_matrix_latest.md cat test-parity/reports/parity_matrix_latest.md >> "$GITHUB_STEP_SUMMARY" - - - name: Upload parity report + - name: Upload merged parity report if: always() uses: actions/upload-artifact@v7 with: @@ -2340,18 +2398,6 @@ jobs: test-parity/reports/parity_matrix_latest.json test-parity/reports/parity_matrix_latest.md - # Capture per-test compile stderr written by run_parity_tests.sh into - # `test-parity/output/*.compile_error.log` so the long-tail - # macOS-14-only compile failures (tracked as `ci-env` in - # known_failures.json) can finally be diagnosed by reading the actual - # error message rather than inferring from the test family. - - name: Upload compile-error logs - if: always() - uses: actions/upload-artifact@v7 - with: - name: parity-compile-errors-${{ runner.os }} - path: test-parity/output/*.compile_error.log - if-no-files-found: ignore # --------------------------------------------------------------------------- # Compile smoke test (all 130+ test files must compile) @@ -3340,6 +3386,7 @@ jobs: - native-abi-evidence-packet - gap-suite - parity + - parity-aggregate - compile-smoke - harmonyos-smoke - drizzle-mysql-smoke diff --git a/changelog.d/8244-parity-shards.md b/changelog.d/8244-parity-shards.md new file mode 100644 index 0000000000..5207467879 --- /dev/null +++ b/changelog.d/8244-parity-shards.md @@ -0,0 +1,12 @@ +### `parity` sharded 8 ways + aggregate fan-in (#8244) + +The first full-tier run of the tiered CI (#8187) proved the unsharded `parity` +job cannot complete: GitHub's 6-hour job cap killed it (run 31935729773). It is +now a plan-driven 8-shard matrix (`run_parity_tests.sh --shard N/8`), with +`parity_known_failures.py` running per shard (shard-safe by design) and a new +`parity-aggregate` fan-in that merges the shard reports via +`scripts/parity_report_merge.py` (self-tested; refuses a missing shard rather +than shrinking the suite) and runs the aggregate-only gates: the threshold +minimums and the per-module matrix trend. `full-suite-gate` — the job +`release-packages.yml`'s `await-tests` waits for — requires the aggregate, so +the release gate can now actually finish. diff --git a/docs/src/testing/ci-tiers.md b/docs/src/testing/ci-tiers.md index 263fce8162..c66974254e 100644 --- a/docs/src/testing/ci-tiers.md +++ b/docs/src/testing/ci-tiers.md @@ -156,3 +156,10 @@ that each main-line sweep produces a *successful* run within its budget. pinned release branch and polls the SHA for a run whose **`full-suite-gate`** job succeeded — a green sweep or PR-tier run on the same SHA does not count. See [Releasing](../contributing/releasing.md). + +`parity` runs as 8 shards (`run_parity_tests.sh --shard N/8`) after the unsharded +job was killed by GitHub's 6-hour job cap on 2026-08-16. `parity_known_failures.py` +runs inside each shard (it is shard-safe by design); the threshold minimums and the +per-module matrix trend run once in `parity-aggregate` over the merged report +(`scripts/parity_report_merge.py`, which refuses a missing shard rather than +shrinking the suite). diff --git a/scripts/ci_plan.py b/scripts/ci_plan.py index 984c80a340..3679154fbf 100755 --- a/scripts/ci_plan.py +++ b/scripts/ci_plan.py @@ -120,6 +120,12 @@ "full": {"mode": "full", "total": 8}, } +# Parity: full tier only, sharded. The unsharded job was killed by GitHub's +# 6-hour job cap on 2026-08-16 (run 31935729773) — 8 shards puts each around +# 45-75 min. `parity-aggregate` (not in JOBS: it keys off `jobs.parity`) +# merges the shard reports and runs the aggregate-only gates. +PARITY_SHARDS = 8 + EXTENDED_LABEL = "run-extended-tests" # --------------------------------------------------------------------------- @@ -279,6 +285,7 @@ def plan( "scope": scope, "jobs": jobs, "gap": gap, + "parity": {"total": PARITY_SHARDS, "shards": list(range(1, PARITY_SHARDS + 1))}, # cargo-test: a pull_request run scopes to the diff via ci_test_scope.py # (`--lib --bins` of the affected crates); everything else -- including a # `workflow_dispatch --tier pr`, which has no PR to read -- runs the full @@ -379,6 +386,7 @@ def check(name: str, cond: bool): full = plan("schedule", "refs/heads/main") check("full: every job except e2e-scoped", all(v for k, v in full["jobs"].items() if k != "e2e_scoped")) check("full: 8 auto-optimize gap shards", full["gap"]["total"] == 8 and full["gap"]["mode"] == "full") + check("full: parity sharded (6h-cap kill, 2026-08-16)", full["parity"]["total"] >= 2 and full["parity"]["shards"][0] == 1) labelled = plan("pull_request", "refs/pull/1/merge", labels=[EXTENDED_LABEL], changed=["README.md"]) check("labelled PR runs the full tier regardless of scope", labelled["jobs"]["parity"] and labelled["jobs"]["gap_suite"]) diff --git a/scripts/parity_report_merge.py b/scripts/parity_report_merge.py new file mode 100755 index 0000000000..f2613615c4 --- /dev/null +++ b/scripts/parity_report_merge.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Merge N sharded parity reports into one whole-suite report. + +WHY: the `parity` job used to run the whole suite in ONE runner and was killed +by GitHub's 6-hour job cap on 2026-08-16 (run 31935729773, 11:44 -> 17:45, +`cancelled`) — the release gate could not complete even in principle. The job +is now sharded (`run_parity_tests.sh --shard N/M`, same round-robin partition +the gap suite uses), and the AGGREGATE gates — `parity_threshold_gate.py` and +`parity_matrix_trend.py` — run once in a fan-in job over the report this +script produces. They stay aggregate on purpose: a per-category minimum +evaluated on a shard's small slice would flap (a 62%-floor category with two +tests in a shard reads 50% on one failure), and the matrix trend's committed +baseline describes the whole suite. `parity_known_failures.py` is the one +gate that IS shard-safe by design ("not in this shard is never flagged"), so +it runs inside each shard. + +Merge semantics: + * `results`: concatenated. A test id appearing in two inputs is an ERROR — + shards partition the suite disjointly, so a duplicate means two inputs + were the same shard (or an artifact was downloaded twice). + * `failures.{parity,compile,crash}`: unioned, sorted, deduplicated. + * `summary`: counts summed; `parity_percentage` recomputed with the + harness's own formula (pass / (pass + parity_fail + crash), in tenths). + * `platform`: must agree across inputs; `generated_at`: max of inputs. + +--expect N is REQUIRED and this script FAILS when given fewer inputs: a lost +shard artifact must not silently shrink the suite into a smaller, greener one +(the #6364/#7856 hazard: absence of evidence reading as passing). + +Usage: + scripts/parity_report_merge.py --expect 8 --output merged.json shard*/latest.json + scripts/parity_report_merge.py --self-test +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + + +def load(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as fh: + data = json.load(fh) + if not isinstance(data, dict) or not isinstance(data.get("results"), list): + raise SystemExit(f"{path}: not a parity report (no results[])") + return data + + +SUM_KEYS = ( + "parity_pass", + "parity_fail", + "compile_fail", + "crash_fail", + "node_fail", + "skipped", + "total_run", +) + + +def merge(reports: list[dict[str, Any]], sources: list[str]) -> dict[str, Any]: + platforms = {r.get("platform") for r in reports} + if len(platforms) != 1: + raise SystemExit(f"platform mismatch across shard reports: {sorted(map(str, platforms))}") + + seen: dict[str, str] = {} + results: list[dict[str, Any]] = [] + for rep, src in zip(reports, sources): + for item in rep["results"]: + if not isinstance(item, dict): + continue + tid = item.get("id") or item.get("test") or item.get("name") + if tid is not None: + if tid in seen: + raise SystemExit( + f"test id {tid!r} appears in both {seen[tid]} and {src} — " + f"shards partition disjointly, so two inputs cover the same shard" + ) + seen[tid] = src + results.append(item) + + summary = {k: 0 for k in SUM_KEYS} + for rep in reports: + s = rep.get("summary") or {} + for k in SUM_KEYS: + v = s.get(k, 0) + if not isinstance(v, int): + raise SystemExit(f"summary.{k} is not an int in one input: {v!r}") + summary[k] += v + # The harness's own formula (run_parity_tests.sh): pass / (pass+fail+crash), + # integer tenths. Reproduced exactly so a merged report of one shard is + # byte-comparable with that shard's own report. + denom = summary["parity_pass"] + summary["parity_fail"] + summary["crash_fail"] + if denom > 0: + tenths = summary["parity_pass"] * 1000 // denom + summary["parity_percentage"] = float(f"{tenths // 10}.{tenths % 10}") + else: + summary["parity_percentage"] = 0.0 + + failures: dict[str, list[str]] = {} + for key in ("parity", "compile", "crash"): + bucket: set[str] = set() + for rep in reports: + for name in (rep.get("failures") or {}).get(key, []) or []: + if name: + bucket.add(name) + failures[key] = sorted(bucket) + + return { + "generated_at": max(str(r.get("generated_at", "")) for r in reports), + "platform": next(iter(platforms)), + "merged_from": len(reports), + "summary": summary, + "failures": failures, + "results": results, + } + + +# --------------------------------------------------------------------------- +def _rep(platform: str = "linux", ids: tuple[str, ...] = (), fails: tuple[str, ...] = ()) -> dict: + return { + "generated_at": "2026-08-16T00:00:00Z", + "platform": platform, + "summary": { + "parity_pass": len(ids) - len(fails), + "parity_fail": len(fails), + "compile_fail": 0, + "crash_fail": 0, + "node_fail": 0, + "skipped": 0, + "total_run": len(ids), + }, + "failures": {"parity": list(fails), "compile": [], "crash": []}, + "results": [ + {"id": i, "status": "parity_fail" if i in fails else "pass"} for i in ids + ], + } + + +def _self_test() -> int: + fails: list[str] = [] + + def check(name: str, cond: bool) -> None: + if not cond: + fails.append(name) + + m = merge([_rep(ids=("a", "b")), _rep(ids=("c",), fails=("c",))], ["s1", "s2"]) + check("results concatenated", len(m["results"]) == 3) + check("counts summed", m["summary"]["parity_pass"] == 2 and m["summary"]["parity_fail"] == 1) + check("pct recomputed (2/3 = 66.6)", m["summary"]["parity_percentage"] == 66.6) + check("failures unioned", m["failures"]["parity"] == ["c"]) + + def raises(fn) -> bool: + try: + fn() + except SystemExit: + return True + return False + + check("duplicate id is an error", raises(lambda: merge([_rep(ids=("a",)), _rep(ids=("a",))], ["s1", "s2"]))) + check("platform mismatch is an error", raises(lambda: merge([_rep("linux"), _rep("macos")], ["s1", "s2"]))) + check( + "single-shard merge preserves the harness pct formula", + merge([_rep(ids=("a", "b", "c"), fails=("c",))], ["s1"])["summary"]["parity_percentage"] == 66.6, + ) + + if fails: + print("parity_report_merge --self-test FAILED:", file=sys.stderr) + for f in fails: + print(f" - {f}", file=sys.stderr) + return 1 + print("parity_report_merge --self-test: OK (7 cases)") + return 0 + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("inputs", nargs="*", type=Path) + ap.add_argument("--expect", type=int, help="exact number of shard reports required") + ap.add_argument("--output", type=Path) + ap.add_argument("--self-test", action="store_true") + args = ap.parse_args(argv) + + if args.self_test: + return _self_test() + if not args.expect or not args.output or not args.inputs: + ap.error("--expect, --output and at least one input are required") + if len(args.inputs) != args.expect: + raise SystemExit( + f"expected exactly {args.expect} shard reports, got {len(args.inputs)}: " + f"{[str(p) for p in args.inputs]} — a missing shard artifact must fail " + f"the merge, not shrink the suite" + ) + + merged = merge([load(p) for p in args.inputs], [str(p) for p in args.inputs]) + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", encoding="utf-8") as fh: + json.dump(merged, fh, indent=1) + fh.write("\n") + s = merged["summary"] + print( + f"merged {len(args.inputs)} shard reports -> {args.output}: " + f"{s['parity_pass']} pass / {s['parity_fail']} parity_fail / " + f"{s['compile_fail']} compile_fail / {s['crash_fail']} crash / " + f"{s['node_fail']} node_fail / {s['skipped']} skipped = {s['parity_percentage']}%" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))