diff --git a/.github/workflows/public-benchmark.yml b/.github/workflows/public-benchmark.yml new file mode 100644 index 00000000..fdbc1cf5 --- /dev/null +++ b/.github/workflows/public-benchmark.yml @@ -0,0 +1,56 @@ +name: Public benchmark + +# Manually refresh the shareable benchmark numbers (#4). Runs the scaling +# harness (bench_report.py) and the fixture harness (bench.py) through +# benchmarks/public_benchmark.py and uploads one markdown + JSON report stamped +# with the freshdata version and runner hardware. Manual only: it never runs +# on push or pull requests and never blocks CI. +on: + workflow_dispatch: + inputs: + scale: + description: "Scaling-case sizes: ci (fits a hosted runner) or full (harness defaults, 10M rows)" + type: choice + options: [ci, full] + default: ci + repeat: + description: "Timing repeats for the fixture benchmarks" + type: string + default: "5" + +permissions: + contents: read + +jobs: + public-benchmark: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + cache: pip + - name: Install + run: | + python -m pip install --upgrade pip + pip install -c constraints/ci.txt -e ".[bench]" + - name: Run public benchmark + env: + SCALE: ${{ inputs.scale }} + REPEAT: ${{ inputs.repeat }} + run: | + python benchmarks/public_benchmark.py \ + --scale "$SCALE" --repeat "$REPEAT" \ + --output-dir benchmarks/results/public + cat benchmarks/results/public/public-benchmark.md >> "$GITHUB_STEP_SUMMARY" + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: public-benchmark-${{ github.run_id }} + path: | + benchmarks/results/public/ + benchmarks/results/report_bench.json + if-no-files-found: error + retention-days: 90 diff --git a/benchmarks/public_benchmark.py b/benchmarks/public_benchmark.py new file mode 100644 index 00000000..15d66cd4 --- /dev/null +++ b/benchmarks/public_benchmark.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python +"""Public benchmark refresh: one reproducible report for sharing numbers. + +Runs the two existing harnesses in subprocesses and folds their output into a +single markdown + JSON report stamped with the freshdata version and the +environment it ran on (no numbers are ever committed; see docs/benchmarks.md): + +* ``benchmarks/bench_report.py`` — the scaling cases behind the + "Strategic-report scaling benchmarks" table (CSV ingest, profile, null-fill, + import time, peak memory), balanced vs aggressive; +* ``benchmarks/bench.py run`` + ``report`` — the per-fixture quality/speed table. + +Usage:: + + python benchmarks/public_benchmark.py # CI-safe sizes + python benchmarks/public_benchmark.py --scale full # harness defaults (10M rows) + python benchmarks/public_benchmark.py --skip-scaling --repeat 1 +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import subprocess +import sys +from datetime import datetime, timezone +from importlib import metadata +from pathlib import Path +from typing import Any, Callable + +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parent +RESULTS_DIR = HERE / "results" +SCALING_JSON = RESULTS_DIR / "report_bench.json" + +#: CLI presets. ``ci`` fits a 2-core / 7 GB GitHub runner; ``full`` uses the +#: harness defaults (100 MB CSV, 10M-row null-fill) for a workstation. +SCALES: dict[str, dict[str, int | None]] = { + "ci": {"csv_mb": 50, "rows": 1_000_000}, + "full": {"csv_mb": 100, "rows": None}, +} + +_SCALING_LABELS = { + "csv_ingest": "CSV ingest + clean", + "profile": "Mixed-schema profile", + "nullfill": "Null-fill / flag", + "import_time": "Import time (`import freshdata`)", + "memory": "Peak memory of a full clean", +} + +Runner = Callable[[list[str]], None] + + +def _run(cmd: list[str]) -> None: + print("+", " ".join(cmd), flush=True) + subprocess.run(cmd, cwd=REPO_ROOT, check=True) # noqa: S603 - fixed argv + + +def _version(dist: str) -> str: + try: + return metadata.version(dist) + except metadata.PackageNotFoundError: + return "not installed" + + +def collect_environment() -> dict[str, Any]: + import freshdata # noqa: PLC0415 - report the version actually benchmarked + + ram = None + try: + import psutil # noqa: PLC0415 + + ram = round(psutil.virtual_memory().total / 1024**3, 1) + except ImportError: # pragma: no cover - psutil is a core dependency + pass + sha = os.environ.get("GITHUB_SHA") + if not sha: + try: + sha = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=REPO_ROOT, capture_output=True, + text=True, check=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + sha = "unknown" + return { + "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "freshdata_version": freshdata.__version__, + "git_sha": sha, + "python": platform.python_version(), + "platform": platform.platform(), + "cpu_count": os.cpu_count(), + "ram_gb": ram, + "packages": { + name: _version(name) for name in ("pandas", "numpy", "pyarrow", "polars", "duckdb") + }, + } + + +# --------------------------------------------------------------------------- # +# harnesses +# --------------------------------------------------------------------------- # + +def run_scaling(*, csv_mb: int, rows: int | None, runner: Runner = _run) -> list[dict[str, Any]]: + cmd = [sys.executable, "benchmarks/bench_report.py", "all", "--mb", str(csv_mb)] + if rows is not None: + cmd += ["--rows", str(rows)] + runner(cmd) + return json.loads(SCALING_JSON.read_text(encoding="utf-8"))["results"] + + +def run_fixtures(*, repeat: int, size: int | None, runner: Runner = _run) -> str: + before = {p.name for p in RESULTS_DIR.iterdir()} if RESULTS_DIR.exists() else set() + cmd = [sys.executable, "benchmarks/bench.py", "run", "--repeat", str(repeat)] + if size is not None: + cmd += ["--size", str(size)] + runner(cmd) + new_runs = sorted( + p for p in RESULTS_DIR.iterdir() if p.is_dir() and p.name not in before + ) + if not new_runs: + raise RuntimeError("bench.py run did not create a results directory") + run_dir = new_runs[-1] + runner([sys.executable, "benchmarks/bench.py", "report", "--run-dir", str(run_dir)]) + return (run_dir / "report.md").read_text(encoding="utf-8") + + +# --------------------------------------------------------------------------- # +# rendering +# --------------------------------------------------------------------------- # + +def _cell(result: dict[str, Any] | None) -> str: + if result is None: + return "n/a" + if "import_sec_median" in result: + median, fastest = result["import_sec_median"], result.get("import_sec_min", 0) + return f"{median:.3f} s median ({fastest:.3f} s min)" + parts = [] + if "wall_sec" in result: + parts.append(f"{result['wall_sec']:.2f} s") + if result.get("peak_rss_mb") is not None: + parts.append(f"{result['peak_rss_mb']:.0f} MB peak RSS") + return ", ".join(parts) or "measured" + + +def _size(results: list[dict[str, Any]]) -> str: + first = results[0] + if first.get("csv_mb"): + return f"{first['csv_mb']} MB" + if first.get("rows"): + return f"{first['rows']:,} rows" + return "" + + +def scaling_table(results: list[dict[str, Any]]) -> list[str]: + lines = ["| Benchmark | Size | Balanced | Aggressive |", "|---|---|---|---|"] + for key, label in _SCALING_LABELS.items(): + rows = [r for r in results if r.get("benchmark") == key] + if not rows: + continue + by_strategy = {r.get("strategy"): r for r in rows} + balanced = by_strategy.get("balanced") or by_strategy.get("n/a") + aggressive = by_strategy.get("aggressive") + lines.append(f"| {label} | {_size(rows)} | {_cell(balanced)} | {_cell(aggressive)} |") + return lines + + +def render_markdown( + environment: dict[str, Any], + scaling: list[dict[str, Any]] | None, + fixture_report: str | None, +) -> str: + packages = ", ".join(f"{k} {v}" for k, v in environment["packages"].items()) + lines = [ + f"# FreshData public benchmark — freshdata {environment['freshdata_version']}", + "", + f"- **freshdata:** `{environment['freshdata_version']}` " + f"(commit `{environment['git_sha'][:12]}`)", + f"- **Generated:** {environment['generated_at']}", + f"- **Hardware:** {environment['cpu_count']} CPUs, {environment['ram_gb']} GB RAM", + f"- **Software:** Python {environment['python']} on {environment['platform']}; {packages}", + "", + "Numbers are environment-specific. Re-run `python benchmarks/public_benchmark.py` " + "on your own hardware before quoting them.", + "", + ] + if scaling: + lines += ["## Strategic-report scaling benchmarks", "", *scaling_table(scaling), ""] + if fixture_report: + body = fixture_report + if body.startswith("# "): + body = body.split("\n", 1)[1] # drop the harness report's own title + lines += ["## Fixture benchmarks (`benchmarks/bench.py`)", "", body.strip(), ""] + return "\n".join(lines) + + +def main(argv: list[str] | None = None, *, runner: Runner = _run) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--scale", choices=sorted(SCALES), default="ci", + help="size preset for the scaling cases (default: ci)") + parser.add_argument("--csv-mb", type=int, default=None, help="override the CSV ingest size") + parser.add_argument("--rows", type=int, default=None, + help="override profile/null-fill/memory row counts") + parser.add_argument("--repeat", type=int, default=5, help="bench.py timing repeats") + parser.add_argument("--size", type=int, default=None, help="bench.py row-count override") + parser.add_argument("--skip-scaling", action="store_true") + parser.add_argument("--skip-fixtures", action="store_true") + parser.add_argument("--output-dir", default=str(RESULTS_DIR / "public"), + help="where public-benchmark.md/json are written") + args = parser.parse_args(argv) + if args.skip_scaling and args.skip_fixtures: + parser.error("nothing to run: both --skip-scaling and --skip-fixtures given") + + preset = SCALES[args.scale] + csv_mb = args.csv_mb if args.csv_mb is not None else preset["csv_mb"] + rows = args.rows if args.rows is not None else preset["rows"] + + environment = collect_environment() + scaling = None + if not args.skip_scaling: + scaling = run_scaling(csv_mb=int(csv_mb), rows=rows, runner=runner) + fixture_report = None if args.skip_fixtures else run_fixtures( + repeat=args.repeat, size=args.size, runner=runner) + + out_dir = Path(args.output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + markdown = render_markdown(environment, scaling, fixture_report) + (out_dir / "public-benchmark.md").write_text(markdown, encoding="utf-8") + (out_dir / "public-benchmark.json").write_text(json.dumps({ + "environment": environment, + "options": {"scale": args.scale, "csv_mb": csv_mb, "rows": rows, + "repeat": args.repeat, "size": args.size}, + "scaling": scaling, + "fixture_report_markdown": fixture_report, + }, indent=2) + "\n", encoding="utf-8") + print(markdown) + print(f"written: {out_dir / 'public-benchmark.md'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index e3bba722..8acd819a 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -115,6 +115,23 @@ python benchmarks/bench_report.py import_time # cold `import freshdata python benchmarks/bench_report.py memory --rows 1000000 # peak-RSS of a full clean ``` +#### Refreshing the public numbers + +`benchmarks/public_benchmark.py` runs these scaling cases together with the +per-fixture harness (`bench.py`) and writes one shareable report, +`benchmarks/results/public/public-benchmark.md` (+ `.json`), stamped with the +freshdata version, commit, hardware and package versions: + +```bash +python benchmarks/public_benchmark.py # CI-safe sizes (1M rows, 50 MB CSV) +python benchmarks/public_benchmark.py --scale full # the full sizes listed above +``` + +To produce it on a GitHub-hosted runner, dispatch the manual **Public benchmark** +workflow (`gh workflow run public-benchmark.yml`); the report is attached as the +`public-benchmark-` artifact and shown in the run summary. It never runs +on pushes or pull requests. + ### Results — not yet measured These numbers are **environment-specific and are not committed**. Run the diff --git a/tests/benchmark/test_public_benchmark.py b/tests/benchmark/test_public_benchmark.py new file mode 100644 index 00000000..2554c5ef --- /dev/null +++ b/tests/benchmark/test_public_benchmark.py @@ -0,0 +1,136 @@ +"""Public benchmark report (#4): rendering, harness orchestration, workflow contract.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import public_benchmark as pb +import pytest +import yaml + +import freshdata + +ENV = { + "generated_at": "2026-09-15T00:00:00+00:00", + "freshdata_version": "9.9.9", + "git_sha": "abcdef1234567890", + "python": "3.12.0", + "platform": "Linux-6.8-x86_64", + "cpu_count": 4, + "ram_gb": 15.6, + "packages": {"pandas": "2.3.3", "numpy": "2.5.1"}, +} + +SCALING = [ + {"benchmark": "csv_ingest", "strategy": "balanced", "rows": 500_000, + "csv_mb": 48.2, "wall_sec": 3.21, "peak_rss_mb": 410.0}, + {"benchmark": "csv_ingest", "strategy": "aggressive", "rows": 500_000, + "csv_mb": 48.2, "wall_sec": 3.9, "peak_rss_mb": 455.0}, + {"benchmark": "profile", "strategy": "n/a", "rows": 1_000_000, + "wall_sec": 1.5, "peak_rss_mb": 120.0}, + {"benchmark": "import_time", "strategy": "n/a", "rows": 0, + "import_sec_median": 0.412, "import_sec_min": 0.39}, +] + +FIXTURE_MD = ( + "# FreshData benchmark report — `run1`\n\n" + "- freshdata: `9.9.9`\n\n" + "| fixture | n_rows |\n|---|--:|\n| crm | 10,000 |\n" +) + + +def test_markdown_carries_version_environment_and_both_tables(): + md = pb.render_markdown(ENV, SCALING, FIXTURE_MD) + assert md.startswith("# FreshData public benchmark — freshdata 9.9.9") + assert "(commit `abcdef123456`)" in md + assert "4 CPUs, 15.6 GB RAM" in md + assert "pandas 2.3.3, numpy 2.5.1" in md + assert ( + "| CSV ingest + clean | 48.2 MB | 3.21 s, 410 MB peak RSS | 3.90 s, 455 MB peak RSS |" + ) in md + assert "| Mixed-schema profile | 1,000,000 rows | 1.50 s, 120 MB peak RSS | n/a |" in md + assert "| Import time (`import freshdata`) | | 0.412 s median (0.390 s min) | n/a |" in md + assert "## Fixture benchmarks" in md + assert "| crm | 10,000 |" in md + assert "# FreshData benchmark report" not in md # nested report title dropped + + +def test_scaling_table_skips_cases_that_did_not_run(): + table = "\n".join(pb.scaling_table(SCALING)) + assert "Null-fill" not in table + assert "Peak memory" not in table + + +def _fake_harness(monkeypatch, tmp_path): + results_dir = tmp_path / "results" + results_dir.mkdir() + monkeypatch.setattr(pb, "RESULTS_DIR", results_dir) + monkeypatch.setattr(pb, "SCALING_JSON", results_dir / "report_bench.json") + monkeypatch.setattr(pb, "collect_environment", lambda: ENV) + calls: list[list[str]] = [] + + def runner(cmd): + args = cmd[1:] + calls.append(args) + if args[0] == "benchmarks/bench_report.py": + pb.SCALING_JSON.write_text(json.dumps({"results": SCALING})) + elif args[:2] == ["benchmarks/bench.py", "run"]: + (results_dir / "20260915T000000").mkdir() + elif args[:2] == ["benchmarks/bench.py", "report"]: + (Path(args[-1]) / "report.md").write_text(FIXTURE_MD) + + return runner, calls + + +def test_main_runs_both_harnesses_and_writes_report(monkeypatch, tmp_path): + runner, calls = _fake_harness(monkeypatch, tmp_path) + out = tmp_path / "public" + assert pb.main(["--repeat", "1", "--output-dir", str(out)], runner=runner) == 0 + assert calls[0] == ["benchmarks/bench_report.py", "all", "--mb", "50", "--rows", "1000000"] + assert calls[1] == ["benchmarks/bench.py", "run", "--repeat", "1"] + assert calls[2][:2] == ["benchmarks/bench.py", "report"] + md = (out / "public-benchmark.md").read_text() + assert "| CSV ingest + clean |" in md and "| crm | 10,000 |" in md + payload = json.loads((out / "public-benchmark.json").read_text()) + assert payload["environment"]["freshdata_version"] == "9.9.9" + assert payload["options"] == {"scale": "ci", "csv_mb": 50, "rows": 1_000_000, + "repeat": 1, "size": None} + + +def test_full_scale_uses_harness_default_rows(monkeypatch, tmp_path): + runner, calls = _fake_harness(monkeypatch, tmp_path) + pb.main(["--scale", "full", "--skip-fixtures", "--output-dir", str(tmp_path / "o")], + runner=runner) + assert calls == [["benchmarks/bench_report.py", "all", "--mb", "100"]] + + +def test_skip_scaling_runs_only_the_fixture_harness(monkeypatch, tmp_path): + runner, calls = _fake_harness(monkeypatch, tmp_path) + out = tmp_path / "public" + pb.main(["--skip-scaling", "--repeat", "1", "--output-dir", str(out)], runner=runner) + assert all(call[0] == "benchmarks/bench.py" for call in calls) + assert "scaling benchmarks" not in (out / "public-benchmark.md").read_text() + + +def test_skipping_everything_is_an_error(): + with pytest.raises(SystemExit): + pb.main(["--skip-scaling", "--skip-fixtures"], runner=lambda cmd: None) + + +def test_environment_reports_the_installed_freshdata_version(): + env = pb.collect_environment() + assert env["freshdata_version"] == freshdata.__version__ + assert env["cpu_count"] + + +def test_workflow_is_manual_only_and_uploads_the_report(): + path = Path(pb.REPO_ROOT) / ".github" / "workflows" / "public-benchmark.yml" + text = path.read_text(encoding="utf-8") + workflow = yaml.safe_load(text) + # PyYAML 1.1 treats the unquoted GitHub Actions key `on` as a boolean. + triggers = workflow.get("on", workflow.get(True)) + assert set(triggers) == {"workflow_dispatch"} + steps = workflow["jobs"]["public-benchmark"]["steps"] + assert any("upload-artifact" in step.get("uses", "") for step in steps) + assert "benchmarks/public_benchmark.py" in text