From 4d57e5099a2a5de6dec4439f276a1e1cf66b30ea Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sat, 4 Jul 2026 22:06:30 +0800 Subject: [PATCH] docs: add MIT LICENSE; rename the guide to CONTRIBUTING.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add MIT LICENSE (matches the problem-reductions library). - Rename SUBMISSION.md → CONTRIBUTING.md so GitHub surfaces it as a tab on the repo overview (custom filenames like SUBMISSION.md are never shown there); update all references (README links, Makefile, Dockerfile, submissions/README, test_docs). Co-Authored-By: Claude Opus 4.8 (1M context) --- SUBMISSION.md => CONTRIBUTING.md | 0 LICENSE | 21 +++++++++++++++++++++ Makefile | 4 ++-- README.md | 4 ++-- benchmark/tests/test_docs.py | 18 +++++++++--------- docker/Dockerfile | 2 +- submissions/README.md | 2 +- 7 files changed, 36 insertions(+), 15 deletions(-) rename SUBMISSION.md => CONTRIBUTING.md (100%) create mode 100644 LICENSE diff --git a/SUBMISSION.md b/CONTRIBUTING.md similarity index 100% rename from SUBMISSION.md rename to CONTRIBUTING.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..48d312c --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Jin-Guo Liu and other contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile index 462dded..06d0d86 100644 --- a/Makefile +++ b/Makefile @@ -56,14 +56,14 @@ preflight: ## Run the budgeted bug-finding agent via Docker → writes ./out/submission.json. ## This RUNS the benchmark locally; it does NOT submit — submitting is a separate step -## (open a GitHub PR adding the file, see SUBMISSION.md). Config lives in submission.env +## (open a GitHub PR adding the file, see CONTRIBUTING.md). Config lives in submission.env ## (copy submission.env.example); run `make preflight` first to validate it. run: @if [ ! -f "$(ENV_FILE)" ]; then \ echo "No $(ENV_FILE) — copy submission.env.example and fill it in (then: make preflight)"; exit 1; fi mkdir -p out docker run --rm --env-file "$(ENV_FILE)" -v "$(PWD)/out:/out" $(IMAGE) - @echo "Wrote ./out/submission.json — now submit it via a GitHub PR (see SUBMISSION.md)." + @echo "Wrote ./out/submission.json — now submit it via a GitHub PR (see CONTRIBUTING.md)." ## Score all submissions in SUBS_DIR with the zero-trust backend (needs pred). ## Writes scored results + leaderboard.json into SCORED. diff --git a/README.md b/README.md index 66e56e8..304aad4 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A benchmark that measures how efficiently AI models find bugs in reduction rules from the [problem-reductions](https://github.com/CodingThrust/problem-reductions) library (290+ rules). -The leaderboard is a static site (`site/`) published to **GitHub Pages**. Submitting is a **GitHub pull request** that adds `submissions//.json`; the submission is re-verified on the PR (after a maintainer approves the run) and the result must pass before merge. See `SUBMISSION.md` to run and submit. +The leaderboard is a static site (`site/`) published to **GitHub Pages**. Submitting is a **GitHub pull request** that adds `submissions//.json`; the submission is re-verified on the PR (after a maintainer approves the run) and the result must pass before merge. See [CONTRIBUTING.md](CONTRIBUTING.md) to run and submit. ## What this measures @@ -46,7 +46,7 @@ class MyRunner(AgentRunner): Then pass it to `Scheduler` in `benchmark/scheduler.py`. See `MiniSweRunner` for a full example. -A run is packaged as a `submission.json` (envelope around the per-rule rows, see `benchmark/submission.schema.json`) and submitted as a **GitHub pull request** that adds it under `submissions/`; the PR re-verifies every certificate with `pred` (after a maintainer approves the run) and must pass before merge. See `SUBMISSION.md`. +A run is packaged as a `submission.json` (envelope around the per-rule rows, see `benchmark/submission.schema.json`) and submitted as a **GitHub pull request** that adds it under `submissions/`; the PR re-verifies every certificate with `pred` (after a maintainer approves the run) and must pass before merge. See [CONTRIBUTING.md](CONTRIBUTING.md). ## How to run locally diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index a179a79..31a2915 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -1,5 +1,5 @@ """ -Structural checks for the two root docs: README.md (overview) and SUBMISSION.md +Structural checks for the two root docs: README.md (overview) and CONTRIBUTING.md (the single run-and-submit guide). No rendering, no external calls. All tests are marked @pytest.mark.judgment. """ @@ -10,7 +10,7 @@ REPO_ROOT = Path(__file__).parent.parent.parent README = REPO_ROOT / "README.md" -SUBMISSION = REPO_ROOT / "SUBMISSION.md" +GUIDE = REPO_ROOT / "CONTRIBUTING.md" def _text(path: Path) -> str: @@ -38,14 +38,14 @@ def test_readme_has_metrics_section(self): assert "bugs/ktok" in t or "bugs_per_ktok" in t -class TestSubmission: - def test_submission_exists(self): - assert SUBMISSION.exists(), "SUBMISSION.md missing from repo root" +class TestGuide: + def test_guide_exists(self): + assert GUIDE.exists(), "CONTRIBUTING.md (run-and-submit guide) missing from repo root" - def test_submission_has_certificate_format(self): - t = _text(SUBMISSION) + def test_guide_has_certificate_format(self): + t = _text(GUIDE) assert "source" in t and "bundle" in t and "violation" in t - def test_submission_has_github_pr_flow(self): - t = _text(SUBMISSION) + def test_guide_has_github_pr_flow(self): + t = _text(GUIDE) assert "pull request" in t and "github pages" in t diff --git a/docker/Dockerfile b/docker/Dockerfile index 3a7d5b7..36e2e49 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -80,7 +80,7 @@ CMD ["--calibrate"] # Run (all config — any provider — in submission.env; key never baked in): # docker run --rm --env-file submission.env -v "$PWD/out:/out" \ # problem-reductions-runner:v0.6.0 -# # → ./out/submission.json (see submission.env.example / SUBMISSION.md) +# # → ./out/submission.json (see submission.env.example / CONTRIBUTING.md) FROM runtime AS runner ARG PR_REF # Agent stack. requirements.txt pins mini-swe-agent (which pulls litellm) + anthropic. diff --git a/submissions/README.md b/submissions/README.md index 549229a..812a82e 100644 --- a/submissions/README.md +++ b/submissions/README.md @@ -9,7 +9,7 @@ submissions//.json ## How to submit 1. **Produce the file.** Run the dockerized runner at the fixed $20 budget (see - `../SUBMISSION.md`): `make run` → `out/submission.json`. + `../CONTRIBUTING.md`): `make run` → `out/submission.json`. 2. **Open a PR** adding it as `submissions//.json` — and nothing else (a PR that also changes code won't be auto-scored; keep submissions in their own PR). - An automated check validates the file against `benchmark/submission.schema.json`