diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..4faecbb2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,8 @@ +# `pre-commit install` once, then ruff runs on every commit, as in CI. +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.17.3 + hooks: + - id: ruff-check + args: [simple, tests] + pass_filenames: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38846dfd..83d46e3f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,10 +29,15 @@ git clone https://github.com//OpenVerifiableLLM cd OpenVerifiableLLM pip install --require-hashes -r requirements/preparation.lock # the same CPU environment as CI pip install pytest gguf safetensors ruff -OVL_DEVICE=cpu OVL_TIMESTAMP=0 python -m pytest -q tests # runs on a CPU, about 15 s +OVL_DEVICE=cpu OVL_TIMESTAMP=0 python -m pytest -q tests # runs on a CPU, about 25 s ruff check simple tests ``` +With `make`, the same steps are `make setup` and `make check` (`make help` lists the rest). Run +`pre-commit install` once to run ruff on every commit. With `uv` instead of pip, install the lock +with `uv pip install --index-strategy unsafe-best-match --require-hashes -r requirements/preparation.lock`: +the lock takes PyTorch from the PyTorch index and everything else from PyPI. + Training and segment checks need an RTX 5090. Everything in `tests/` runs on a CPU. To add a dependency, add it to the lock file, so that every install stays pinned by hash. diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..da19f0b6 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +# Everyday commands. The environment is the one CI uses: see CONTRIBUTING.md. +PYTHON ?= python +export OVL_DEVICE ?= cpu +export OVL_TIMESTAMP ?= 0 +export TOKENIZERS_PARALLELISM ?= false + +.PHONY: help setup test lint check check-release + +help: ## list the commands + @grep -E '^[a-z-]+:.*## ' $(MAKEFILE_LIST) | sed 's/:.*## /\t/' + +setup: ## install the hash-locked CPU environment and the dev tools into the current Python + $(PYTHON) -m pip install --require-hashes -r requirements/preparation.lock + $(PYTHON) -m pip install pytest pytest-cov ruff gguf safetensors pre-commit + +test: ## run the tests (CPU, about 25 s) + $(PYTHON) -m pytest -q tests + +lint: ## run ruff, as CI does + $(PYTHON) -m ruff check simple tests + +check: lint test ## everything CI runs + +check-release: ## check the published release on this computer (about 2 minutes) + bash simple/check_release.sh diff --git a/pyproject.toml b/pyproject.toml index 493f2f08..899aca9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,9 @@ version = "1.1.0" description = "Train a small language model on English Wikipedia so that anyone can recompute it bit for bit." readme = "README.md" requires-python = ">=3.12" -# Pinned with hashes in requirements/preparation.lock, which CI installs. +# For the tools and the CPU checks; pinned with hashes in requirements/preparation.lock, which CI +# installs. Training and segment checks don't use these: they run in the training image +# (PyTorch 2.8.0, CUDA 12.8), the only setup that reproduces the published bits. dependencies = ["torch>=2.10.0,<3.0", "numpy>=2.4,<3.0", "tokenizers==0.23.2"] [project.optional-dependencies]