verify: run on a laptop without OVL_DEVICE; clearer help and errors

simple/ovl.py

@@ -1,24 +1,27 @@
-"""OpenVerifiableLLM, simple production loop: train -> verify segments -> chat.
-
-One file, one GPU. Reuses the project's prepared streams, batch order and model:
-  G03 reproducible init     seeded CPU init (no pretrained weights); `verify` regenerates it
-  G04 exactly one pass      every target id checked in order; Wikipedia covered exactly once
-  G05 committed progress    chain.jsonl: hash-linked state digests at fixed boundaries, and
-                            every boundary checkpoint kept so anyone can start from it
-  G06 conversation phase    same loop on the conversation stream, fresh AdamW at transition
-  G07 exact replay          split into segments (boundary k -> k+1). Each segment is checked
-                            independently, bit for bit, so verification runs in parallel and
-                            the model is usable as soon as training ends.
-
-Usage (on a pod):
+"""OpenVerifiableLLM: train a model so that anyone can recompute its training bit for bit.
+
+How a run is built:
+  start          a seeded random initialization; `verify --data` regenerates it
+  one pass       every Wikipedia token is trained on exactly once, in a fixed order
+  the record     chain.jsonl: at each boundary, a hash of the full model and optimizer state,
+                 linked to the previous entry. Every boundary checkpoint is kept
+  conversation   the same loop on the conversation data, with a fresh optimizer
+  segments       the training between two boundaries. Each one can be recomputed on its own,
+                 from its starting checkpoint, and compared with the record bit for bit
+
+On any computer, no GPU needed:
+  python simple/ovl.py verify --run RUN --weights RUN/conversation-model.pt   # release vs. record
+  python simple/ovl.py chat   --run RUN --tokenizer PREP/tokenizer/tokenizer.json "What is Wikipedia?"
+
+On an RTX 5090 (bit-exact results need the training GPU type):
   python simple/ovl.py train  --data PREP --out RUN
   python simple/ovl.py verify --data PREP --run RUN --random 2        # spot check
   python simple/ovl.py verify --data PREP --run RUN --segment 7 8     # chosen segments
   python simple/ovl.py verify --data PREP --run RUN --all             # full replay
-  python simple/ovl.py verify --run RUN --weights RUN/conversation-model.pt
-  python simple/ovl.py chat   --run RUN --tokenizer PREP/tokenizer/tokenizer.json "What is Wikipedia?"
   python simple/ovl.py bench  --data PREP --seconds 600
 
+RUN is a run directory (chain.jsonl, anchor.json, models, checkpoints); PREP is the prepared data.
+
 Each new chain entry is also timestamped by two RFC 3161 authorities while training runs
 (simple/stamp.py; OVL_TIMESTAMP=0 turns it off), so the record shows when each boundary existed.
 """
@@ -55,14 +58,24 @@
                        "conversation": "0.0001"},
           "model": {"attn_impl": "manual", "dropout": 0, "embed_dim": 384, "max_seq_len": 512,
                     "num_heads": 6, "num_layers": 6, "vocab_size": 32000}}
-DEV = os.environ.get("OVL_DEVICE", "cuda")  # "cpu" only for smoke tests
+# The GPU when there is one. Training and segment checks need it (see require_gpu); checking
+# a release's weights and record works on a CPU.
+DEV = os.environ.get("OVL_DEVICE") or ("cuda" if torch.cuda.is_available() else "cpu")
 if os.environ.get("OVL_BOUNDARY_EVERY"):  # smoke tests only
     RECIPE["boundary_every"] = int(os.environ["OVL_BOUNDARY_EVERY"])
     RECIPE["schedule"].update(warmup=2, wikipedia_updates=4)
 STOP = "stop"
 TIMESTAMP = os.environ.get("OVL_TIMESTAMP", "1") != "0"
 
 
+def require_gpu(what):
+    """Stop before work that only reproduces the published bits on the training GPU type, unless
+    OVL_DEVICE=cpu asks for a CPU run on purpose (tests and smoke runs)."""
+    if DEV == "cpu" and os.environ.get("OVL_DEVICE") != "cpu":
+        sys.exit(f"{what} needs an NVIDIA GPU, and none was found. The release reproduces bit for bit only "
+                 "on an RTX 5090 (see docs/verify.md). To run on the CPU anyway, for a test, set OVL_DEVICE=cpu.")
+
+
 def stages():
     return [("wikipedia", 0)] + [("conversation", e) for e in range(RECIPE["conversation_epochs"])]
 
@@ -410,6 +423,7 @@ def load_checkpoint(run, entry, model):
 
 
 def cmd_train(a):
+    require_gpu("Training")
     out = Path(a.out)
     out.mkdir(parents=True, exist_ok=True)
     # A crash right after the file was opened can leave it empty: that is a run with no entries yet.
@@ -581,8 +595,18 @@ def check_anchor_timestamps(anchor, data):
 
 
 def cmd_verify(a):
+    if a.data and (a.all or a.segment or a.random):
+        require_gpu("Recomputing a segment")
+    if a.weights and not Path(a.weights).is_file():
+        sys.exit(f"weights file {a.weights} not found")
     run = Path(a.run)
     chain = read_chain(run)
+    recipe = hashlib.sha256(json.dumps(RECIPE, sort_keys=True).encode()).hexdigest()
+    if a.data and chain[0].get("recipe") not in (None, recipe):  # fail before any GPU time is spent
+        sys.exit("this ovl.py's RECIPE differs from the one the chain was trained with; use ovl.py from the release's tag")
+    for phase in ("wikipedia", "conversation") if a.data else ():  # before the anchor's network checks
+        if not (Path(a.data) / phase / "stream.json").is_file():
+            sys.exit(f"{a.data} is not a prepared data directory: {phase}/stream.json not found")
     anchor = Path(a.anchor) if a.anchor else run / "anchor.json"
     checks = []
     if anchor.exists():
@@ -607,9 +631,6 @@ def cmd_verify(a):
     else:
         print("WARNING: --no-anchor: the chain is not compared with a timestamped anchor", flush=True)
     report = {"time": time.time(), "environment": environment(), "chain_head": chain[-1]["state"], "checks": checks}
-    recipe = hashlib.sha256(json.dumps(RECIPE, sort_keys=True).encode()).hexdigest()
-    if a.data and chain[0].get("recipe") not in (None, recipe):  # fail before any GPU time is spent
-        sys.exit("this ovl.py's RECIPE differs from the one the chain was trained with; use ovl.py from the release's tag")
     if a.data and chain[0]["data"] != {p: hashlib.sha256((Path(a.data) / p / "stream.json").read_bytes()).hexdigest()
                                        for p in ("wikipedia", "conversation")}:
         sys.exit("prepared data differs from the data the chain trained on (the stream.json hashes in entry 0)")
@@ -639,11 +660,15 @@ def cmd_verify(a):
         f.write(json.dumps(report, sort_keys=True) + "\n")
     print(f"verify {report['result']}: {len(report['checks'])} checks; "
           f"{sum('segment' in c for c in report['checks'])} of {len(chain) - 1} segments", flush=True)
+    if report["result"] == "PASS" and not any("segment" in c for c in report["checks"]):
+        print("The release matches its record. No training was recomputed: to check a segment, "
+              "see docs/verify.md (needs an RTX 5090).", flush=True)
     if report["result"] != "PASS":
         sys.exit(1)
 
 
 def cmd_bench(a):
+    require_gpu("bench")
     model, opt = fresh()
     ctl, n, t0 = initial_control(), 0, None
     for b, _ in prefetch(a.data, "wikipedia", [0, 0]):
@@ -690,11 +715,16 @@ def main():
     p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     s = p.add_subparsers(dest="cmd", required=True)
     t = s.add_parser("train"); t.add_argument("--data", required=True); t.add_argument("--out", required=True)
-    v = s.add_parser("verify"); v.add_argument("--run", required=True); v.add_argument("--data")
-    v.add_argument("--segment", type=int, nargs="*"); v.add_argument("--random", type=int, default=0)
-    v.add_argument("--all", action="store_true"); v.add_argument("--weights")
+    v = s.add_parser("verify", help="check a run or a release against its record")
+    v.add_argument("--run", required=True, help="the run directory: chain.jsonl, anchor.json, models")
+    v.add_argument("--data", help="the prepared data directory; needed to recompute segments")
+    v.add_argument("--segment", type=int, nargs="*", help="recompute these segments (k means boundary k to k+1)")
+    v.add_argument("--random", type=int, default=0, help="also recompute this many randomly chosen segments")
+    v.add_argument("--all", action="store_true", help="recompute every segment")
+    v.add_argument("--weights", help="check that this model file (.pt or .safetensors) is in the record")
     anchored = v.add_mutually_exclusive_group()
-    anchored.add_argument("--anchor"); anchored.add_argument("--no-anchor", action="store_true")
+    anchored.add_argument("--anchor", help="the timestamped anchor to compare with (default: RUN/anchor.json)")
+    anchored.add_argument("--no-anchor", action="store_true", help="only for a run that has no anchor yet")
     v.add_argument("--no-timestamps", action="store_true", help="skip the anchor's RFC 3161 timestamp check")
     fk = s.add_parser("fork"); fk.add_argument("--run", required=True); fk.add_argument("--data", required=True)
     fk.add_argument("--at", type=int, required=True); fk.add_argument("--exclude-page", nargs="+", required=True)

tests/simple/test_cli.py

@@ -0,0 +1,97 @@
+"""The command line on a computer without a GPU: what runs, what stops early, and the messages."""
+import argparse
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+os.environ["OVL_DEVICE"] = "cpu"
+os.environ["OVL_TIMESTAMP"] = "0"
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT / "simple"))
+import ovl  # noqa: E402
+
+
+def verify_args(**kw):
+    base = dict(run=None, data=None, segment=None, random=0, all=False, weights=None, anchor=None,
+                no_anchor=True, no_timestamps=False)
+    return argparse.Namespace(**{**base, **kw})
+
+
+class RequireGpuTests(unittest.TestCase):
+    def test_stops_on_cpu_when_no_device_was_chosen(self):
+        with mock.patch.object(ovl, "DEV", "cpu"), mock.patch.dict(os.environ, clear=False) as env:
+            env.pop("OVL_DEVICE", None)
+            with self.assertRaisesRegex(SystemExit, "Training needs an NVIDIA GPU.*OVL_DEVICE=cpu"):
+                ovl.require_gpu("Training")
+
+    def test_an_empty_device_setting_is_not_a_cpu_choice(self):
+        with mock.patch.object(ovl, "DEV", "cpu"), mock.patch.dict(os.environ, {"OVL_DEVICE": ""}):
+            with self.assertRaisesRegex(SystemExit, "Training needs an NVIDIA GPU"):
+                ovl.require_gpu("Training")
+
+    def test_runs_on_cpu_when_asked_explicitly(self):
+        with mock.patch.object(ovl, "DEV", "cpu"), mock.patch.dict(os.environ, {"OVL_DEVICE": "cpu"}):
+            ovl.require_gpu("Training")
+
+    def test_runs_on_a_gpu(self):
+        with mock.patch.object(ovl, "DEV", "cuda"), mock.patch.dict(os.environ, clear=False) as env:
+            env.pop("OVL_DEVICE", None)
+            ovl.require_gpu("Training")
+
+    def test_segment_check_without_a_gpu_stops_before_reading_anything(self):
+        with mock.patch.object(ovl, "DEV", "cpu"), mock.patch.dict(os.environ, clear=False) as env:
+            env.pop("OVL_DEVICE", None)
+            with self.assertRaisesRegex(SystemExit, "Recomputing a segment needs an NVIDIA GPU"):
+                ovl.cmd_verify(verify_args(run="/nonexistent", data="/nonexistent", random=1))
+
+
+class DeviceDefaultTests(unittest.TestCase):
+    def test_falls_back_to_cpu_without_a_gpu(self):
+        env = {k: v for k, v in os.environ.items() if k != "OVL_DEVICE"}
+        code = "import sys; sys.path.insert(0, 'simple'); import torch; torch.cuda.is_available = lambda: False; import ovl; print(ovl.DEV)"
+        out = subprocess.run([sys.executable, "-c", code], cwd=ROOT, env=env, capture_output=True, text=True, check=True)
+        self.assertEqual(out.stdout.strip(), "cpu")
+
+
+class MissingInputTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.run_dir = Path(self.tmp.name)
+        (self.run_dir / "chain.jsonl").write_text('{"index": 0, "kind": "initial", "state": "s", "model": "m", "previous": null}\n')
+
+    def test_missing_weights_file_is_reported(self):
+        with self.assertRaisesRegex(SystemExit, "weights file .*nope.pt not found"):
+            ovl.cmd_verify(verify_args(run=str(self.run_dir), weights=str(self.run_dir / "nope.pt")))
+
+    def test_data_directory_without_streams_is_reported(self):
+        with self.assertRaisesRegex(SystemExit, "is not a prepared data directory"):
+            ovl.cmd_verify(verify_args(run=str(self.run_dir), data=str(self.run_dir)))
+
+    def test_data_directory_without_conversation_stream_is_reported(self):
+        (self.run_dir / "wikipedia").mkdir()
+        (self.run_dir / "wikipedia" / "stream.json").write_text("{}")
+        with self.assertRaisesRegex(SystemExit, "conversation/stream.json not found"):
+            ovl.cmd_verify(verify_args(run=str(self.run_dir), data=str(self.run_dir)))
+
+    def test_data_directory_is_checked_before_the_anchor(self):
+        # An anchor that would fail (and, with timestamps on, go online) must not be reached first.
+        (self.run_dir / "anchor.json").write_text("not json")
+        with self.assertRaisesRegex(SystemExit, "is not a prepared data directory"):
+            ovl.cmd_verify(verify_args(run=str(self.run_dir), data=str(self.run_dir), no_anchor=False))
+
+
+class HelpTests(unittest.TestCase):
+    def test_verify_options_are_described(self):
+        out = subprocess.run([sys.executable, str(ROOT / "simple" / "ovl.py"), "verify", "--help"],
+                             capture_output=True, text=True, check=True).stdout
+        for words in ("the run directory", "recompute every segment", "this model file"):
+            self.assertIn(words, out)
+
+
+if __name__ == "__main__":
+    unittest.main()