From 29449c66ce1034468eac0d16a86d7bd27f14dd4d Mon Sep 17 00:00:00 2001 From: Jon Froehlich Date: Sun, 26 Jul 2026 18:25:00 -0700 Subject: [PATCH 1/2] Add the supervised YOLO baseline to the comparison harness (#51) The architecture-vs-data ablation for issue #51: does a generic detector trained on the RampNet dataset also beat the zero-shot field, or is RampNet's keypoint architecture doing the work? - YoloDetector + yolo_results_to_boxes in detectors.py, plus the `yolo` branch in build_detector. Identity is the weights file stem, so the detection cache key is machine-independent; signature() hashes the weights bytes, so a retrain busts the cache. It emits scored boxes, so YOLO gets AP, PR curves and --sweep like the other open detectors. - compare.py: --yolo-model / --yolo-conf (0.05 floor) / --yolo-iou / --yolo-imgsz. requirements-vlm.txt: ultralytics. - prepare_yolo_dataset.py: tiles+pano dataset builder with pluggable box-size strategies and deterministic background thinning. - run_yolo_{data,prep,train}.slurm and hyak_yolo_runbook.sh (repo root, beside hyak_qwen_runbook.sh). - +10 tests. The training record these scripts produced is PR #77, kept separate so this PR is reviewable as code. Co-Authored-By: Claude Opus 5 (1M context) --- hyak_yolo_runbook.sh | 192 ++++++++++ requirements-vlm.txt | 7 + scripts/model_comparison/compare.py | 21 +- scripts/model_comparison/detectors.py | 139 ++++++- .../model_comparison/prepare_yolo_dataset.py | 352 ++++++++++++++++++ scripts/model_comparison/run_yolo_data.slurm | 42 +++ scripts/model_comparison/run_yolo_prep.slurm | 53 +++ scripts/model_comparison/run_yolo_train.slurm | 120 ++++++ tests/test_model_comparison.py | 159 +++++++- 9 files changed, 1076 insertions(+), 9 deletions(-) create mode 100644 hyak_yolo_runbook.sh create mode 100644 scripts/model_comparison/prepare_yolo_dataset.py create mode 100644 scripts/model_comparison/run_yolo_data.slurm create mode 100644 scripts/model_comparison/run_yolo_prep.slurm create mode 100644 scripts/model_comparison/run_yolo_train.slurm diff --git a/hyak_yolo_runbook.sh b/hyak_yolo_runbook.sh new file mode 100644 index 0000000..55a43e5 --- /dev/null +++ b/hyak_yolo_runbook.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# ============================================================================ +# YOLO supervised-baseline training on Hyak (klone) — self-contained runbook. +# The training half of issue #51 (the 'yolo' provider + eval land elsewhere). +# +# Tracked operator runbook (like hyak_qwen_runbook.sh). Paths/identity are +# parameterized ($USER, $SCRATCH), so edit nothing to run it. Claude cannot drive +# klone (UW password + Duo per connection), so YOU run these stages; each is +# idempotent and safe to re-run. +# +# Usage on klone: bash hyak_yolo_runbook.sh +# Stages: env | data | prepsmoke | prep | train | status | collect +# Windows-side: bash hyak_yolo_runbook.sh push (prints the rsync commands) +# +# Unlike the VLM runbook, this does NOT upload benchmark panos: YOLO is trained on +# the RampNet dataset (pulled from HF straight onto the cluster), and EVAL RUNS +# LOCALLY — you only rsync the small best.pt files back (see `collect`). +# +# CRITICAL PATH (the two slow, unattended things; start them first): +# 1. `env` — lean venv on scratch with ultralytics (minutes) +# 2. `data` — download the 214k-pano dataset from HF (HOURS, big IO) +# 3. `prepsmoke` -> eyeball overlays -> `prep` (CPU, ~hours) +# 4. `train` — 6 configs, concurrent single-GPU jobs (~hours, parallel) +# ============================================================================ +set -euo pipefail + +REPO="${REPO:-$HOME/RampNet}" +USER="${USER:-$(whoami)}" +SCRATCH="${SCRATCH:-/gscratch/scrubbed/$USER}" +export HF_HOME="${HF_HOME:-$SCRATCH/hf}" +export YOLO_CONFIG_DIR="${YOLO_CONFIG_DIR:-$SCRATCH/ultralytics}" + +ENVDIR="${ENVDIR:-$SCRATCH/envs/yolo}" +PYBIN="${PYBIN:-$ENVDIR/bin/python}" +DATA="${DATA:-$SCRATCH/rampnet_dataset}" # HF dataset lands here (via ./dataset symlink) +YOLODATA="${YOLODATA:-$SCRATCH/yolo}" # prepared tiles/ and pano/ datasets +PROJECT="${PROJECT:-$SCRATCH/yolo_runs}" # training outputs (weights) +ACCOUNT="${ACCOUNT:-gpu-l40s-makelab}" # -A for sbatch; override if yours differs +BOX_SIZE="${BOX_SIZE:-fixed:0.03}" # A/B pitch/gps later by re-running prep+train +BG_KEEP="${BG_KEEP:-0.15}" # keep 15% of background tiles (tames the skew) +# YOLO26's checkpoint name/availability depends on the ultralytics version — check +# `python -c "from ultralytics import YOLO; YOLO('yolo26l.pt')"` and override YOLO26. +YOLO26="${YOLO26:-yolo26l.pt}" +STAGE="${1:-help}" + +banner() { echo; echo "=== $* ==="; echo; } + +case "$STAGE" in + +# --------------------------------------------------------------------------- +help|push) + cat <<'TXT' +Run from a shell with rsync (Windows: WSL). Define a `klone` host in ~/.ssh/config +(klone.hyak.uw.edu, your UW netid) with ControlMaster/ControlPath/ControlPersist so +Duo is entered once. Only the repo code goes up — no imagery, no venv, no caches: + + cd + rsync -av --exclude .venv --exclude .model_cache --exclude 'benchmark/*/panos' \ + --exclude 'benchmark/*/gallery' --exclude view_dump --exclude dataset \ + --exclude runs --exclude '*.pt' \ + RampNet/ klone:~/RampNet/ + +The dataset is pulled from HF ON the cluster (`data` stage), not uploaded. When +training finishes, pull ONLY the small weight files back and score locally: + + rsync -av --include '*/' --include 'best.pt' --include 'args.yaml' --exclude '*' \ + klone:/gscratch/scrubbed//yolo_runs/ ./yolo_runs/ +TXT + ;; + +# --------------------------------------------------------------------------- +env) + banner "Lean env on scratch: python 3.11 + torch(cu126) + ultralytics" + module load conda/Miniforge3-25.9.1-0 + if [ ! -x "$PYBIN" ]; then + conda create -p "$ENVDIR" python=3.11 -y + else + echo "$ENVDIR already exists; skipping create" + fi + # cu126 wheels — the lean-env path from docs/model_comparison.md, no CPU-fallback trap. + "$PYBIN" -m pip install --upgrade pip + "$PYBIN" -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cu126 + # ultralytics pulls numpy/pillow/opencv/pyyaml; datasets+hf_hub for download_dataset.py. + "$PYBIN" -m pip install ultralytics datasets huggingface_hub + banner "Verify — CUDA build survived, ultralytics imports" + "$PYBIN" - <<'PY' +import torch, ultralytics +print("torch", torch.__version__, "| ultralytics", ultralytics.__version__) +if "cpu" in torch.__version__: + raise SystemExit("STOP: CPU-only torch — reinstall from the cu126 index.") +PY + echo "OK. PYBIN=$PYBIN Next: bash hyak_yolo_runbook.sh data" + ;; + +# --------------------------------------------------------------------------- +data) + banner "Download the RampNet dataset (~214k panos) from HF onto scratch" + mkdir -p "$DATA" + ln -sfn "$DATA" "$REPO/dataset" # download_dataset.py writes ./dataset/{train,val,test} + cd "$REPO" + echo "This is the long pole (tens of GB). Landing in: $DATA" + "$PYBIN" download_dataset.py + du -sh "$DATA" | tail -1 + # Glob-free count: `ls "$DATA/$s"/*.jpg` overflows ARG_MAX on the ~150k-file train + # split, so `ls` errors out (to the suppressed stderr) and `wc -l` reports 0 — a + # false "train: 0 panos" even on a healthy download. Enumerate via readdir instead. + for s in train val test; do printf " %s: %s panos\n" "$s" "$(ls -U "$DATA/$s" 2>/dev/null | grep -c '\.jpg$')"; done + echo "OK. Next: bash hyak_yolo_runbook.sh prepsmoke" + ;; + +# --------------------------------------------------------------------------- +prepsmoke) + banner "200-pano tiles build + overlays — eyeball boxes before the full run" + cd "$REPO" + "$PYBIN" scripts/model_comparison/prepare_yolo_dataset.py \ + --dataset-root "$REPO/dataset" --out "$YOLODATA/tiles_smoke" \ + --geometry tiles --box-size "$BOX_SIZE" --bg-keep-frac "$BG_KEEP" \ + --subset 200 --overlay-dir "$YOLODATA/overlay_smoke" + echo + echo "scp $YOLODATA/overlay_smoke/*.jpg back and check red boxes sit ON ramps." + echo "If using --box-size gps, confirm the 'fell back to pitch' count is near 0." + echo "OK. Next: bash hyak_yolo_runbook.sh prep" + ;; + +# --------------------------------------------------------------------------- +prep) + banner "Full tiles + pano datasets (box-size=$BOX_SIZE, bg-keep=$BG_KEEP)" + cd "$REPO" + "$PYBIN" scripts/model_comparison/prepare_yolo_dataset.py \ + --dataset-root "$REPO/dataset" --out "$YOLODATA/tiles" \ + --geometry tiles --box-size "$BOX_SIZE" --bg-keep-frac "$BG_KEEP" + "$PYBIN" scripts/model_comparison/prepare_yolo_dataset.py \ + --dataset-root "$REPO/dataset" --out "$YOLODATA/pano" \ + --geometry pano --box-size "$BOX_SIZE" + echo "OK. Next: bash hyak_yolo_runbook.sh train" + ;; + +# --------------------------------------------------------------------------- +train) + banner "6 training jobs, concurrent (yolo11l/x/26 x tiles/pano)" + cd "$REPO" + mkdir -p logs + # sbatch inherits the environment (--export=ALL default), so PYTHON/HF_HOME/etc. + # reach the job. PYTHON=the lean venv; the .slurm falls back to conda otherwise. + export PYTHON="$PYBIN" HF_HOME YOLO_CONFIG_DIR PROJECT + SLURM=scripts/model_comparison/run_yolo_train.slurm + sub() { # sub + YOLO_CKPT="$1" YOLO_DATA="$2" YOLO_IMGSZ="$3" NAME="$4" \ + sbatch -A "$ACCOUNT" "$SLURM" + } + sub yolo11l.pt "$YOLODATA/tiles/data.yaml" 1024 y11l_tiles + sub yolo11x.pt "$YOLODATA/tiles/data.yaml" 1024 y11x_tiles + sub "$YOLO26" "$YOLODATA/tiles/data.yaml" 1024 y26_tiles + sub yolo11l.pt "$YOLODATA/pano/data.yaml" 1280 y11l_pano + sub yolo11x.pt "$YOLODATA/pano/data.yaml" 1280 y11x_pano + sub "$YOLO26" "$YOLODATA/pano/data.yaml" 1280 y26_pano + squeue -u "$USER" + echo "OK. Watch: bash hyak_yolo_runbook.sh status" + ;; + +# --------------------------------------------------------------------------- +status) + squeue -u "$USER" || true + echo + for f in $(ls -t "$REPO"/logs/yolo_train_*.out 2>/dev/null | head -4); do + echo "--- $f"; tail -12 "$f" + done + echo + echo "Weights so far:"; find "$PROJECT" -name best.pt 2>/dev/null + ;; + +# --------------------------------------------------------------------------- +collect) + banner "Trained weights (pull these home; eval runs locally)" + find "$PROJECT" -name best.pt 2>/dev/null + echo + echo "From your machine (small files only):" + echo " rsync -av --include '*/' --include 'best.pt' --include 'args.yaml' --exclude '*' \\" + echo " klone:$PROJECT/ ./yolo_runs/" + echo + echo "Then locally, per weight (default tiling = tiles; --tiling none for pano):" + echo " for c in bend richmond clovis; do" + echo " python scripts/model_comparison/compare.py benchmark/\$c \\" + echo " --models rampnet,yolo --yolo-model ./yolo_runs/y11l_tiles/weights/best.pt \\" + echo " --sweep --pr-out pr_out/\$c; done" + ;; + +*) + echo "unknown stage '$STAGE' (env | data | prepsmoke | prep | train | status | collect | push)" + exit 2 + ;; +esac diff --git a/requirements-vlm.txt b/requirements-vlm.txt index 232f6be..feb65db 100644 --- a/requirements-vlm.txt +++ b/requirements-vlm.txt @@ -36,3 +36,10 @@ requests # models use — the harness itself imports fine on 4.57.1, so only Molmo's # interpreter differs. See docs/model_comparison.md. # (`decord2` from the card is video-only; this harness only sends still views.) + +# YOLO supervised baseline (issue #51): Ultralytics for both training the detector +# on the RampNet dataset (scripts/model_comparison/prepare_yolo_dataset.py) and the +# 'yolo' provider's inference. Pulls its own torch/torchvision if absent, but the +# conda env / cluster wheels already provide CUDA builds — install this last so it +# doesn't override them. Weights (.pt) are passed via --yolo-model, not bundled. +ultralytics diff --git a/scripts/model_comparison/compare.py b/scripts/model_comparison/compare.py index c6e190f..2b4a4a4 100644 --- a/scripts/model_comparison/compare.py +++ b/scripts/model_comparison/compare.py @@ -416,8 +416,9 @@ def main(): ap.add_argument("bundle", help="Bundle dir (e.g. benchmark/richmond) with records.jsonl + verdicts.json.") ap.add_argument("--models", default="rampnet", help="Comma-separated detectors. Each is a provider (rampnet/gemini/qwen/" - "owlv2/gdino/molmo, using its default model) or provider:model_id to " - "pin a variant, e.g. 'rampnet,gemini:gemini-2.5-flash,owlv2'.") + "owlv2/gdino/molmo/yolo, using its default model) or provider:model_id to " + "pin a variant, e.g. 'rampnet,gemini:gemini-2.5-flash,owlv2'. yolo needs " + "trained weights: 'yolo:' or --yolo-model.") ap.add_argument("--gemini-model", default="gemini-3.6-flash") ap.add_argument("--qwen-model", default="Qwen/Qwen3-VL-8B-Instruct") ap.add_argument("--qwen-coord-space", choices=["auto", "norm1000", "pixels"], default="auto", @@ -440,6 +441,20 @@ def main(): ap.add_argument("--molmo-coord-scale", choices=["auto", "100", "1000"], default="auto", help="Divisor for Molmo point coordinates: Molmo 1 emits percentages " "(100), Molmo 2 emits 0-1000. 'auto' infers it from the tag syntax.") + ap.add_argument("--yolo-model", + help="Trained YOLO weights (.pt) for the 'yolo' provider — the supervised " + "baseline (issue #51). Required for --models yolo; e.g. " + "runs/detect/train/weights/best.pt. Also settable as 'yolo:'.") + ap.add_argument("--yolo-conf", type=float, default=0.05, + help="Score floor for YOLO boxes (default 0.05). Like --score-threshold for " + "the open-vocab detectors, this is a CACHE floor in the signature, not " + "the operating point; higher points are free re-scores (--op-threshold, " + "--sweep).") + ap.add_argument("--yolo-iou", type=float, default=0.5, + help="YOLO NMS IoU threshold (default 0.5).") + ap.add_argument("--yolo-imgsz", type=int, default=1024, + help="YOLO inference image size (default 1024, matching the perspective view " + "size). For --tiling none, set this to the pano-geometry training size.") ap.add_argument("--tiling", choices=["perspective", "none"], default="perspective", help="VLM input: 'perspective' reprojects the pano into rectilinear " "views (fair); 'none' uses one whole-pano call (lower bound). " @@ -452,7 +467,7 @@ def main(): "the cache); models without confidences are unaffected.") ap.add_argument("--sweep", action="store_true", help="Also print a threshold sweep for every model whose detections carry " - "confidences (RampNet, owlv2, gdino) — the tunable operating range.") + "confidences (RampNet, owlv2, gdino, yolo) — the tunable operating range.") ap.add_argument("--pr-out", help="Directory to write PR curves to (JSON per model, plus a " "combined PNG when matplotlib is installed).") ap.add_argument("--limit", type=int, diff --git a/scripts/model_comparison/detectors.py b/scripts/model_comparison/detectors.py index 9c6eb48..dbb7a64 100644 --- a/scripts/model_comparison/detectors.py +++ b/scripts/model_comparison/detectors.py @@ -18,6 +18,12 @@ AP / PR curves and threshold sweeps possible for a non-RampNet model. - ``MolmoDetector`` is **live** and emits **points**, not boxes — RampNet's native output format, so it avoids the box->center reduction every other VLM needs. +- ``YoloDetector`` is the one **supervised** model here: an Ultralytics YOLO box + detector *trained on the RampNet dataset* (the architecture-vs-data baseline, + issue #51). ``--yolo-model`` points at trained weights, not an HF id, so its + signature also hashes the weights file. Boxes carry a calibrated score, so it + gets AP / PR / sweep like the open-vocab detectors; tiled by default, with + ``--tiling none`` as the whole-pano ablation. """ import importlib.util import json @@ -266,6 +272,31 @@ def pixel_boxes_to_points(items, img_w, img_h): return points +def yolo_results_to_boxes(result, threshold=None): + """Normalize one Ultralytics ``Results`` into ``[{"box": [x1, y1, x2, y2] + (pixels), "score": float}]`` — the same shape ``pixel_boxes_to_points`` consumes. + + ``result.boxes.xyxy`` are absolute pixels in the frame the model was shown and + ``result.boxes.conf`` are calibrated per-box confidences; both are torch tensors + in a live run and plain lists in tests, so they go through ``_as_list``. The + score is **carried through** — it is what lets YOLO get AP / PR / a sweep.""" + boxes = getattr(result, "boxes", None) + if boxes is None: + return [] + xyxy = _as_list(getattr(boxes, "xyxy", None)) + conf = _as_list(getattr(boxes, "conf", None)) + items = [] + for i, box in enumerate(xyxy): + box = [float(v) for v in _as_list(box)] + if len(box) != 4: + continue + score = float(conf[i]) if i < len(conf) else None + if threshold is not None and score is not None and score < threshold: + continue + items.append({"box": box, "score": score}) + return items + + # --- Molmo point parsing (pure, unit-tested) -------------------------------- # Molmo emits points as XML-ish tags, and the two generations disagree on both the @@ -904,14 +935,98 @@ def signature(self): return sig +def _weights_fingerprint(path): + """Content hash of a YOLO weights file, so retraining to the same path/label + invalidates the detection cache — the weights *are* the model here, unlike a + stable HF id. Machine-independent (hashes bytes, not the path). Returns ``None`` + if the file is absent (e.g. scoring a rsynced cache without the weights present), + falling back to the label for identity.""" + import hashlib + try: + h = hashlib.sha1() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + except OSError: + return None + + +class YoloDetector(_VLMDetector): + """Ultralytics YOLO — the one **supervised** detector here, trained on the + RampNet dataset (issue #51: is RampNet's keypoint architecture doing the work, + or would any detector trained on the auto-generated data also beat the zero-shot + field?). Boxes carry objectness x class confidence, so it gets AP / PR / a sweep + like the open-vocabulary detectors, via the same box->center->score path. + + ``model_id`` here is a **weights path** (``.pt``), not an HF id. Its identity for + the results table and the cache is the file **stem** (machine-independent), and + ``signature`` additionally hashes the weights bytes so a re-trained checkpoint at + the same path invalidates cached detections. Tiled by default (perspective + views, matching how the VLMs are scored); ``--tiling none`` runs the whole-pano + ablation with pano-geometry weights. + + Cache-gap caveat (shared with the other providers): the key does not include the + box->point parser version. If ``_parse`` / ``yolo_results_to_boxes`` changes, + bump something in ``signature()`` to bust ``.model_cache``.""" + + name = "yolo" + prompt = None # not a prompted model; the base "prompt" cache key stays None + score_threshold = 0.05 # cache floor, like the zero-shot detectors + + def __init__(self, weights, label=None, conf=None, iou=0.5, imgsz=1024, + tile=True, views=None): + # Identity is the file stem, not the absolute path, so the cache is + # machine-independent (score a rsynced .model_cache without re-running). + stem = label or os.path.splitext(os.path.basename(str(weights)))[0] + super().__init__(stem, tile=tile, views=views) + self.weights = weights + self.conf = self.score_threshold if conf is None else float(conf) + self.score_threshold = self.conf # attribute read to feed the --sweep floor + self.iou = float(iou) + self.imgsz = int(imgsz) + self._model = None + self._device = "cpu" + + def _ensure_ready(self): + if self._model is not None: + return + try: + import torch + from ultralytics import YOLO + except ImportError as e: + raise ImportError( + "YoloDetector needs `ultralytics` " + "(pip install ultralytics; see requirements-vlm.txt)") from e + self._device = "cuda" if torch.cuda.is_available() else "cpu" + self._model = YOLO(self.weights) + + def _raw_detect(self, image): + # conf is a low cache floor; higher operating points are free re-scores + # (--op-threshold / --sweep), exactly like the open-vocab detectors. + results = self._model.predict(image, conf=self.conf, iou=self.iou, + imgsz=self.imgsz, device=self._device, verbose=False) + return yolo_results_to_boxes(results[0]) + + def _parse(self, raw, img_w, img_h): + return pixel_boxes_to_points(raw, img_w, img_h) + + def signature(self): + sig = super().signature() + sig.update({"weights_hash": _weights_fingerprint(self.weights), + "conf": self.conf, "iou": self.iou, "imgsz": self.imgsz}) + return sig + + def parse_model_spec(token): """Parse a ``--models`` token into ``(provider, model_id_or_None)``. A token is either a bare provider (``rampnet`` / ``gemini`` / ``qwen`` / - ``owlv2`` / ``gdino`` / ``molmo``, which uses that provider's default model) or - ``provider:model_id`` to pin a specific variant — e.g. ``gemini:gemini-2.5-flash`` - vs ``gemini:gemini-3.6-flash`` — so several variants of the same provider can be - compared in one run.""" + ``owlv2`` / ``gdino`` / ``molmo`` / ``yolo``, which uses that provider's default + model) or ``provider:model_id`` to pin a specific variant — e.g. + ``gemini:gemini-2.5-flash`` vs ``gemini:gemini-3.6-flash``, or + ``yolo:runs/detect/train/weights/best.pt`` for a trained checkpoint — so several + variants of the same provider can be compared in one run.""" provider, _, model_id = token.partition(":") return provider.strip(), (model_id.strip() or None) @@ -949,5 +1064,19 @@ def build_detector(provider, model_id, records, args): scale = getattr(args, "molmo_coord_scale", "auto") return mid, MolmoDetector(model_id=mid, tile=tile, coord_scale=None if scale in (None, "auto") else float(scale)) + if provider == "yolo": + # model_id (from yolo:) or --yolo-model is a trained weights path, not + # an HF id; the table label + cache identity is its machine-independent stem. + weights = model_id or getattr(args, "yolo_model", None) + if not weights: + raise ValueError("yolo needs a trained weights path: " + "--models yolo: or --yolo-model ") + iou = getattr(args, "yolo_iou", None) + imgsz = getattr(args, "yolo_imgsz", None) + label = os.path.splitext(os.path.basename(str(weights)))[0] + return label, YoloDetector(weights=weights, label=label, tile=tile, + conf=getattr(args, "yolo_conf", None), + iou=0.5 if iou is None else float(iou), + imgsz=1024 if imgsz is None else int(imgsz)) raise ValueError(f"unknown provider '{provider}' (choose from: rampnet, gemini, qwen, " - "owlv2, gdino, molmo)") + "owlv2, gdino, molmo, yolo)") diff --git a/scripts/model_comparison/prepare_yolo_dataset.py b/scripts/model_comparison/prepare_yolo_dataset.py new file mode 100644 index 0000000..d49dd9d --- /dev/null +++ b/scripts/model_comparison/prepare_yolo_dataset.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +"""Build an Ultralytics YOLO detection dataset from the RampNet point dataset. + +The supervised baseline of issue #51: RampNet's training data is per-pano *points* +(``curb_ramp_points_normalized``), not boxes, so this synthesizes boxes to train a +generic detector on the same data. Two geometries (pick with ``--geometry``): + +- ``tiles`` (the headline baseline): reproject each pano into the SAME ring of + rectilinear views the harness scores with (``equirect_tiling.default_views``), + project every GT point into each view with ``equirect_point_to_perspective``, and + write a box per in-view point. Train and inference geometry match by construction. +- ``pano``: one whole-pano (downscaled equirect) image per pano, boxes placed at the + points directly. Matches RampNet's own full-pano input; the ``--tiling none`` eval. + +**Box size is a train-only knob** — evaluation reduces every prediction back to a +center point and matches at a fixed radius, so box w/h never enter the metric; they +only shape YOLO's assignment / NMS / confidence. ``--box-size`` picks the strategy: + +- ``fixed:`` (default): constant fraction of the view/pano. Simplest. +- ``pitch``: apparent size proportional to 1/distance, with distance from the point's + ground-plane pitch (its vertical position). Self-contained, no extra data. +- ``gps``: exact ground distance via haversine of ``pano_coord`` <-> ``curb_ramp_coords`` + (the JSON already carries both). Falls back to ``pitch`` per-pano when the point + count and coord count disagree (the placed points are not guaranteed 1:1 with the + GPS list); the summary reports how often that happened, so you can trust or drop it. + +Output is an Ultralytics dataset: ``/images/{train,val}/*.jpg`` + +``/labels/{train,val}/*.txt`` + ``/data.yaml`` (test is reserved, not used +for training). ``tiles`` produces ~6x as many files as panos; use ``--subset`` while +iterating and ``--bg-keep-frac`` to thin the many label-less (background) tiles. +""" +import argparse +import hashlib +import json +import math +import os +import sys +from collections import defaultdict, namedtuple +from glob import glob +from multiprocessing import Pool + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +from equirect_tiling import ( # noqa: E402 + default_views, equirect_to_perspective, equirect_point_to_perspective) + +CLASS_NAME = "curb_ramp" +SPLIT_MAP = {"train": "train", "val": "val"} # RampNet split -> YOLO split + +Config = namedtuple("Config", [ + "geometry", "strategy", "fixed_frac", "min_frac", "max_frac", + "ramp_size_m", "camera_height_m", "source_max_edge", "pano_w", "pano_h", + "views", "out", "overlay_dir", "bg_keep_frac", +]) + +_CFG = None # set per worker via the Pool initializer (avoids re-pickling cfg per task) + + +# --- geometry / distance helpers -------------------------------------------- + +def _haversine_m(lat1, lon1, lat2, lon2): + R = 6371000.0 + p1, p2 = math.radians(lat1), math.radians(lat2) + dphi, dlmb = math.radians(lat2 - lat1), math.radians(lon2 - lon1) + a = math.sin(dphi / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dlmb / 2) ** 2 + return 2 * R * math.asin(min(1.0, math.sqrt(a))) + + +def _ground_distance_m(y_norm, camera_height_m): + """Ground distance to a pano point under a flat-ground, fixed-camera-height + model. ``y`` is 0 (top/up) -> 1 (bottom/down); below the horizon (y>0.5) the + downward pitch is ``(y-0.5)*pi``. At/above the horizon -> inf (the min box).""" + pitch_below = (y_norm - 0.5) * math.pi + if pitch_below <= 1e-3: + return float("inf") + return camera_height_m / math.tan(pitch_below) + + +def _clamp_frac(f, cfg): + return max(cfg.min_frac, min(cfg.max_frac, f)) + + +def _resolve_distances(cfg, points, pano_coord, coords): + """Per-point distance in meters, or None when it should come from the point's + own y (fixed ignores it; pitch derives it; gps uses it when correspondence is + clean). Returns ``(dists, gps_used)``.""" + if (cfg.strategy == "gps" and points and pano_coord and coords + and len(coords) == len(points)): + try: + return ([_haversine_m(pano_coord[0], pano_coord[1], c[0], c[1]) for c in coords], + True) + except (TypeError, IndexError): + pass + return [None] * len(points), False + + +def _box_wh(cfg, x, y, dist_m, fov_h_deg, fov_v_deg): + """(w, h) box size normalized to the target frame (the square tile, or the pano). + + ``fixed`` returns a constant; the distance-aware strategies convert a physical + ramp size at ``dist_m`` into an angular size and then a fraction of the frame.""" + if cfg.strategy == "fixed": + return cfg.fixed_frac, cfg.fixed_frac + if dist_m is None: # pitch, or gps that fell back to pitch + dist_m = _ground_distance_m(y, cfg.camera_height_m) + if not math.isfinite(dist_m) or dist_m <= 0: + return cfg.min_frac, cfg.min_frac + alpha = cfg.ramp_size_m / dist_m # small-angle subtended size (radians) + if cfg.geometry == "tiles": + return (_clamp_frac(alpha / math.radians(fov_h_deg), cfg), + _clamp_frac(alpha / math.radians(fov_v_deg), cfg)) + # pano (equirect): a full turn is 2*pi horizontally, pi vertically; longitude is + # stretched by 1/cos(lat), so widen the box by that (clamped away from the poles). + lat = (0.5 - y) * math.pi + return (_clamp_frac(alpha / (2 * math.pi) / max(0.2, math.cos(lat)), cfg), + _clamp_frac(alpha / math.pi, cfg)) + + +def _keep_background(cfg, stem): + """Deterministically thin label-less tiles to ``--bg-keep-frac`` (stable across + processes/runs — a salted hash() would differ per worker).""" + if cfg.bg_keep_frac >= 1.0: + return True + h = int(hashlib.md5(stem.encode("utf-8")).hexdigest(), 16) % 1000 + return h < cfg.bg_keep_frac * 1000 + + +# --- IO helpers ------------------------------------------------------------- + +def _valid_pt(p): + return (isinstance(p, (list, tuple)) and len(p) == 2 + and all(isinstance(v, (int, float)) for v in p) + and 0.0 <= p[0] <= 1.0 and 0.0 <= p[1] <= 1.0) + + +def _downscale(img, max_edge): + from PIL import Image + if max_edge and max(img.size) > max_edge: + s = max_edge / max(img.size) + img = img.resize((round(img.width * s), round(img.height * s)), Image.BILINEAR) + return img + + +def _write_pair(out, yolo_split, stem, image, lines): + image.save(os.path.join(out, "images", yolo_split, stem + ".jpg"), quality=90) + with open(os.path.join(out, "labels", yolo_split, stem + ".txt"), "w") as f: + f.write("\n".join(lines) + ("\n" if lines else "")) + + +def _save_overlay(overlay_dir, stem, image, lines): + """Draw the synthesized boxes on the tile so a human can eyeball that they land + on ramps (the label-gen analog of dump_detections' inference overlays).""" + from PIL import ImageDraw + img = image.convert("RGB").copy() + draw = ImageDraw.Draw(img) + W, H = img.size + for line in lines: + _, cx, cy, w, h = (float(t) for t in line.split()) + x0, y0 = (cx - w / 2) * W, (cy - h / 2) * H + x1, y1 = (cx + w / 2) * W, (cy + h / 2) * H + draw.rectangle([x0, y0, x1, y1], outline=(255, 0, 0), width=2) + img.save(os.path.join(overlay_dir, stem + ".jpg"), quality=90) + + +# --- per-pano worker -------------------------------------------------------- + +def _init_worker(cfg): + global _CFG + _CFG = cfg + + +def process_pano(task): + """Render one pano's tiles (or its whole-pano image) + YOLO labels. Returns a + stats dict; workers write distinct files (keyed by pano id) so there is no race.""" + from PIL import Image + split, jpg_path, json_path, overlay = task + cfg = _CFG + st = defaultdict(int) + st["panos"] = 1 + try: + with open(json_path) as f: + meta = json.load(f) + except (OSError, json.JSONDecodeError): + st["read_errors"] = 1 + return st + + points = [tuple(p) for p in meta.get("curb_ramp_points_normalized", []) if _valid_pt(p)] + st["points"] = len(points) + dists, gps_used = _resolve_distances(cfg, points, meta.get("pano_coord"), + meta.get("curb_ramp_coords")) + if cfg.strategy == "gps" and points and not gps_used: + st["gps_mismatch"] = 1 + + try: + img = Image.open(jpg_path).convert("RGB") + except OSError: + st["read_errors"] = 1 + return st + pid = str(meta.get("pano_id") or os.path.splitext(os.path.basename(jpg_path))[0]) + yolo_split = SPLIT_MAP[split] + + if cfg.geometry == "tiles": + src = _downscale(img, cfg.source_max_edge) + for k, view in enumerate(cfg.views): + lines = [] + for (x, y), d in zip(points, dists): + uv = equirect_point_to_perspective(x, y, view) + if uv is None: + continue + u, v = uv + w, h = _box_wh(cfg, x, y, d, view.fov_h_deg, view.fov_v_deg) + lines.append(f"0 {u:.6f} {v:.6f} {w:.6f} {h:.6f}") + stem = f"{pid}_v{k}" + if not lines and not _keep_background(cfg, stem): + st["bg_skipped"] += 1 + continue + _write_pair(cfg.out, yolo_split, stem, equirect_to_perspective(src, view), lines) + st["tiles"] += 1 + st["boxes"] += len(lines) + st["bg_tiles"] += 1 if not lines else 0 + if overlay and cfg.overlay_dir: + _save_overlay(cfg.overlay_dir, stem, equirect_to_perspective(src, view), lines) + else: # pano + pimg = img.resize((cfg.pano_w, cfg.pano_h), Image.BILINEAR) + lines = [] + for (x, y), d in zip(points, dists): + w, h = _box_wh(cfg, x, y, d, None, None) + lines.append(f"0 {x:.6f} {y:.6f} {w:.6f} {h:.6f}") + _write_pair(cfg.out, yolo_split, pid, pimg, lines) + st["tiles"] += 1 + st["boxes"] += len(lines) + st["bg_tiles"] += 1 if not lines else 0 + if overlay and cfg.overlay_dir: + _save_overlay(cfg.overlay_dir, pid, pimg, lines) + return st + + +# --- driver ----------------------------------------------------------------- + +def parse_box_size(s): + s = s.strip().lower() + if s.startswith("fixed"): + _, _, val = s.partition(":") + return "fixed", (float(val) if val else 0.03) + if s in ("pitch", "gps"): + return s, 0.0 + raise argparse.ArgumentTypeError("box-size must be fixed[:frac], pitch, or gps") + + +def write_data_yaml(out): + with open(os.path.join(out, "data.yaml"), "w") as f: + f.write(f"# Generated by prepare_yolo_dataset.py\npath: {os.path.abspath(out)}\n") + f.write("train: images/train\nval: images/val\n") + f.write(f"names:\n 0: {CLASS_NAME}\n") + + +def build_config(args): + strategy, fixed_frac = args.box_size + views = default_views(fov_h_deg=args.view_fov, fov_v_deg=args.view_fov, + pitch_deg=args.view_pitch, n_yaw=args.n_yaw, + width=args.view_size, height=args.view_size) + return Config( + geometry=args.geometry, strategy=strategy, fixed_frac=fixed_frac, + min_frac=args.min_frac, max_frac=args.max_frac, ramp_size_m=args.ramp_size_m, + camera_height_m=args.camera_height_m, source_max_edge=args.source_max_edge, + pano_w=args.pano_width, pano_h=args.pano_width // 2, views=views, + out=os.path.abspath(args.out), overlay_dir=os.path.abspath(args.overlay_dir) + if args.overlay_dir else None, bg_keep_frac=args.bg_keep_frac) + + +def gather_tasks(args, cfg): + tasks = [] + for split in SPLIT_MAP: + d = os.path.join(args.dataset_root, split) + jpgs = sorted(glob(os.path.join(d, "*.jpg"))) + if args.subset: + jpgs = jpgs[:args.subset] + for i, jpg in enumerate(jpgs): + js = jpg[:-4] + ".json" + if os.path.exists(js): + overlay = bool(cfg.overlay_dir) and i < args.overlay_n + tasks.append((split, jpg, js, overlay)) + return tasks + + +def main(): + ap = argparse.ArgumentParser(description="Build a YOLO dataset from the RampNet point dataset.") + ap.add_argument("--dataset-root", default="dataset", + help="Root with train/ val/ (test/) of .jpg + .json (default dataset).") + ap.add_argument("--out", required=True, help="Output Ultralytics dataset dir.") + ap.add_argument("--geometry", choices=["tiles", "pano"], default="tiles") + ap.add_argument("--box-size", type=parse_box_size, default=("fixed", 0.03), + help="fixed[:frac] (default fixed:0.03), pitch, or gps. Train-only knob.") + ap.add_argument("--subset", type=int, help="Use at most N panos PER SPLIT (smoke tests).") + ap.add_argument("--workers", type=int, default=max(1, (os.cpu_count() or 4) - 1)) + ap.add_argument("--bg-keep-frac", type=float, default=1.0, + help="Fraction of label-less (background) tiles to keep, thinned " + "deterministically (default 1.0). ~0.1-0.2 tames the tile skew.") + # Box-size model constants (distance-aware strategies only). + ap.add_argument("--min-frac", type=float, default=0.008) + ap.add_argument("--max-frac", type=float, default=0.12) + ap.add_argument("--ramp-size-m", type=float, default=1.5) + ap.add_argument("--camera-height-m", type=float, default=2.5) + # Geometry — defaults MATCH equirect_tiling.default_views() so train == inference. + ap.add_argument("--n-yaw", type=int, default=6) + ap.add_argument("--view-fov", type=float, default=90.0) + ap.add_argument("--view-pitch", type=float, default=-30.0) + ap.add_argument("--view-size", type=int, default=1024) + ap.add_argument("--source-max-edge", type=int, default=4096, + help="Downscale the pano to this longest edge before tiling " + "(matches the harness's source_max_edge).") + ap.add_argument("--pano-width", type=int, default=2048, + help="Width of the whole-pano image for --geometry pano (height = width/2).") + ap.add_argument("--overlay-dir", help="Also render boxes-on-tiles for QA.") + ap.add_argument("--overlay-n", type=int, default=20, help="Panos to overlay (default 20).") + args = ap.parse_args() + + cfg = build_config(args) + if (args.n_yaw, args.view_fov, args.view_pitch, args.view_size) != (6, 90.0, -30.0, 1024): + print("WARNING: view geometry differs from default_views(); tiles won't match " + "the harness's inference rig unless the 'yolo' provider uses the same --views.") + for sp in set(SPLIT_MAP.values()): + os.makedirs(os.path.join(cfg.out, "images", sp), exist_ok=True) + os.makedirs(os.path.join(cfg.out, "labels", sp), exist_ok=True) + if cfg.overlay_dir: + os.makedirs(cfg.overlay_dir, exist_ok=True) + + tasks = gather_tasks(args, cfg) + print(f"{len(tasks)} panos | geometry={cfg.geometry} | box-size={args.box_size[0]}" + f"{':' + str(cfg.fixed_frac) if cfg.strategy == 'fixed' else ''} | " + f"workers={args.workers} | bg-keep={cfg.bg_keep_frac} -> {cfg.out}") + + agg = defaultdict(int) + with Pool(args.workers, initializer=_init_worker, initargs=(cfg,)) as pool: + for n, st in enumerate(pool.imap_unordered(process_pano, tasks, chunksize=8), 1): + for k, v in st.items(): + agg[k] += v + if n % 2000 == 0: + print(f" {n}/{len(tasks)} panos | {agg['tiles']} imgs | {agg['boxes']} boxes") + + write_data_yaml(cfg.out) + print(f"\nDone. panos={agg['panos']} images={agg['tiles']} boxes={agg['boxes']} " + f"bg_images={agg['bg_tiles']} bg_skipped={agg['bg_skipped']} " + f"points={agg['points']} read_errors={agg['read_errors']}") + if args.box_size[0] == "gps": + print(f"gps: fell back to pitch on {agg['gps_mismatch']} panos " + f"(point/coord count mismatch). Trust gps only if this is near 0.") + print(f"data.yaml -> {os.path.join(cfg.out, 'data.yaml')}") + + +if __name__ == "__main__": + main() diff --git a/scripts/model_comparison/run_yolo_data.slurm b/scripts/model_comparison/run_yolo_data.slurm new file mode 100644 index 0000000..91fab6b --- /dev/null +++ b/scripts/model_comparison/run_yolo_data.slurm @@ -0,0 +1,42 @@ +#!/bin/bash +# Download the RampNet dataset (~214k panos) from HF onto scratch, as a Slurm job. +# This is the `data` stage of hyak_yolo_runbook.sh, wrapped so the multi-hour, +# 8-process JPEG re-encode runs on a COMPUTE node rather than the login node — +# klone's login nodes kill heavy, long-running processes, which would silently +# abort an unattended download partway through. +# +# The GPU is requested only to satisfy the gpu-l40s partition's constraints; the +# download itself is CPU/IO-bound (HF parquet fetch + `.map` re-encode) and does +# not touch it. Idempotent: HF caches shards and the per-pano files are written +# incrementally, so a re-run resumes rather than starting over. +# +# mkdir -p logs +# sbatch -A gpu-l40s-makelab scripts/model_comparison/run_yolo_data.slurm +# +#SBATCH -p gpu-l40s +#SBATCH --job-name=yolo_data_dl +#SBATCH --time=24:00:00 +#SBATCH --mem=64G +#SBATCH --cpus-per-task=12 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --gpus-per-node=1 +#SBATCH --output=logs/yolo_data_%j.out +#SBATCH --error=logs/yolo_data_%j.err + +set -euo pipefail + +REPO="${REPO:-$HOME/RampNet}" +cd "$REPO" + +echo "--- RampNet dataset download (issue #51 data stage) ---" +echo "node: ${SLURMD_NODENAME:-?}" +echo "cpus: ${SLURM_CPUS_PER_TASK:-?} mem: ${SLURM_MEM_PER_NODE:-?}M" +echo "start: $(date -Is)" +echo "-------------------------------------------------------" + +# Single source of truth for the download procedure: the runbook's `data` stage +# sets HF_HOME/PYBIN/DATA, symlinks ./dataset -> scratch, and runs download_dataset.py. +bash "$REPO/hyak_yolo_runbook.sh" data + +echo "--- download job done: $(date -Is) ---" diff --git a/scripts/model_comparison/run_yolo_prep.slurm b/scripts/model_comparison/run_yolo_prep.slurm new file mode 100644 index 0000000..cbfb652 --- /dev/null +++ b/scripts/model_comparison/run_yolo_prep.slurm @@ -0,0 +1,53 @@ +#!/bin/bash +# Full YOLO prep (issue #51): build the tiles + pano training datasets from the ~214k-pano +# RampNet dataset with box-size=pitch (distance-aware; fixed:0.03 was too small and gps fell +# back to pitch on ~85% of panos, so pitch is the consistent choice). Runs on a COMPUTE node: +# the 150k-pano tiling is multi-hour and CPU-heavy, and klone's login nodes reap heavy procs +# (that reap also kills the ssh control-master). Writes to shared scratch. +# +# sbatch -A gpu-l40s-makelab scripts/model_comparison/run_yolo_prep.slurm +# # override box model: BOX_SIZE=pitch RAMP_SIZE_M=1.8 sbatch ... +# +#SBATCH -p gpu-l40s +#SBATCH --job-name=yolo_prep +#SBATCH --time=12:00:00 +#SBATCH --mem=48G +#SBATCH --cpus-per-task=16 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --gpus-per-node=1 +#SBATCH --output=/mmfs1/home/jfroehli/RampNet/logs/yolo_prep_%j.out +#SBATCH --error=/mmfs1/home/jfroehli/RampNet/logs/yolo_prep_%j.err +set -uo pipefail + +REPO="${REPO:-/mmfs1/home/jfroehli/RampNet}" +PY="${PYBIN:-/gscratch/scrubbed/jfroehli/envs/yolo/bin/python}" +Y="${YOLODATA:-/gscratch/scrubbed/jfroehli/yolo}" +BOX="${BOX_SIZE:-pitch}" +RAMP="${RAMP_SIZE_M:-1.8}" +BG="${BG_KEEP:-0.15}" +W="${WORKERS:-${SLURM_CPUS_PER_TASK:-16}}" +export OPENBLAS_NUM_THREADS=1 # one BLAS thread per worker (no RLIMIT_NPROC storm) +cd "$REPO" + +echo "node: $(hostname) start: $(date -Is) cpus: ${SLURM_CPUS_PER_TASK:-?} box=$BOX ramp=${RAMP}m workers=$W" + +echo "=== TILES ===" +"$PY" scripts/model_comparison/prepare_yolo_dataset.py \ + --dataset-root "$REPO/dataset" --out "$Y/tiles" --geometry tiles \ + --box-size "$BOX" --ramp-size-m "$RAMP" --bg-keep-frac "$BG" --workers "$W" + +echo "=== PANO ===" +"$PY" scripts/model_comparison/prepare_yolo_dataset.py \ + --dataset-root "$REPO/dataset" --out "$Y/pano" --geometry pano \ + --box-size "$BOX" --ramp-size-m "$RAMP" --workers "$W" + +echo "=== output image/label counts (glob-free) ===" +for g in tiles pano; do + for d in images labels; do + for s in train val; do + printf " %-5s %-6s %-5s = %s\n" "$g" "$d" "$s" "$(ls -U "$Y/$g/$d/$s" 2>/dev/null | wc -l)" + done + done +done +echo "ALL DONE @ $(date -Is)" diff --git a/scripts/model_comparison/run_yolo_train.slurm b/scripts/model_comparison/run_yolo_train.slurm new file mode 100644 index 0000000..7da8a7d --- /dev/null +++ b/scripts/model_comparison/run_yolo_train.slurm @@ -0,0 +1,120 @@ +#!/bin/bash +# Train one YOLO curb-ramp detector on the RampNet dataset, on Hyak (klone). +# The supervised baseline of issue #51. +# +# Pass your Slurm account with -A (no usable default; a submission without one is +# rejected). Find it with: sacctmgr -nP show assoc user=$USER format=Account,QOS +# +# ONE config per job, parameterized by env vars, so the 6 configs +# (yolo11l/x/26 x tiles/pano) are the same script sbatch'd concurrently. They run on +# the ckpt (scavenger) partition so they NEVER touch the lab's dedicated gpu-l40s +# allocation — students keep that. ckpt is PREEMPTABLE (PreemptMode=REQUEUE): an +# owner reclaiming a node requeues the job, which then RESUMES from weights/last.pt +# (see the resume block below), so a preemption costs at most the in-progress epoch. +# +# YOLO_CKPT=yolo11l.pt YOLO_DATA=$DATA/tiles/data.yaml BATCH=6 NAME=y11l_tiles \ +# sbatch -A ckpt-makelab scripts/model_comparison/run_yolo_train.slurm +# YOLO_CKPT=yolo11x.pt YOLO_DATA=$DATA/tiles/data.yaml BATCH=3 NAME=y11x_tiles \ +# sbatch -A ckpt-makelab scripts/model_comparison/run_yolo_train.slurm +# YOLO_CKPT=yolo11l.pt YOLO_DATA=$DATA/pano/data.yaml YOLO_IMGSZ=1280 BATCH=4 NAME=y11l_pano \ +# sbatch -A ckpt-makelab scripts/model_comparison/run_yolo_train.slurm +# +# BATCH is pinned per config (not auto -1) so the LR schedule is identical no matter +# which ckpt GPU (l40/l40s/h200) the job lands or resumes on. Sizes are tuned for >=48G. +# +# imgsz MUST match the eval side: tiles are rendered at 1024 (default), so the +# 'yolo' provider defaults --yolo-imgsz 1024; a pano-geometry model wants the +# training imgsz passed to --yolo-imgsz at eval time too. +# +# Only best.pt is needed downstream — rsync it back and score locally (the eval is +# ~110-125 panos/city and runs fine on a laptop GPU; see hyak_yolo_runbook.sh). +#SBATCH -p ckpt-g2 +#SBATCH -q ckpt-gpu +#SBATCH --requeue +#SBATCH --job-name=yolo_curb_ramp_train +#SBATCH --time=72:00:00 +#SBATCH --mem=64G +#SBATCH --cpus-per-task=12 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --gpus-per-node=1 +#SBATCH --output=logs/yolo_train_%j.out +#SBATCH --error=logs/yolo_train_%j.err + +set -euo pipefail + +YOLO_CKPT="${YOLO_CKPT:-yolo11l.pt}" # base weights (or an absolute path) +YOLO_DATA="${YOLO_DATA:?set YOLO_DATA to a prepared data.yaml}" +YOLO_IMGSZ="${YOLO_IMGSZ:-1024}" +EPOCHS="${EPOCHS:-60}" +BATCH="${BATCH:--1}" # pin per config (e.g. 6/3) for ckpt; -1 = auto +PATIENCE="${PATIENCE:-20}" # early-stop if val mAP plateaus +# Fixed-epoch, resume-safe training (default). On ckpt a job can be preempted and +# requeued any number of times; a fixed EPOCHS + resume-from-last.pt accumulates +# cleanly across requeues, whereas ultralytics `time=` restarts its clock on +# every requeue (unbounded). So TRAIN_HOURS defaults to 0. Set it >0 only for a +# one-shot run on a non-preemptable partition, where it fits the LR schedule to the +# wall budget and saves cleanly. +TRAIN_HOURS="${TRAIN_HOURS:-0}" +NAME="${NAME:-yolo_run}" + +# Keep everything on scratch (home quota is tiny; runs + base-weight downloads are +# large). /gscratch/scrubbed auto-purges after ~21 idle days. +export HF_HOME="${HF_HOME:-/gscratch/scrubbed/$USER/hf}" +PROJECT="${PROJECT:-/gscratch/scrubbed/$USER/yolo_runs}" +# Base-weight autodownload + Ultralytics settings/datasets dir on scratch, not $HOME. +export YOLO_CONFIG_DIR="${YOLO_CONFIG_DIR:-/gscratch/scrubbed/$USER/ultralytics}" +mkdir -p "$PROJECT" "$YOLO_CONFIG_DIR" + +# PYTHON points at a ready interpreter with ultralytics (a lean venv on scratch — +# see the runbook 'env' stage). Falls back to the conda env, which has ultralytics +# only if requirements-vlm.txt was installed into it. +PYTHON="${PYTHON:-}" +if [ -z "$PYTHON" ]; then + # shellcheck disable=SC1091 + source activate sidewalkcv2 + PYTHON=python +fi + +echo "--- YOLO train (issue #51 supervised baseline) ---" +echo "base: ${YOLO_CKPT}" +echo "data: ${YOLO_DATA}" +echo "imgsz: ${YOLO_IMGSZ} epochs: ${EPOCHS} batch: ${BATCH} patience: ${PATIENCE} time_hours: ${TRAIN_HOURS}" +echo "out: ${PROJECT}/${NAME}/weights/best.pt" +echo "GPU: ${SLURM_GPUS_PER_NODE:-1} on ${SLURMD_NODENAME:-?}" +nvidia-smi --query-gpu=name,memory.total --format=csv || true +echo "--------------------------------------------------" + +# Programmatic call (interpreter-explicit; avoids depending on the `yolo` console +# script being on PATH for whichever venv PYTHON is). +"$PYTHON" - "$YOLO_CKPT" "$YOLO_DATA" "$YOLO_IMGSZ" "$EPOCHS" "$BATCH" "$PATIENCE" \ + "$PROJECT" "$NAME" "$TRAIN_HOURS" <<'PY' +import os, sys +from ultralytics import YOLO +ckpt, data, imgsz, epochs, batch, patience, project, name, hours = sys.argv[1:10] +last = os.path.join(project, name, "weights", "last.pt") +if os.path.exists(last): + # Preempted-and-requeued: resume the in-progress run from its last checkpoint. + # resume=True reuses every training arg saved in the checkpoint, so the LR + # schedule continues seamlessly no matter how many times ckpt requeues us. + print(f"[resume] {last} exists -> resuming", flush=True) + YOLO(last).train(resume=True) +else: + # A pinned batch must be a Python int — torch's BatchSampler rejects a float + # (batch=6.0 -> ValueError). -1 (auto) and 0 0: + # Overrides epochs: ultralytics fits the LR schedule to this wall budget. + # (Only safe on a non-preemptable partition — see TRAIN_HOURS note above.) + kw["time"] = float(hours) + YOLO(ckpt).train(**kw) +PY + +echo "--- done: ${PROJECT}/${NAME}/weights/best.pt ---" +echo "rsync it back, then: compare.py benchmark/ --models rampnet,yolo \\" +echo " --yolo-model best.pt --yolo-imgsz ${YOLO_IMGSZ} --sweep" diff --git a/tests/test_model_comparison.py b/tests/test_model_comparison.py index 0c7ec54..d1c9b31 100644 --- a/tests/test_model_comparison.py +++ b/tests/test_model_comparison.py @@ -14,11 +14,12 @@ from detectors import ( # noqa: E402 BundleRampNetDetector, GeminiDetector, GroundingDinoDetector, MolmoDetector, - OwlV2Detector, QwenDetector, PanoSample, _VLMDetector, + OwlV2Detector, QwenDetector, YoloDetector, PanoSample, _VLMDetector, gemini_boxes_to_points, qwen_boxes_to_points, boxes_from_gemini_response, boxes_from_qwen_text, infer_qwen_coord_space, parse_model_spec, build_detector, molmo_points_from_text, molmo_token_points_to_items, infer_molmo_mode, owlv2_target_size, pixel_boxes_to_points, zero_shot_results_to_boxes, + yolo_results_to_boxes, CURB_RAMP_DEFINITION, DETECTION_PROMPT, GDINO_QUERY, MOLMO_PROMPT, OWLV2_QUERY, ) from dump_detections import detections_to_view_shapes # noqa: E402 @@ -30,6 +31,17 @@ load_manual_ground_truths, operating_report, rescore, sweep_rows, ) from rampnet.detection_eval import GroundTruth, radius_sq_for # noqa: E402 +from prepare_yolo_dataset import ( # noqa: E402 + parse_box_size, _ground_distance_m, _box_wh, _resolve_distances, Config as YoloPrepConfig, +) + + +def _prep_cfg(**over): + base = dict(geometry="tiles", strategy="fixed", fixed_frac=0.03, min_frac=0.008, + max_frac=0.12, ramp_size_m=1.5, camera_height_m=2.5, source_max_edge=4096, + pano_w=2048, pano_h=1024, views=(), out="", overlay_dir=None, bg_keep_frac=1.0) + base.update(over) + return YoloPrepConfig(**base) class _Args: @@ -44,6 +56,10 @@ class _Args: gdino_text_threshold = None score_threshold = None molmo_coord_scale = "auto" + yolo_model = None + yolo_conf = 0.05 + yolo_iou = 0.5 + yolo_imgsz = 1024 tiling = "perspective" @@ -212,6 +228,44 @@ def test_pixel_boxes_to_points_drops_boxes_in_the_pad_region(): assert pixel_boxes_to_points(items, 1000, 600) == [(0.05, 1 / 12, 0.5)] +class _FakeYoloBoxes: + """Stands in for ultralytics ``Results.boxes``: .xyxy / .conf as tensors (here, + objects with .tolist()) so yolo_results_to_boxes exercises the tensor path.""" + def __init__(self, xyxy, conf): + self.xyxy = _Tolistable(xyxy) + self.conf = _Tolistable(conf) + + +class _Tolistable: + def __init__(self, data): + self._data = data + + def tolist(self): + return self._data + + +class _FakeYoloResult: + def __init__(self, xyxy, conf): + self.boxes = _FakeYoloBoxes(xyxy, conf) if xyxy is not None else None + + +def test_yolo_results_to_boxes_carries_confidence_through_the_tensor_path(): + # Like the open-vocab detectors, YOLO's per-box score must survive to the scorer + # (that is what earns it AP / a PR curve / a sweep). xyxy are absolute pixels. + res = _FakeYoloResult([[100.0, 200.0, 300.0, 400.0]], [0.42]) + boxes = yolo_results_to_boxes(res) + assert boxes == [{"box": [100.0, 200.0, 300.0, 400.0], "score": 0.42}] + # And it reduces to the same normalized center the open detectors produce. + assert pixel_boxes_to_points(boxes, 1000, 2000) == [(0.2, 0.15, 0.42)] + + +def test_yolo_results_to_boxes_filters_and_survives_no_detections(): + res = _FakeYoloResult([[0, 0, 10, 10], [5, 5, 15, 15]], [0.1, 0.9]) + assert yolo_results_to_boxes(res, threshold=0.5) == [{"box": [5.0, 5.0, 15.0, 15.0], + "score": 0.9}] + assert yolo_results_to_boxes(_FakeYoloResult(None, None)) == [] + + def test_open_vocab_queries_are_short_and_key_the_cache(): # These are not chat models: the query is the prompt, and it must be in the # signature so changing it doesn't silently reuse detections from the old one. @@ -452,10 +506,113 @@ def test_build_detector_rejects_unknown_provider(): build_detector("clip", None, {}, _Args()) except ValueError as e: assert "owlv2" in str(e) # the message lists what is available + assert "yolo" in str(e) # ...including the supervised baseline return raise AssertionError("expected an unknown provider to raise") +def test_build_detector_wires_yolo_and_labels_by_weights_stem(tmp_path): + weights = tmp_path / "yolo11l_tiles.pt" + weights.write_bytes(b"fake-weights") + + class _Y(_Args): + yolo_model = str(weights) + # Bare provider uses --yolo-model; the table label is the file STEM (not the + # absolute path), so the cache is machine-independent. + label, det = build_detector("yolo", None, {}, _Y()) + assert label == "yolo11l_tiles" and isinstance(det, YoloDetector) + assert det.weights == str(weights) and det.model_id == "yolo11l_tiles" + assert det.conf == 0.05 and det.score_threshold == 0.05 and det.tile is True + # A pinned path via yolo: works too and labels the same way. + label2, det2 = build_detector("yolo", str(weights), {}, _Y()) + assert label2 == "yolo11l_tiles" + + +def test_build_detector_yolo_requires_weights(): + class _NoWeights(_Args): + yolo_model = None + try: + build_detector("yolo", None, {}, _NoWeights()) + except ValueError as e: + assert "weights" in str(e).lower() + return + raise AssertionError("expected yolo without weights to raise") + + +def test_yolo_signature_hashes_weights_and_is_machine_independent(tmp_path): + w1 = tmp_path / "a.pt" + w1.write_bytes(b"weights-one") + w2 = tmp_path / "b.pt" + w2.write_bytes(b"weights-two") + # Same label, different weights CONTENT -> different cache identity, so a + # re-trained checkpoint can't silently reuse the old detections. + d1 = YoloDetector(weights=str(w1), label="m") + d2 = YoloDetector(weights=str(w2), label="m") + assert d1.signature()["weights_hash"] != d2.signature()["weights_hash"] + assert cache_key("m", d1.signature(), "c", "p") != cache_key("m", d2.signature(), "c", "p") + # Identity is the label, not a machine-specific path. + assert d1.signature()["model_id"] == "m" + assert d1.score_threshold == d1.conf == 0.05 + # The extra keys live only on YOLO's signature (Gemini's is undisturbed). + extra = set(d1.signature()) - set(GeminiDetector(model_id="gemini-3.6-flash").signature()) + assert extra == {"weights_hash", "conf", "iou", "imgsz"} + # Absent weights -> None hash (score a rsynced cache without the .pt present). + w1.unlink() + assert YoloDetector(weights=str(w1), label="m").signature()["weights_hash"] is None + + +# --- prepare_yolo_dataset.py: box-size + gps-correspondence logic ------------ + +def test_parse_box_size_variants(): + assert parse_box_size("fixed") == ("fixed", 0.03) + assert parse_box_size("fixed:0.05") == ("fixed", 0.05) + assert parse_box_size("pitch") == ("pitch", 0.0) + assert parse_box_size("gps") == ("gps", 0.0) + try: + parse_box_size("bogus") + except Exception: + return + raise AssertionError("expected an invalid box-size to raise") + + +def test_ground_distance_is_monotonic_and_diverges_at_the_horizon(): + # Lower in the frame (larger y) -> steeper down-look -> closer ground point. + near = _ground_distance_m(0.95, 2.5) + far = _ground_distance_m(0.55, 2.5) + assert 0 < near < far + # At / above the horizon the flat-ground model diverges -> inf (the min box). + assert _ground_distance_m(0.5, 2.5) == float("inf") + assert _ground_distance_m(0.3, 2.5) == float("inf") + + +def test_box_wh_fixed_ignores_distance(): + cfg = _prep_cfg(strategy="fixed", fixed_frac=0.04) + assert _box_wh(cfg, 0.5, 0.9, 3.0, 90.0, 90.0) == (0.04, 0.04) + assert _box_wh(cfg, 0.5, 0.6, None, 90.0, 90.0) == (0.04, 0.04) + + +def test_box_wh_distance_aware_shrinks_with_distance_and_clamps(): + cfg = _prep_cfg(strategy="gps") + near_w, _ = _box_wh(cfg, 0.5, 0.8, 3.0, 90.0, 90.0) + far_w, _ = _box_wh(cfg, 0.5, 0.8, 30.0, 90.0, 90.0) + assert far_w < near_w # farther ramp -> smaller box + assert cfg.min_frac <= far_w <= cfg.max_frac + assert cfg.min_frac <= near_w <= cfg.max_frac + # A non-finite distance collapses to the min box, never a crash. + assert _box_wh(cfg, 0.5, 0.8, float("inf"), 90.0, 90.0) == (cfg.min_frac, cfg.min_frac) + + +def test_resolve_distances_gps_needs_matching_point_and_coord_counts(): + cfg = _prep_cfg(strategy="gps") + pts = [(0.1, 0.7), (0.2, 0.8)] + dists, used = _resolve_distances(cfg, pts, [47.6, -122.3], + [[47.6001, -122.3001], [47.6002, -122.3002]]) + assert used and all(d is not None and d > 0 for d in dists) + # Count mismatch -> fall back (None triggers the pitch model downstream). + dists2, used2 = _resolve_distances(cfg, pts, [47.6, -122.3], [[47.6001, -122.3001]]) + assert used2 is False and dists2 == [None, None] + + def test_open_model_detectors_construct_without_weights(): for det in (OwlV2Detector(), GroundingDinoDetector(), MolmoDetector()): assert det._model is None and det._processor is None From ebbc27b2c4d0f3ac5c7b49bb7529ba190d3e6c23 Mon Sep 17 00:00:00 2001 From: Jon Froehlich Date: Tue, 28 Jul 2026 14:46:10 -0700 Subject: [PATCH 2/2] Apply review fixes to the YOLO baseline harness (#51) - Runbook: default ACCOUNT to ckpt-makelab (run_yolo_train.slurm pins the ckpt partition, which the old gpu-l40s default cannot submit to) and pin per-config BATCH in the train stage -- unpinned, jobs ran batch=-1 (auto), defeating the identical-LR-schedule rationale the .slurm header states - Runbook: align BOX_SIZE default with the as-run prep (pitch, ramp 1.8m, per run_yolo_prep.slurm) and point the multi-hour data/prep stages at their compute-node sbatch wrappers (login nodes reap heavy processes) - prepare_yolo_dataset.py: stamp the full prep config into data.yaml as comments -- box size et al. are train-only knobs no downstream args.yaml records, and exactly that gap made the #77 provenance question unanswerable from disk; regression test added (188 pass) - prepare_yolo_dataset.py: render each overlay view once, not twice; --overlay-n help now says PER SPLIT - detectors.py: hoist the hashlib import to module top Co-Authored-By: Claude Opus 4.8 --- hyak_yolo_runbook.sh | 46 +++++++++++++------ scripts/model_comparison/detectors.py | 2 +- .../model_comparison/prepare_yolo_dataset.py | 29 +++++++++--- tests/test_model_comparison.py | 27 ++++++++++- 4 files changed, 81 insertions(+), 23 deletions(-) diff --git a/hyak_yolo_runbook.sh b/hyak_yolo_runbook.sh index 55a43e5..80a14ec 100644 --- a/hyak_yolo_runbook.sh +++ b/hyak_yolo_runbook.sh @@ -35,8 +35,13 @@ PYBIN="${PYBIN:-$ENVDIR/bin/python}" DATA="${DATA:-$SCRATCH/rampnet_dataset}" # HF dataset lands here (via ./dataset symlink) YOLODATA="${YOLODATA:-$SCRATCH/yolo}" # prepared tiles/ and pano/ datasets PROJECT="${PROJECT:-$SCRATCH/yolo_runs}" # training outputs (weights) -ACCOUNT="${ACCOUNT:-gpu-l40s-makelab}" # -A for sbatch; override if yours differs -BOX_SIZE="${BOX_SIZE:-fixed:0.03}" # A/B pitch/gps later by re-running prep+train +# -A for the `train` sbatch: run_yolo_train.slurm pins the ckpt (scavenger) partition, +# which needs the ckpt account. The data/prep slurm wrappers use gpu-l40s-makelab. +ACCOUNT="${ACCOUNT:-ckpt-makelab}" +# As-run choice (see run_yolo_prep.slurm): fixed:0.03 was too small on the overlays, +# and gps fell back to pitch on ~85% of panos — pitch is the consistent strategy. +BOX_SIZE="${BOX_SIZE:-pitch}" +RAMP_SIZE_M="${RAMP_SIZE_M:-1.8}" # physical ramp size the pitch model assumes BG_KEEP="${BG_KEEP:-0.15}" # keep 15% of background tiles (tames the skew) # YOLO26's checkpoint name/availability depends on the ultralytics version — check # `python -c "from ultralytics import YOLO; YOLO('yolo26l.pt')"` and override YOLO26. @@ -95,6 +100,9 @@ PY # --------------------------------------------------------------------------- data) banner "Download the RampNet dataset (~214k panos) from HF onto scratch" + # Login nodes reap heavy long-running processes (and the SSH master with them) — + # for the real multi-hour download, submit this stage as a compute-node job: + # sbatch -A gpu-l40s-makelab scripts/model_comparison/run_yolo_data.slurm mkdir -p "$DATA" ln -sfn "$DATA" "$REPO/dataset" # download_dataset.py writes ./dataset/{train,val,test} cd "$REPO" @@ -114,7 +122,8 @@ prepsmoke) cd "$REPO" "$PYBIN" scripts/model_comparison/prepare_yolo_dataset.py \ --dataset-root "$REPO/dataset" --out "$YOLODATA/tiles_smoke" \ - --geometry tiles --box-size "$BOX_SIZE" --bg-keep-frac "$BG_KEEP" \ + --geometry tiles --box-size "$BOX_SIZE" --ramp-size-m "$RAMP_SIZE_M" \ + --bg-keep-frac "$BG_KEEP" \ --subset 200 --overlay-dir "$YOLODATA/overlay_smoke" echo echo "scp $YOLODATA/overlay_smoke/*.jpg back and check red boxes sit ON ramps." @@ -124,14 +133,18 @@ prepsmoke) # --------------------------------------------------------------------------- prep) - banner "Full tiles + pano datasets (box-size=$BOX_SIZE, bg-keep=$BG_KEEP)" + banner "Full tiles + pano datasets (box-size=$BOX_SIZE, ramp=${RAMP_SIZE_M}m, bg-keep=$BG_KEEP)" + # The 150k-pano tiling is multi-hour and CPU-heavy — on klone, submit it as a + # compute-node job instead of running here on a login node: + # sbatch -A gpu-l40s-makelab scripts/model_comparison/run_yolo_prep.slurm cd "$REPO" "$PYBIN" scripts/model_comparison/prepare_yolo_dataset.py \ --dataset-root "$REPO/dataset" --out "$YOLODATA/tiles" \ - --geometry tiles --box-size "$BOX_SIZE" --bg-keep-frac "$BG_KEEP" + --geometry tiles --box-size "$BOX_SIZE" --ramp-size-m "$RAMP_SIZE_M" \ + --bg-keep-frac "$BG_KEEP" "$PYBIN" scripts/model_comparison/prepare_yolo_dataset.py \ --dataset-root "$REPO/dataset" --out "$YOLODATA/pano" \ - --geometry pano --box-size "$BOX_SIZE" + --geometry pano --box-size "$BOX_SIZE" --ramp-size-m "$RAMP_SIZE_M" echo "OK. Next: bash hyak_yolo_runbook.sh train" ;; @@ -144,16 +157,21 @@ train) # reach the job. PYTHON=the lean venv; the .slurm falls back to conda otherwise. export PYTHON="$PYBIN" HF_HOME YOLO_CONFIG_DIR PROJECT SLURM=scripts/model_comparison/run_yolo_train.slurm - sub() { # sub - YOLO_CKPT="$1" YOLO_DATA="$2" YOLO_IMGSZ="$3" NAME="$4" \ + sub() { # sub + YOLO_CKPT="$1" YOLO_DATA="$2" YOLO_IMGSZ="$3" BATCH="$4" NAME="$5" \ sbatch -A "$ACCOUNT" "$SLURM" } - sub yolo11l.pt "$YOLODATA/tiles/data.yaml" 1024 y11l_tiles - sub yolo11x.pt "$YOLODATA/tiles/data.yaml" 1024 y11x_tiles - sub "$YOLO26" "$YOLODATA/tiles/data.yaml" 1024 y26_tiles - sub yolo11l.pt "$YOLODATA/pano/data.yaml" 1280 y11l_pano - sub yolo11x.pt "$YOLODATA/pano/data.yaml" 1280 y11x_pano - sub "$YOLO26" "$YOLODATA/pano/data.yaml" 1280 y26_pano + # Batches PINNED to the as-run values (sized for >=45G GPUs) so the LR schedule is + # identical wherever a ckpt job lands or resumes — see run_yolo_train.slurm. + # Caveats from the 2026-07 runs: y11x_tiles never finished an epoch inside a ckpt + # scheduling slice (dropped 2026-07-27, even after a 3->12 batch bump) — run it only + # on a non-preemptable partition; YOLO11-pano collapsed at physical batch 2-4 (#70). + sub yolo11l.pt "$YOLODATA/tiles/data.yaml" 1024 6 y11l_tiles + sub yolo11x.pt "$YOLODATA/tiles/data.yaml" 1024 12 y11x_tiles + sub "$YOLO26" "$YOLODATA/tiles/data.yaml" 1024 6 y26_tiles + sub yolo11l.pt "$YOLODATA/pano/data.yaml" 1280 4 y11l_pano + sub yolo11x.pt "$YOLODATA/pano/data.yaml" 1280 2 y11x_pano + sub "$YOLO26" "$YOLODATA/pano/data.yaml" 1280 4 y26_pano squeue -u "$USER" echo "OK. Watch: bash hyak_yolo_runbook.sh status" ;; diff --git a/scripts/model_comparison/detectors.py b/scripts/model_comparison/detectors.py index dbb7a64..f7f07a1 100644 --- a/scripts/model_comparison/detectors.py +++ b/scripts/model_comparison/detectors.py @@ -25,6 +25,7 @@ gets AP / PR / sweep like the open-vocab detectors; tiled by default, with ``--tiling none`` as the whole-pano ablation. """ +import hashlib import importlib.util import json import os @@ -941,7 +942,6 @@ def _weights_fingerprint(path): stable HF id. Machine-independent (hashes bytes, not the path). Returns ``None`` if the file is absent (e.g. scoring a rsynced cache without the weights present), falling back to the label for identity.""" - import hashlib try: h = hashlib.sha1() with open(path, "rb") as f: diff --git a/scripts/model_comparison/prepare_yolo_dataset.py b/scripts/model_comparison/prepare_yolo_dataset.py index d49dd9d..4a71891 100644 --- a/scripts/model_comparison/prepare_yolo_dataset.py +++ b/scripts/model_comparison/prepare_yolo_dataset.py @@ -214,12 +214,13 @@ def process_pano(task): if not lines and not _keep_background(cfg, stem): st["bg_skipped"] += 1 continue - _write_pair(cfg.out, yolo_split, stem, equirect_to_perspective(src, view), lines) + view_img = equirect_to_perspective(src, view) + _write_pair(cfg.out, yolo_split, stem, view_img, lines) st["tiles"] += 1 st["boxes"] += len(lines) st["bg_tiles"] += 1 if not lines else 0 if overlay and cfg.overlay_dir: - _save_overlay(cfg.overlay_dir, stem, equirect_to_perspective(src, view), lines) + _save_overlay(cfg.overlay_dir, stem, view_img, lines) else: # pano pimg = img.resize((cfg.pano_w, cfg.pano_h), Image.BILINEAR) lines = [] @@ -247,9 +248,22 @@ def parse_box_size(s): raise argparse.ArgumentTypeError("box-size must be fixed[:frac], pitch, or gps") -def write_data_yaml(out): - with open(os.path.join(out, "data.yaml"), "w") as f: - f.write(f"# Generated by prepare_yolo_dataset.py\npath: {os.path.abspath(out)}\n") +def write_data_yaml(cfg, args): + strategy, fixed_frac = args.box_size + box = f"fixed:{fixed_frac}" if strategy == "fixed" else strategy + with open(os.path.join(cfg.out, "data.yaml"), "w") as f: + # Stamp the full prep config as comments: box size et al. are train-only knobs + # that no downstream record (ultralytics args.yaml) captures, so the dataset's + # provenance must be answerable from disk. + f.write("# Generated by prepare_yolo_dataset.py\n") + f.write(f"# geometry={cfg.geometry} box-size={box} ramp-size-m={cfg.ramp_size_m} " + f"camera-height-m={cfg.camera_height_m}\n") + f.write(f"# min-frac={cfg.min_frac} max-frac={cfg.max_frac} " + f"bg-keep-frac={cfg.bg_keep_frac} source-max-edge={cfg.source_max_edge}\n") + f.write(f"# n-yaw={args.n_yaw} view-fov={args.view_fov} view-pitch={args.view_pitch} " + f"view-size={args.view_size} pano-width={args.pano_width} " + f"subset={args.subset} dataset-root={os.path.abspath(args.dataset_root)}\n") + f.write(f"path: {cfg.out}\n") f.write("train: images/train\nval: images/val\n") f.write(f"names:\n 0: {CLASS_NAME}\n") @@ -312,7 +326,8 @@ def main(): ap.add_argument("--pano-width", type=int, default=2048, help="Width of the whole-pano image for --geometry pano (height = width/2).") ap.add_argument("--overlay-dir", help="Also render boxes-on-tiles for QA.") - ap.add_argument("--overlay-n", type=int, default=20, help="Panos to overlay (default 20).") + ap.add_argument("--overlay-n", type=int, default=20, + help="Panos to overlay PER SPLIT (default 20).") args = ap.parse_args() cfg = build_config(args) @@ -338,7 +353,7 @@ def main(): if n % 2000 == 0: print(f" {n}/{len(tasks)} panos | {agg['tiles']} imgs | {agg['boxes']} boxes") - write_data_yaml(cfg.out) + write_data_yaml(cfg, args) print(f"\nDone. panos={agg['panos']} images={agg['tiles']} boxes={agg['boxes']} " f"bg_images={agg['bg_tiles']} bg_skipped={agg['bg_skipped']} " f"points={agg['points']} read_errors={agg['read_errors']}") diff --git a/tests/test_model_comparison.py b/tests/test_model_comparison.py index d1c9b31..9840715 100644 --- a/tests/test_model_comparison.py +++ b/tests/test_model_comparison.py @@ -32,7 +32,8 @@ ) from rampnet.detection_eval import GroundTruth, radius_sq_for # noqa: E402 from prepare_yolo_dataset import ( # noqa: E402 - parse_box_size, _ground_distance_m, _box_wh, _resolve_distances, Config as YoloPrepConfig, + parse_box_size, _ground_distance_m, _box_wh, _resolve_distances, write_data_yaml, + Config as YoloPrepConfig, ) @@ -613,6 +614,30 @@ def test_resolve_distances_gps_needs_matching_point_and_coord_counts(): assert used2 is False and dists2 == [None, None] +def test_write_data_yaml_stamps_prep_provenance(tmp_path): + # Box size / bg-keep are train-only knobs that NO downstream record captures + # (ultralytics args.yaml never sees them) — data.yaml is where the dataset's + # provenance must live, as comments the YAML parser ignores. + class _P: + box_size = ("pitch", 0.0) + n_yaw = 6 + view_fov = 90.0 + view_pitch = -30.0 + view_size = 1024 + pano_width = 2048 + subset = None + dataset_root = "dataset" + cfg = _prep_cfg(strategy="pitch", ramp_size_m=1.8, bg_keep_frac=0.15, out=str(tmp_path)) + write_data_yaml(cfg, _P()) + text = (tmp_path / "data.yaml").read_text() + assert "box-size=pitch" in text and "ramp-size-m=1.8" in text + assert "bg-keep-frac=0.15" in text and "view-size=1024" in text + # The stamp is comments only — the mapping ultralytics reads is untouched. + data_lines = [l for l in text.splitlines() if l and not l.startswith("#")] + assert data_lines[0] == f"path: {tmp_path}" + assert "train: images/train" in data_lines and "val: images/val" in data_lines + + def test_open_model_detectors_construct_without_weights(): for det in (OwlV2Detector(), GroundingDinoDetector(), MolmoDetector()): assert det._model is None and det._processor is None