From 5f893c1ad705a285e3ab816c00913404a281ca74 Mon Sep 17 00:00:00 2001 From: hanhainebula <2512674094@qq.com> Date: Thu, 19 Nov 2026 18:01:55 +0800 Subject: [PATCH 1/5] Support cgroup v1 and v2 resource verification --- docs/evaluation/cli.md | 9 +- docs/evaluation/runtime.md | 39 +++ environments/cgroups/cgroup_resources.py | 205 ++++++++++++++ environments/cgroups/v1_oom.py | 91 ++++++ scripts/evaluation/harbor_environments.py | 30 ++ scripts/evaluation/v1_oom.py | 92 ++++++ src/search_swe/regrade.py | 5 +- src/search_swe/runner.py | 59 +++- .../tests/cgroup_resources.py | 205 ++++++++++++++ .../tests/docker-compose.yaml | 3 + .../tests/grader.py | 40 ++- .../budgeted-bimetric-retrieval/tests/test.sh | 2 +- .../tests/v1_oom.py | 91 ++++++ .../environment/docs/environment.md | 2 +- .../environment/runtime/cgroup_resources.py | 205 ++++++++++++++ .../environment/runtime/evaluate.py | 22 +- .../tests/grader.py | 3 + .../tests/runtime/cgroup_resources.py | 205 ++++++++++++++ .../tests/runtime/evaluate.py | 22 +- .../tests/test.sh | 2 +- .../environment/docs/environment.md | 2 +- .../environment/runtime/baseline_model.py | 7 +- .../runtime/baseline_properties.py | 10 +- .../environment/runtime/cgroup_resources.py | 205 ++++++++++++++ .../environment/runtime/evaluate.py | 19 +- .../tests/docs/environment.md | 2 +- .../tests/runtime/baseline_model.py | 7 +- .../tests/runtime/baseline_properties.py | 10 +- .../tests/runtime/cgroup_resources.py | 205 ++++++++++++++ .../tests/runtime/evaluate.py | 19 +- .../tests/test.sh | 2 +- .../tests/test_gpu_resources.py | 14 +- .../environment/Dockerfile | 2 +- .../environment/budget_evidence.py | 27 +- .../environment/cgroup_resources.py | 205 ++++++++++++++ .../environment/docker-compose.yaml | 1 + .../environment/v1_oom.py | 91 ++++++ .../tests/cgroup_resources.py | 205 ++++++++++++++ .../tests/development.py | 8 + .../tests/docker-compose.yaml | 3 + .../tests/grader.py | 28 +- .../tests/runtime.py | 4 +- .../tests/test.sh | 2 +- .../tests/test_development.py | 19 ++ .../tests/test_runtime.py | 4 +- .../tests/v1_oom.py | 91 ++++++ .../README.md | 2 +- .../tests/cgroup_resources.py | 205 ++++++++++++++ .../tests/grader.py | 30 +- .../tests/test.sh | 2 +- .../authoring/tests/test_runtime_limits.py | 13 +- .../tests/cgroup_resources.py | 205 ++++++++++++++ .../tests/efficiency_grade.py | 5 + tasks/training-error-tracing/tests/grade.py | 41 +-- tasks/training-error-tracing/tests/test.sh | 2 +- tests/unit/test_cgroup_resources.py | 266 ++++++++++++++++++ tests/unit/test_cli.py | 86 +++++- tests/unit/test_submission_java.py | 7 +- 58 files changed, 3268 insertions(+), 120 deletions(-) create mode 100644 environments/cgroups/cgroup_resources.py create mode 100644 environments/cgroups/v1_oom.py create mode 100644 scripts/evaluation/v1_oom.py create mode 100644 tasks/budgeted-bimetric-retrieval/tests/cgroup_resources.py create mode 100644 tasks/budgeted-bimetric-retrieval/tests/v1_oom.py create mode 100644 tasks/budgeted-candidate-reranking/environment/runtime/cgroup_resources.py create mode 100644 tasks/budgeted-candidate-reranking/tests/runtime/cgroup_resources.py create mode 100644 tasks/constrained-molecular-search/environment/runtime/cgroup_resources.py create mode 100644 tasks/constrained-molecular-search/tests/runtime/cgroup_resources.py create mode 100644 tasks/large-scale-formula-retrieval/environment/cgroup_resources.py create mode 100644 tasks/large-scale-formula-retrieval/environment/v1_oom.py create mode 100644 tasks/large-scale-formula-retrieval/tests/cgroup_resources.py create mode 100644 tasks/large-scale-formula-retrieval/tests/v1_oom.py create mode 100644 tasks/memory-constrained-dense-retrieval/tests/cgroup_resources.py create mode 100644 tasks/training-error-tracing/tests/cgroup_resources.py create mode 100644 tests/unit/test_cgroup_resources.py diff --git a/docs/evaluation/cli.md b/docs/evaluation/cli.md index 64a593a..1918900 100644 --- a/docs/evaluation/cli.md +++ b/docs/evaluation/cli.md @@ -234,8 +234,13 @@ Each batch creates `jobs/evaluations//` or a new `--output` director | `logs/` | Asset preparation and evaluation logs | | Harbor result paths recorded in `run.json` | Scores, verifier health, trajectories and artifacts | -`completed` means the launcher exited successfully; check Harbor reports for -scores and verifier health. Ctrl-C/SIGTERM cancels active runs and lets storage +`completed` requires a successful launch, a finished Harbor trial and verifier, +and a primary `reward` between 0 and 1 inclusive. A graded zero counts as completed, including when the +agent timed out and the verifier scored its work. A setup failure or missing +trial result marks the task `failed`. Invalid cgroup evidence marks it +`invalid-measurement`. Both statuses give the batch a nonzero exit code. + +Ctrl-C/SIGTERM cancels active runs and lets storage teardown finish. Cleanup failure stops the batch, cancels other active runs, records `cleanup-error` and leaves queued tasks `not-started`; follow the recovery instructions before another batch. Tasks with diff --git a/docs/evaluation/runtime.md b/docs/evaluation/runtime.md index 3071992..49ded3f 100644 --- a/docs/evaluation/runtime.md +++ b/docs/evaluation/runtime.md @@ -22,6 +22,45 @@ checks. Task libraries run in the published CPU/GPU [base images](../../docker/README.md); ordinary runs pull these images. See [data and models](assets.md) to plan space for task copies, cache and outputs. +## Linux cgroups + +The resource checks use the current container's CPU and memory controllers. +This lets the same task budgets apply across different CPU and GPU models. + +| Task | cgroup v1 requirements | +| --- | --- | +| `constrained-molecular-search` | CPU CFS quota and a finite aggregate RAM limit | +| `budgeted-candidate-reranking` | A finite RAM limit and the kernel's `oom_kill` counter for the container's memory cgroup | +| `memory-constrained-dense-retrieval`, `training-error-tracing` | A finite RAM limit, a container-scoped `oom_kill` counter and no active host swap | +| `budgeted-bimetric-retrieval`, `large-scale-formula-retrieval` | A finite RAM limit and no active host swap. The launcher subscribes to the kernel’s OOM notifications for the task and its parent. | + +On v1, the effective RAM cap includes ancestor memory and combined RAM+swap +limits reported by the kernel, even when the container has a private cgroup +namespace. The v1 OOM-kill checks use a leaf memory cgroup, the normal layout for a +single Docker task container. The CPU affinity requirements and GPU +requirements still apply. A missing diagnostic memory peak is recorded as +`null`. Missing required evidence produces a verifier +`cgroup-status.json` with `measurement_valid: false` and the failing check. +For Formula, the agent environment also checks for the required evidence +before starting the coding agent. If that health check fails, the batch records +a failed trial with the Harbor exception type. + +For Bimetric and Formula, the launcher registers v1 notifications on the host +and exposes a read-only bind mount with a private socket to the trusted task +process. This distinguishes successful page-cache reclaim from an OOM, including +an OOM without a kill. Bimetric checks the interval after verifier startup. +Formula also checks the full development container lifetime. If notifications +are unavailable, untouched RAM and combined RAM+swap limits can still establish +a clean run. Formula also recognizes a kernel-recorded OOM kill. Unattributable +pressure produces an invalid measurement. + +The probes were verified on Linux 6.8 in a Docker v2 container and an OCI +container on a v1 guest, including successful reclaim and a kernel OOM kill. + +See the [Linux cgroup v1 memory controller](https://docs.kernel.org/admin-guide/cgroup-v1/memory.html), +[cgroup v2 interface](https://docs.kernel.org/admin-guide/cgroup-v2.html) +and [CPU bandwidth control](https://docs.kernel.org/scheduler/sched-bwc.html). + ## Select a task Use `search-swe list` to discover `tasks/*/task.toml`. Read the chosen package: diff --git a/environments/cgroups/cgroup_resources.py b/environments/cgroups/cgroup_resources.py new file mode 100644 index 0000000..ef60a48 --- /dev/null +++ b/environments/cgroups/cgroup_resources.py @@ -0,0 +1,205 @@ +"""Read Linux cgroup counters for the current process and its container. + +Canonical source is environments/cgroups/cgroup_resources.py. Task image +contexts contain identical copies so they can be built independently. +""" + +from dataclasses import dataclass +from functools import cache +import json +import os +from pathlib import Path +import re + + +class CgroupError(RuntimeError): + """The environment cannot establish the task's required resource budget.""" + + +def _read(path): + try: + return path.read_text(encoding="ascii") + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + +def _integer(value, path, minimum=0): + try: + result = int(value) + if result >= minimum: + return result + except ValueError: + pass + raise CgroupError(f"Invalid cgroup value in {path}: {value!r}") + + +def _mount_path(value): + return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + + +@cache +def controller(name): + """Resolve a controller using this process's membership and visible mounts.""" + return resolve_controller(name, Path("/proc/self/cgroup")) + + +def resolve_controller(name, membership_file): + """Resolve a process's membership against mounts visible to the reader.""" + memberships = {} + for line in _read(membership_file).splitlines(): + fields = line.split(":", 2) + if len(fields) != 3: + raise CgroupError(f"Invalid cgroup membership: {line!r}") + _, controllers, location = fields + for key in controllers.split(","): + memberships[key] = Path(location) + # A hybrid host can mount v2 alongside active v1 controllers. + version = 1 if name in memberships else 2 + member = memberships.get(name if version == 1 else "") + if member is None or not member.is_absolute() or ".." in member.parts: + raise CgroupError(f"No visible cgroup membership for the {name} controller") + candidates = [] + for line in _read(Path("/proc/self/mountinfo")).splitlines(): + before, _, after = line.partition(" - ") + mount, filesystem = before.split(), after.split() + if len(mount) < 6 or len(filesystem) < 3: + continue + if version == 1: + if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + continue + elif filesystem[0] != "cgroup2": + continue + root, target = map(_mount_path, mount[3:5]) + if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): + candidates.append((len(root.parts), target / member.relative_to(root))) + if not candidates: + raise CgroupError(f"No cgroup v{version} mount covers {name} membership {member}") + _, directory = max(candidates) + return Controller(version, directory) + + +@dataclass(frozen=True) +class Controller: + version: int + path: Path + + def _text(self, name): + return _read(self.path / name) + + def _number(self, name, minimum=0): + return _integer(self._text(name).strip(), self.path / name, minimum) + + def cpu_quota(self): + """Return (quota in microseconds or None for unlimited, period in microseconds).""" + if self.version == 1: + quota = self._number("cpu.cfs_quota_us", -1) + period = self._number("cpu.cfs_period_us", 1) + if quota == 0: + raise CgroupError(f"Invalid zero CPU quota in {self.path / 'cpu.cfs_quota_us'}") + return (None if quota == -1 else quota), period + path = self.path / "cpu.max" + fields = _read(path).split() + if len(fields) != 2: + raise CgroupError(f"Expected quota and period in {path}") + quota, period = fields + return (None if quota == "max" else _integer(quota, path, 1)), _integer(period, path, 1) + + def memory_limit(self): + if self.version == 2: + value = self._text("memory.max").strip() + return None if value == "max" else _integer(value, self.path / "memory.max", 1) + # memory.stat includes the tightest ancestor even in a private cgroup namespace. + path = self.path / "memory.stat" + bounds = {} + for line in _read(path).splitlines(): + fields = line.split() + if fields and fields[0] in ("hierarchical_memory_limit", "hierarchical_memsw_limit"): + if len(fields) != 2 or fields[0] in bounds: + raise CgroupError(f"Malformed {fields[0]} in {path}") + bounds[fields[0]] = _integer(fields[1], path) + if "hierarchical_memory_limit" not in bounds: + raise CgroupError(f"Missing hierarchical_memory_limit in {path}") + if "hierarchical_memsw_limit" not in bounds and (self.path / "memory.memsw.limit_in_bytes").exists(): + raise CgroupError(f"Missing hierarchical_memsw_limit in {path}") + limit = min(self._number("memory.limit_in_bytes"), *bounds.values()) + # v1 uses page-rounded LONG_MAX to represent an unbounded limit. + page_size = os.sysconf("SC_PAGE_SIZE") + unlimited = (2**63 - 1) // page_size * page_size + return None if limit >= unlimited else limit + + def swap_limit(self): + """Return a proven swap-only byte limit, or None when v2 allows 'max'.""" + if self.version == 2: + name = "memory.swap.max" + value = self._text(name).strip() + return None if value == "max" else _integer(value, self.path / name) + # The v1 combined RAM+swap limit does not prove zero swapped pages. + # Linux cannot swap when the host has no active swap devices or files. + swaps = _read(Path("/proc/swaps")).splitlines() + if swaps and swaps[0].split() == ["Filename", "Type", "Size", "Used", "Priority"] and not any( + line.strip() for line in swaps[1:]): + return 0 + raise CgroupError("cgroup v1 cannot verify zero swap with active or unreadable host swap") + + def memory_charge(self): + name = "memory.usage_in_bytes" if self.version == 1 else "memory.current" + return self._number(name) + + def memory_peak(self): + """Peak is a report-only kernel counter.""" + name = "memory.max_usage_in_bytes" if self.version == 1 else "memory.peak" + try: + return self._number(name) + except CgroupError: + return None + + def cpu_stat(self): + return self._text("cpu.stat") + + @cache + def _require_v1_leaf(self): + # A v1 counter can be local. A leaf has the same process scope as + # the hierarchical v2 counter, including forked submission processes. + try: + if any(path.is_dir() for path in self.path.iterdir()): + raise CgroupError(f"Cannot establish container-wide v1 OOM counts in {self.path} with child cgroups") + except OSError as error: + raise CgroupError(f"Cannot inspect the v1 memory cgroup {self.path}: {error}") from error + + def v1_limit_hits(self): + # v1 may reject a combined memory+swap charge before charging RAM. + # Kernels without swap accounting omit the memsw counter entirely. + try: + combined = (self.path / "memory.memsw.failcnt").read_text(encoding="ascii") + except FileNotFoundError: + combined = None + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read memory.memsw.failcnt: {error}") from error + return [self._number("memory.failcnt"), + None if combined is None else _integer(combined.strip(), self.path / "memory.memsw.failcnt")] + + def memory_events(self, *required): + name = "memory.oom_control" if self.version == 1 else "memory.events" + if self.version == 1: + if "oom" in required: + raise CgroupError(f"{self.path / name} does not expose a cumulative oom event count") + self._require_v1_leaf() + raw = self._text(name) + counters = {} + for line in raw.splitlines(): + fields = line.split() + if len(fields) != 2 or fields[0] in counters: + raise CgroupError(f"Malformed cgroup events in {self.path / name}") + key, value = fields + counters[key] = _integer(value, self.path / name) + for key in required: + if key not in counters: + raise CgroupError(f"Missing {key} counter in {self.path / name}") + return raw, counters + + +def write_resource_error(directory, error): + """Record why a verifier could not measure a resource-constrained task.""" + path = Path(directory) / "cgroup-status.json" + path.write_text(json.dumps({"measurement_valid": False, "failure_kind": "cgroup_environment", + "error": str(error)}) + "\n") diff --git a/environments/cgroups/v1_oom.py b/environments/cgroups/v1_oom.py new file mode 100644 index 0000000..da33623 --- /dev/null +++ b/environments/cgroups/v1_oom.py @@ -0,0 +1,91 @@ +"""Container-side OOM evidence shared by Formula and Bimetric on cgroup v1.""" + +import json +import socket + +if __package__: + from .cgroup_resources import CgroupError +else: # Independent task image. + from cgroup_resources import CgroupError + + +SOCKET = "/run/search-swe-memory/oom.sock" + + +def snapshot(memory): + """Sample the limit counters first so the notification check covers that interval.""" + hits = memory.v1_limit_hits() + _, counters = memory.memory_events("oom_kill") + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(5) + client.connect(SOCKET) + with client.makefile("rb") as stream: + watch = json.loads(stream.readline(4096)) + except (FileNotFoundError, PermissionError, ConnectionError, TimeoutError): + watch = None # Raw cgroup counters can still establish some outcomes. + except (OSError, ValueError) as error: + raise CgroupError(f"Cannot read v1 OOM observer: {error}") from error + if watch is not None: + try: + info = memory.path.stat() + except OSError as error: + raise CgroupError(f"Cannot identify the task memory cgroup: {error}") from error + if not isinstance(watch, dict) or watch.get("cgroup") != [info.st_dev, info.st_ino]: + raise CgroupError("Host OOM observer and task see different memory cgroups") + return {"limit_hits": hits, "oom_kill": counters["oom_kill"], "observer": watch} + + +def _counter(value): + return type(value) is int and value >= 0 + + +def _limit_hits(value): + return (isinstance(value, list) and len(value) == 2 and _counter(value[0]) + and (value[1] is None or _counter(value[1]))) + + +def _validate(sample): + if (not isinstance(sample, dict) or not _limit_hits(sample.get("limit_hits")) + or not _counter(sample.get("oom_kill")) or "observer" not in sample): + raise CgroupError("Invalid v1 memory counters") + watch = sample["observer"] + if watch is not None and (not isinstance(watch, dict) or not isinstance(watch.get("id"), str) + or not watch["id"] or not _counter(watch.get("events")) or not _counter(watch.get("parent_events")) + or not _limit_hits(watch.get("initial_hits"))): + raise CgroupError("Invalid v1 OOM notification evidence") + + +def violation(sample, baseline=None, *, include_kills): + """Formula checks the container lifetime. Bimetric supplies a verifier baseline.""" + _validate(sample) + hits = sample["limit_hits"] + old_hits = [0, None if hits[1] is None else 0] + old_kills = 0 + old_watch = None + if baseline is not None: + _validate(baseline) + old_hits, old_kills, old_watch = baseline["limit_hits"], baseline["oom_kill"], baseline["observer"] + if (sample["oom_kill"] < old_kills + or any((a is None) != (b is None) or (a is not None and a < b) for a, b in zip(hits, old_hits))): + raise CgroupError("Reset v1 memory counters") + if include_kills and sample["oom_kill"] > old_kills: + return True + watch = sample["observer"] + if watch and (baseline is None or (old_watch and old_watch["id"] == watch["id"])): + start = old_watch["events"] if old_watch else 0 + if watch["events"] < start or (old_watch and watch["parent_events"] < old_watch["parent_events"]): + raise CgroupError("Reset v1 OOM notifications") + # Ancestor OOMs notify descendants too. A plain eventfd cannot + # attribute a notification if an ancestor was also under pressure. + if watch["parent_events"] == 0: + if watch["events"] > start: + return True + if old_watch is not None or not any(watch["initial_hits"]): + return False + # With stable caps, local memcg OOM first requires a failed RAM or memsw + # charge. Successful reclaim also increments these counters. + if hits == old_hits: + return False + raise CgroupError("The v1 memory limit was reached, but local OOM could not be distinguished " + "from reclaim or ancestor pressure in this measurement") diff --git a/scripts/evaluation/harbor_environments.py b/scripts/evaluation/harbor_environments.py index c4d0f47..b00c099 100644 --- a/scripts/evaluation/harbor_environments.py +++ b/scripts/evaluation/harbor_environments.py @@ -19,7 +19,9 @@ from scripts.egress.compose import SERVICE, declared_services, is_separate_verifier_context, validate_final from scripts.egress.config import DEFAULT_IMAGE, direct_config, exact_hosts, load_config, require_supported_harbor from scripts.egress.ownership import ProjectLock, PrivateDirectory, owner_root +from environments.cgroups.cgroup_resources import controller from environments.egress.gateway import LEASE_SECONDS +from scripts.evaluation.v1_oom import V1OomObserver from scripts.evaluation.gpu_device import GPU_UUID, select_compose_gpu from scripts.evaluation.cpu_affinity import normalize_compose_cpu_quota, parse_cpu_set, select_compose_cpus from scripts.evaluation.bounded_logs import BoundedLogs @@ -71,6 +73,33 @@ def __init__(self, *args, gpu_device=None, cpu_set=None, **kwargs): self.cpu_set = cpu_set super().__init__(*args, **kwargs) + async def start(self, force_build): + self._oom_observer = None + source = self._environment_docker_compose_path + if source.is_file(): + extension = (yaml.safe_load(source.read_text()) or {}).get("x-search-swe", {}) + if extension.get("v1_oom_notifications") is True and controller("memory").version == 1: + self._oom_observer = V1OomObserver() + try: + if self._oom_observer: + self._mounts.append(await self._oom_observer.start()) + await super().start(force_build) + except BaseException: + self._close_oom_observer() + raise + + def _close_oom_observer(self): + if getattr(self, "_oom_observer", None) is not None: + self._mounts = [mount for mount in self._mounts if mount.get("source") != self._oom_observer.directory.name] + self._oom_observer.close() + self._oom_observer = None + + async def stop(self, delete): + try: + await super().stop(delete) + finally: + self._close_oom_observer() + @property def _docker_compose_paths(self): paths = super()._docker_compose_paths @@ -520,6 +549,7 @@ async def stop(self, delete): if cancelled: raise asyncio.CancelledError finally: + self._close_oom_observer() self._release_owner() async def _stop_owned(self, delete): diff --git a/scripts/evaluation/v1_oom.py b/scripts/evaluation/v1_oom.py new file mode 100644 index 0000000..df84474 --- /dev/null +++ b/scripts/evaluation/v1_oom.py @@ -0,0 +1,92 @@ +"""Expose v1 kernel OOM notifications to trusted graders on read-only cgroup mounts.""" + +import asyncio +from contextlib import ExitStack +import json +import os +from pathlib import Path +import socket +import struct +import tempfile +import uuid + +from environments.cgroups.cgroup_resources import CgroupError, resolve_controller +from environments.cgroups.v1_oom import SOCKET + + +class OomNotifications: + def __init__(self, memory): + self.files = ExitStack() + info = memory.path.stat() + self.cgroup = [info.st_dev, info.st_ino] + self.id = uuid.uuid4().hex + self.counts = [0, 0] + try: + if memory.memory_events("oom_kill_disable")[1]["oom_kill_disable"] != 0: + raise CgroupError("v1 OOM notifications require the kernel OOM killer") + parent = memory.path.parent + # An ancestor's OOM is also announced to its descendants. Register + # at the parent first to distinguish this from an OOM in the task. + self.parent_fd = self.register(parent) + self.fd = self.register(memory.path) + self.initial_hits = memory.v1_limit_hits() + except BaseException: + self.files.close() + raise + + def register(self, directory): + fd = os.eventfd(0, os.EFD_CLOEXEC | os.EFD_NONBLOCK) + self.files.callback(os.close, fd) + with (directory / "memory.oom_control").open("rb") as control: + with (directory / "cgroup.event_control").open("wb", buffering=0) as registration: + registration.write(f"{fd} {control.fileno()}".encode()) + return fd + + def sample(self): + for i, fd in enumerate((self.fd, self.parent_fd)): + try: + self.counts[i] += os.eventfd_read(fd) + except BlockingIOError: + pass + return {"id": self.id, "cgroup": self.cgroup, "events": self.counts[0], "parent_events": self.counts[1], + "initial_hits": self.initial_hits} + + +class V1OomObserver: + """One private endpoint per task phase. The peer cannot select a host path.""" + def __init__(self): + self.directory = tempfile.TemporaryDirectory(prefix="search-swe-oom-", dir="/tmp") + self.memory = self.watch = None + self.server = None + + async def start(self): + path = Path(self.directory.name) / Path(SOCKET).name + self.server = await asyncio.start_unix_server(self.respond, path=path) + path.chmod(0o600) + return {"type": "bind", "source": self.directory.name, + "target": str(Path(SOCKET).parent), "read_only": True} + + def respond(self, _reader, writer): + sample = None + try: + peer = writer.get_extra_info("socket") + pid, uid, _ = struct.unpack("3i", peer.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) + if uid == 0 and self.server is not None: + memory = resolve_controller("memory", Path(f"/proc/{pid}/cgroup")) + if self.watch is None and memory.version == 1 and memory.memory_limit() is not None: + self.watch = OomNotifications(memory) + self.memory = memory + if self.watch is not None and memory == self.memory: + sample = self.watch.sample() + except (OSError, CgroupError): + pass # The grader can still prove untouched limits or a Formula OOM kill. + writer.write(json.dumps(sample).encode() + b"\n") + writer.close() + + def close(self): + if self.server is not None: + self.server.close() + self.server = None + if self.watch is not None: + self.watch.files.close() + self.directory.cleanup() diff --git a/src/search_swe/regrade.py b/src/search_swe/regrade.py index 4c8b580..6497d30 100644 --- a/src/search_swe/regrade.py +++ b/src/search_swe/regrade.py @@ -93,8 +93,11 @@ def walk_error(error): def measurement_valid(verifier, task_name): + cgroup_valid = runner.cgroup_measurement_valid(verifier / "cgroup-status.json") + if cgroup_valid is False: + return False evaluation = f"{task_name.rsplit('/', 1)[-1]}-eval/evaluation.json" - health = [] + health = [] if cgroup_valid is None else [cgroup_valid] for name in ("reward-details.json", "judge-status.json", evaluation): try: data = json.loads((verifier / name).read_text()) diff --git a/src/search_swe/runner.py b/src/search_swe/runner.py index 76ec9bd..55e1ed2 100644 --- a/src/search_swe/runner.py +++ b/src/search_swe/runner.py @@ -4,6 +4,7 @@ from datetime import datetime, timezone import importlib.util import json +import math import os from pathlib import Path import shutil @@ -129,6 +130,62 @@ def wait_cancelled(processes): process.wait() +def cgroup_measurement_valid(path): + """Return None for no diagnostic and False for invalid or unreadable evidence.""" + try: + status = json.loads(path.read_text()) + except FileNotFoundError: + return None + except (OSError, ValueError): + return False + return isinstance(status, dict) and status.get("measurement_valid") is True + + +def trial_outcome(output, returncode): + from harbor.models.trial.result import TrialResult + + # run_task starts one Harbor job with one trial and zero retries. + results = list(output.glob("*/*/result.json")) + # Probe the known trial directly so an unreadable verifier directory does + # not hide its marker. A verifier can also fail before result.json is saved. + markers = ([results[0].parent / "verifier/cgroup-status.json"] if len(results) == 1 + else output.glob("*/*/verifier/cgroup-status.json")) + for marker in markers: + if cgroup_measurement_valid(marker) is False: + return {"status": "invalid-measurement", "failure_kind": "cgroup_environment", + "verifier_status": str(marker)} + if returncode: + return {"status": "failed", "failure_kind": "launcher_error"} + if len(results) != 1: + return {"status": "failed", "failure_kind": "expected_one_trial_result"} + path = results[0] + failure = {"status": "failed", "trial_result": str(path)} + try: + trial = TrialResult.model_validate_json(path.read_bytes(), strict=True) + except (OSError, ValueError): + return {**failure, "failure_kind": "invalid_trial_result"} + if trial.exception_info: + failure["exception_type"] = trial.exception_info.exception_type + # Harbor can grade the work left by an interrupted agent. Setup and + # verifier exceptions invalidate the run even if a reward was written. + agent, verifier = trial.agent_execution, trial.verifier + try: + before_verification = (agent and agent.started_at and agent.finished_at and verifier + and verifier.started_at and agent.started_at <= + trial.exception_info.occurred_at <= verifier.started_at) + except TypeError: # Incompatible time zones cannot establish the phase. + before_verification = False + if not before_verification: + return {**failure, "failure_kind": "trial_error"} + rewards = trial.verifier_result.rewards if trial.verifier_result else None + if not (trial.finished_at and trial.verifier and trial.verifier.started_at and trial.verifier.finished_at): + return {**failure, "failure_kind": "incomplete_verification"} + if not (rewards and "reward" in rewards and 0 <= rewards["reward"] <= 1 and + all(isinstance(value, int) or math.isfinite(value) for value in rewards.values())): + return {**failure, "failure_kind": "invalid_reward"} + return {"status": "completed", "trial_result": str(path)} + + def run_batch(root, tasks, setting, env, pool, concurrency, output=None, offline=False, model_config=None): batch_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8] output = Path(output).expanduser().resolve() if output else root / "jobs/evaluations" / batch_id @@ -242,7 +299,7 @@ def cancel(*_): active.remove(item) stream.close() row = report["tasks"][task.name] - row.update(returncode=process.returncode, status="completed" if process.returncode == 0 else "failed") + row.update(returncode=process.returncode, **trial_outcome(Path(row["output"]), process.returncode)) try: context.__exit__(None, None, None) except Exception as error: diff --git a/tasks/budgeted-bimetric-retrieval/tests/cgroup_resources.py b/tasks/budgeted-bimetric-retrieval/tests/cgroup_resources.py new file mode 100644 index 0000000..ef60a48 --- /dev/null +++ b/tasks/budgeted-bimetric-retrieval/tests/cgroup_resources.py @@ -0,0 +1,205 @@ +"""Read Linux cgroup counters for the current process and its container. + +Canonical source is environments/cgroups/cgroup_resources.py. Task image +contexts contain identical copies so they can be built independently. +""" + +from dataclasses import dataclass +from functools import cache +import json +import os +from pathlib import Path +import re + + +class CgroupError(RuntimeError): + """The environment cannot establish the task's required resource budget.""" + + +def _read(path): + try: + return path.read_text(encoding="ascii") + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + +def _integer(value, path, minimum=0): + try: + result = int(value) + if result >= minimum: + return result + except ValueError: + pass + raise CgroupError(f"Invalid cgroup value in {path}: {value!r}") + + +def _mount_path(value): + return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + + +@cache +def controller(name): + """Resolve a controller using this process's membership and visible mounts.""" + return resolve_controller(name, Path("/proc/self/cgroup")) + + +def resolve_controller(name, membership_file): + """Resolve a process's membership against mounts visible to the reader.""" + memberships = {} + for line in _read(membership_file).splitlines(): + fields = line.split(":", 2) + if len(fields) != 3: + raise CgroupError(f"Invalid cgroup membership: {line!r}") + _, controllers, location = fields + for key in controllers.split(","): + memberships[key] = Path(location) + # A hybrid host can mount v2 alongside active v1 controllers. + version = 1 if name in memberships else 2 + member = memberships.get(name if version == 1 else "") + if member is None or not member.is_absolute() or ".." in member.parts: + raise CgroupError(f"No visible cgroup membership for the {name} controller") + candidates = [] + for line in _read(Path("/proc/self/mountinfo")).splitlines(): + before, _, after = line.partition(" - ") + mount, filesystem = before.split(), after.split() + if len(mount) < 6 or len(filesystem) < 3: + continue + if version == 1: + if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + continue + elif filesystem[0] != "cgroup2": + continue + root, target = map(_mount_path, mount[3:5]) + if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): + candidates.append((len(root.parts), target / member.relative_to(root))) + if not candidates: + raise CgroupError(f"No cgroup v{version} mount covers {name} membership {member}") + _, directory = max(candidates) + return Controller(version, directory) + + +@dataclass(frozen=True) +class Controller: + version: int + path: Path + + def _text(self, name): + return _read(self.path / name) + + def _number(self, name, minimum=0): + return _integer(self._text(name).strip(), self.path / name, minimum) + + def cpu_quota(self): + """Return (quota in microseconds or None for unlimited, period in microseconds).""" + if self.version == 1: + quota = self._number("cpu.cfs_quota_us", -1) + period = self._number("cpu.cfs_period_us", 1) + if quota == 0: + raise CgroupError(f"Invalid zero CPU quota in {self.path / 'cpu.cfs_quota_us'}") + return (None if quota == -1 else quota), period + path = self.path / "cpu.max" + fields = _read(path).split() + if len(fields) != 2: + raise CgroupError(f"Expected quota and period in {path}") + quota, period = fields + return (None if quota == "max" else _integer(quota, path, 1)), _integer(period, path, 1) + + def memory_limit(self): + if self.version == 2: + value = self._text("memory.max").strip() + return None if value == "max" else _integer(value, self.path / "memory.max", 1) + # memory.stat includes the tightest ancestor even in a private cgroup namespace. + path = self.path / "memory.stat" + bounds = {} + for line in _read(path).splitlines(): + fields = line.split() + if fields and fields[0] in ("hierarchical_memory_limit", "hierarchical_memsw_limit"): + if len(fields) != 2 or fields[0] in bounds: + raise CgroupError(f"Malformed {fields[0]} in {path}") + bounds[fields[0]] = _integer(fields[1], path) + if "hierarchical_memory_limit" not in bounds: + raise CgroupError(f"Missing hierarchical_memory_limit in {path}") + if "hierarchical_memsw_limit" not in bounds and (self.path / "memory.memsw.limit_in_bytes").exists(): + raise CgroupError(f"Missing hierarchical_memsw_limit in {path}") + limit = min(self._number("memory.limit_in_bytes"), *bounds.values()) + # v1 uses page-rounded LONG_MAX to represent an unbounded limit. + page_size = os.sysconf("SC_PAGE_SIZE") + unlimited = (2**63 - 1) // page_size * page_size + return None if limit >= unlimited else limit + + def swap_limit(self): + """Return a proven swap-only byte limit, or None when v2 allows 'max'.""" + if self.version == 2: + name = "memory.swap.max" + value = self._text(name).strip() + return None if value == "max" else _integer(value, self.path / name) + # The v1 combined RAM+swap limit does not prove zero swapped pages. + # Linux cannot swap when the host has no active swap devices or files. + swaps = _read(Path("/proc/swaps")).splitlines() + if swaps and swaps[0].split() == ["Filename", "Type", "Size", "Used", "Priority"] and not any( + line.strip() for line in swaps[1:]): + return 0 + raise CgroupError("cgroup v1 cannot verify zero swap with active or unreadable host swap") + + def memory_charge(self): + name = "memory.usage_in_bytes" if self.version == 1 else "memory.current" + return self._number(name) + + def memory_peak(self): + """Peak is a report-only kernel counter.""" + name = "memory.max_usage_in_bytes" if self.version == 1 else "memory.peak" + try: + return self._number(name) + except CgroupError: + return None + + def cpu_stat(self): + return self._text("cpu.stat") + + @cache + def _require_v1_leaf(self): + # A v1 counter can be local. A leaf has the same process scope as + # the hierarchical v2 counter, including forked submission processes. + try: + if any(path.is_dir() for path in self.path.iterdir()): + raise CgroupError(f"Cannot establish container-wide v1 OOM counts in {self.path} with child cgroups") + except OSError as error: + raise CgroupError(f"Cannot inspect the v1 memory cgroup {self.path}: {error}") from error + + def v1_limit_hits(self): + # v1 may reject a combined memory+swap charge before charging RAM. + # Kernels without swap accounting omit the memsw counter entirely. + try: + combined = (self.path / "memory.memsw.failcnt").read_text(encoding="ascii") + except FileNotFoundError: + combined = None + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read memory.memsw.failcnt: {error}") from error + return [self._number("memory.failcnt"), + None if combined is None else _integer(combined.strip(), self.path / "memory.memsw.failcnt")] + + def memory_events(self, *required): + name = "memory.oom_control" if self.version == 1 else "memory.events" + if self.version == 1: + if "oom" in required: + raise CgroupError(f"{self.path / name} does not expose a cumulative oom event count") + self._require_v1_leaf() + raw = self._text(name) + counters = {} + for line in raw.splitlines(): + fields = line.split() + if len(fields) != 2 or fields[0] in counters: + raise CgroupError(f"Malformed cgroup events in {self.path / name}") + key, value = fields + counters[key] = _integer(value, self.path / name) + for key in required: + if key not in counters: + raise CgroupError(f"Missing {key} counter in {self.path / name}") + return raw, counters + + +def write_resource_error(directory, error): + """Record why a verifier could not measure a resource-constrained task.""" + path = Path(directory) / "cgroup-status.json" + path.write_text(json.dumps({"measurement_valid": False, "failure_kind": "cgroup_environment", + "error": str(error)}) + "\n") diff --git a/tasks/budgeted-bimetric-retrieval/tests/docker-compose.yaml b/tasks/budgeted-bimetric-retrieval/tests/docker-compose.yaml index 536c895..1f9df8f 100644 --- a/tasks/budgeted-bimetric-retrieval/tests/docker-compose.yaml +++ b/tasks/budgeted-bimetric-retrieval/tests/docker-compose.yaml @@ -1,3 +1,6 @@ +x-search-swe: + v1_oom_notifications: true + services: main: cpuset: ${BIMETRIC_CPUSET:-0-15} diff --git a/tasks/budgeted-bimetric-retrieval/tests/grader.py b/tasks/budgeted-bimetric-retrieval/tests/grader.py index 3f49dfd..667959f 100644 --- a/tasks/budgeted-bimetric-retrieval/tests/grader.py +++ b/tasks/budgeted-bimetric-retrieval/tests/grader.py @@ -11,6 +11,8 @@ import time import traceback import numpy as np +from cgroup_resources import CgroupError, controller, write_resource_error +import v1_oom from metrics import ndcg10, reward, valid_ranking from oracle import Oracle, QueryFailure from runtime import Worker, Violation, signal_members, writable_size, asset_size @@ -63,6 +65,17 @@ def verify_storage(fixed): return runtime +def monitor_memory(memory): + if memory.memory_limit() != 32 << 30: + raise CgroupError('Verifier requires a 32 GiB aggregate memory cgroup') + if memory.version == 1: + memory.swap_limit() + initial = v1_oom.snapshot(memory) + return lambda: v1_oom.violation(v1_oom.snapshot(memory), initial, include_kills=False) + initial = memory.memory_events('oom')[1]['oom'] + return lambda: memory.memory_events('oom')[1]['oom'] != initial + + def main(): verification_started = time.monotonic() LOGS.mkdir(parents=True, exist_ok=True) @@ -100,13 +113,9 @@ def main(): query_vectors = np.memmap(PRIVATE/'queries.f32', dtype=' /logs/verifier/reward.txt printf '{"reward":0}\n' > /logs/verifier/reward.json -rm -f /logs/verifier/reward-details.json /logs/verifier/audit.json /logs/verifier/audit-status.json /logs/verifier/evaluation.json +rm -f /logs/verifier/cgroup-status.json /logs/verifier/reward-details.json /logs/verifier/audit.json /logs/verifier/audit-status.json /logs/verifier/evaluation.json /opt/conda/bin/python /tests/integrity_gate.py prepare # Deny every general-purpose writable path; indexes/caches live on the quota volume. chmod 755 /tmp /var/tmp /dev/shm /run/lock diff --git a/tasks/budgeted-bimetric-retrieval/tests/v1_oom.py b/tasks/budgeted-bimetric-retrieval/tests/v1_oom.py new file mode 100644 index 0000000..da33623 --- /dev/null +++ b/tasks/budgeted-bimetric-retrieval/tests/v1_oom.py @@ -0,0 +1,91 @@ +"""Container-side OOM evidence shared by Formula and Bimetric on cgroup v1.""" + +import json +import socket + +if __package__: + from .cgroup_resources import CgroupError +else: # Independent task image. + from cgroup_resources import CgroupError + + +SOCKET = "/run/search-swe-memory/oom.sock" + + +def snapshot(memory): + """Sample the limit counters first so the notification check covers that interval.""" + hits = memory.v1_limit_hits() + _, counters = memory.memory_events("oom_kill") + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(5) + client.connect(SOCKET) + with client.makefile("rb") as stream: + watch = json.loads(stream.readline(4096)) + except (FileNotFoundError, PermissionError, ConnectionError, TimeoutError): + watch = None # Raw cgroup counters can still establish some outcomes. + except (OSError, ValueError) as error: + raise CgroupError(f"Cannot read v1 OOM observer: {error}") from error + if watch is not None: + try: + info = memory.path.stat() + except OSError as error: + raise CgroupError(f"Cannot identify the task memory cgroup: {error}") from error + if not isinstance(watch, dict) or watch.get("cgroup") != [info.st_dev, info.st_ino]: + raise CgroupError("Host OOM observer and task see different memory cgroups") + return {"limit_hits": hits, "oom_kill": counters["oom_kill"], "observer": watch} + + +def _counter(value): + return type(value) is int and value >= 0 + + +def _limit_hits(value): + return (isinstance(value, list) and len(value) == 2 and _counter(value[0]) + and (value[1] is None or _counter(value[1]))) + + +def _validate(sample): + if (not isinstance(sample, dict) or not _limit_hits(sample.get("limit_hits")) + or not _counter(sample.get("oom_kill")) or "observer" not in sample): + raise CgroupError("Invalid v1 memory counters") + watch = sample["observer"] + if watch is not None and (not isinstance(watch, dict) or not isinstance(watch.get("id"), str) + or not watch["id"] or not _counter(watch.get("events")) or not _counter(watch.get("parent_events")) + or not _limit_hits(watch.get("initial_hits"))): + raise CgroupError("Invalid v1 OOM notification evidence") + + +def violation(sample, baseline=None, *, include_kills): + """Formula checks the container lifetime. Bimetric supplies a verifier baseline.""" + _validate(sample) + hits = sample["limit_hits"] + old_hits = [0, None if hits[1] is None else 0] + old_kills = 0 + old_watch = None + if baseline is not None: + _validate(baseline) + old_hits, old_kills, old_watch = baseline["limit_hits"], baseline["oom_kill"], baseline["observer"] + if (sample["oom_kill"] < old_kills + or any((a is None) != (b is None) or (a is not None and a < b) for a, b in zip(hits, old_hits))): + raise CgroupError("Reset v1 memory counters") + if include_kills and sample["oom_kill"] > old_kills: + return True + watch = sample["observer"] + if watch and (baseline is None or (old_watch and old_watch["id"] == watch["id"])): + start = old_watch["events"] if old_watch else 0 + if watch["events"] < start or (old_watch and watch["parent_events"] < old_watch["parent_events"]): + raise CgroupError("Reset v1 OOM notifications") + # Ancestor OOMs notify descendants too. A plain eventfd cannot + # attribute a notification if an ancestor was also under pressure. + if watch["parent_events"] == 0: + if watch["events"] > start: + return True + if old_watch is not None or not any(watch["initial_hits"]): + return False + # With stable caps, local memcg OOM first requires a failed RAM or memsw + # charge. Successful reclaim also increments these counters. + if hits == old_hits: + return False + raise CgroupError("The v1 memory limit was reached, but local OOM could not be distinguished " + "from reclaim or ancestor pressure in this measurement") diff --git a/tasks/budgeted-candidate-reranking/environment/docs/environment.md b/tasks/budgeted-candidate-reranking/environment/docs/environment.md index 26f3528..bf1de15 100644 --- a/tasks/budgeted-candidate-reranking/environment/docs/environment.md +++ b/tasks/budgeted-candidate-reranking/environment/docs/environment.md @@ -12,4 +12,4 @@ Read-only inputs are under `/task/data`, `/task/models/Qwen3-1.7B`, `/task/runti The container permits 256 tasks, including the trusted runtime and strategy descendants. OpenMP, OpenBLAS, MKL and NumExpr default to 16 threads per pool; the model thread count follows `deployment.json`. -Linux Landlock ABI 3 or newer, libseccomp and cgroup v2 are required. Use `/opt/conda/bin/python /task/runtime/evaluate.py --submission /app/submission --output /app/dev-report` for the public self-check. The instruction defines the interface, query budgets and permitted deployment settings. +Linux Landlock ABI 3 or newer, libseccomp and a cgroup that exposes aggregate memory and OOM kills are required. Use `/opt/conda/bin/python /task/runtime/evaluate.py --submission /app/submission --output /app/dev-report` for the public self-check. The instruction defines the interface, query budgets and permitted deployment settings. diff --git a/tasks/budgeted-candidate-reranking/environment/runtime/cgroup_resources.py b/tasks/budgeted-candidate-reranking/environment/runtime/cgroup_resources.py new file mode 100644 index 0000000..ef60a48 --- /dev/null +++ b/tasks/budgeted-candidate-reranking/environment/runtime/cgroup_resources.py @@ -0,0 +1,205 @@ +"""Read Linux cgroup counters for the current process and its container. + +Canonical source is environments/cgroups/cgroup_resources.py. Task image +contexts contain identical copies so they can be built independently. +""" + +from dataclasses import dataclass +from functools import cache +import json +import os +from pathlib import Path +import re + + +class CgroupError(RuntimeError): + """The environment cannot establish the task's required resource budget.""" + + +def _read(path): + try: + return path.read_text(encoding="ascii") + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + +def _integer(value, path, minimum=0): + try: + result = int(value) + if result >= minimum: + return result + except ValueError: + pass + raise CgroupError(f"Invalid cgroup value in {path}: {value!r}") + + +def _mount_path(value): + return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + + +@cache +def controller(name): + """Resolve a controller using this process's membership and visible mounts.""" + return resolve_controller(name, Path("/proc/self/cgroup")) + + +def resolve_controller(name, membership_file): + """Resolve a process's membership against mounts visible to the reader.""" + memberships = {} + for line in _read(membership_file).splitlines(): + fields = line.split(":", 2) + if len(fields) != 3: + raise CgroupError(f"Invalid cgroup membership: {line!r}") + _, controllers, location = fields + for key in controllers.split(","): + memberships[key] = Path(location) + # A hybrid host can mount v2 alongside active v1 controllers. + version = 1 if name in memberships else 2 + member = memberships.get(name if version == 1 else "") + if member is None or not member.is_absolute() or ".." in member.parts: + raise CgroupError(f"No visible cgroup membership for the {name} controller") + candidates = [] + for line in _read(Path("/proc/self/mountinfo")).splitlines(): + before, _, after = line.partition(" - ") + mount, filesystem = before.split(), after.split() + if len(mount) < 6 or len(filesystem) < 3: + continue + if version == 1: + if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + continue + elif filesystem[0] != "cgroup2": + continue + root, target = map(_mount_path, mount[3:5]) + if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): + candidates.append((len(root.parts), target / member.relative_to(root))) + if not candidates: + raise CgroupError(f"No cgroup v{version} mount covers {name} membership {member}") + _, directory = max(candidates) + return Controller(version, directory) + + +@dataclass(frozen=True) +class Controller: + version: int + path: Path + + def _text(self, name): + return _read(self.path / name) + + def _number(self, name, minimum=0): + return _integer(self._text(name).strip(), self.path / name, minimum) + + def cpu_quota(self): + """Return (quota in microseconds or None for unlimited, period in microseconds).""" + if self.version == 1: + quota = self._number("cpu.cfs_quota_us", -1) + period = self._number("cpu.cfs_period_us", 1) + if quota == 0: + raise CgroupError(f"Invalid zero CPU quota in {self.path / 'cpu.cfs_quota_us'}") + return (None if quota == -1 else quota), period + path = self.path / "cpu.max" + fields = _read(path).split() + if len(fields) != 2: + raise CgroupError(f"Expected quota and period in {path}") + quota, period = fields + return (None if quota == "max" else _integer(quota, path, 1)), _integer(period, path, 1) + + def memory_limit(self): + if self.version == 2: + value = self._text("memory.max").strip() + return None if value == "max" else _integer(value, self.path / "memory.max", 1) + # memory.stat includes the tightest ancestor even in a private cgroup namespace. + path = self.path / "memory.stat" + bounds = {} + for line in _read(path).splitlines(): + fields = line.split() + if fields and fields[0] in ("hierarchical_memory_limit", "hierarchical_memsw_limit"): + if len(fields) != 2 or fields[0] in bounds: + raise CgroupError(f"Malformed {fields[0]} in {path}") + bounds[fields[0]] = _integer(fields[1], path) + if "hierarchical_memory_limit" not in bounds: + raise CgroupError(f"Missing hierarchical_memory_limit in {path}") + if "hierarchical_memsw_limit" not in bounds and (self.path / "memory.memsw.limit_in_bytes").exists(): + raise CgroupError(f"Missing hierarchical_memsw_limit in {path}") + limit = min(self._number("memory.limit_in_bytes"), *bounds.values()) + # v1 uses page-rounded LONG_MAX to represent an unbounded limit. + page_size = os.sysconf("SC_PAGE_SIZE") + unlimited = (2**63 - 1) // page_size * page_size + return None if limit >= unlimited else limit + + def swap_limit(self): + """Return a proven swap-only byte limit, or None when v2 allows 'max'.""" + if self.version == 2: + name = "memory.swap.max" + value = self._text(name).strip() + return None if value == "max" else _integer(value, self.path / name) + # The v1 combined RAM+swap limit does not prove zero swapped pages. + # Linux cannot swap when the host has no active swap devices or files. + swaps = _read(Path("/proc/swaps")).splitlines() + if swaps and swaps[0].split() == ["Filename", "Type", "Size", "Used", "Priority"] and not any( + line.strip() for line in swaps[1:]): + return 0 + raise CgroupError("cgroup v1 cannot verify zero swap with active or unreadable host swap") + + def memory_charge(self): + name = "memory.usage_in_bytes" if self.version == 1 else "memory.current" + return self._number(name) + + def memory_peak(self): + """Peak is a report-only kernel counter.""" + name = "memory.max_usage_in_bytes" if self.version == 1 else "memory.peak" + try: + return self._number(name) + except CgroupError: + return None + + def cpu_stat(self): + return self._text("cpu.stat") + + @cache + def _require_v1_leaf(self): + # A v1 counter can be local. A leaf has the same process scope as + # the hierarchical v2 counter, including forked submission processes. + try: + if any(path.is_dir() for path in self.path.iterdir()): + raise CgroupError(f"Cannot establish container-wide v1 OOM counts in {self.path} with child cgroups") + except OSError as error: + raise CgroupError(f"Cannot inspect the v1 memory cgroup {self.path}: {error}") from error + + def v1_limit_hits(self): + # v1 may reject a combined memory+swap charge before charging RAM. + # Kernels without swap accounting omit the memsw counter entirely. + try: + combined = (self.path / "memory.memsw.failcnt").read_text(encoding="ascii") + except FileNotFoundError: + combined = None + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read memory.memsw.failcnt: {error}") from error + return [self._number("memory.failcnt"), + None if combined is None else _integer(combined.strip(), self.path / "memory.memsw.failcnt")] + + def memory_events(self, *required): + name = "memory.oom_control" if self.version == 1 else "memory.events" + if self.version == 1: + if "oom" in required: + raise CgroupError(f"{self.path / name} does not expose a cumulative oom event count") + self._require_v1_leaf() + raw = self._text(name) + counters = {} + for line in raw.splitlines(): + fields = line.split() + if len(fields) != 2 or fields[0] in counters: + raise CgroupError(f"Malformed cgroup events in {self.path / name}") + key, value = fields + counters[key] = _integer(value, self.path / name) + for key in required: + if key not in counters: + raise CgroupError(f"Missing {key} counter in {self.path / name}") + return raw, counters + + +def write_resource_error(directory, error): + """Record why a verifier could not measure a resource-constrained task.""" + path = Path(directory) / "cgroup-status.json" + path.write_text(json.dumps({"measurement_valid": False, "failure_kind": "cgroup_environment", + "error": str(error)}) + "\n") diff --git a/tasks/budgeted-candidate-reranking/environment/runtime/evaluate.py b/tasks/budgeted-candidate-reranking/environment/runtime/evaluate.py index c5036dd..8d0a2b3 100644 --- a/tasks/budgeted-candidate-reranking/environment/runtime/evaluate.py +++ b/tasks/budgeted-candidate-reranking/environment/runtime/evaluate.py @@ -12,6 +12,7 @@ import sys import time +from cgroup_resources import CgroupError, controller, write_resource_error import isolation from metrics import ndcg, validate_result from model import BUDGET, MODEL, deployment_config, verify_model @@ -132,15 +133,14 @@ def __init__(self, submission, model_root=MODEL, fixed_inputs=()): seen.add(identity) self.fixed_bytes += s.st_size self.last_disk = 0 - cgroup = Path('/sys/fs/cgroup') - maximum = cgroup.joinpath('memory.max').read_text().strip() - if maximum == 'max' or int(maximum) > 32 << 30: - raise RuntimeError('A <=32 GiB aggregate container memory cgroup is required') + self.memory = controller('memory') + maximum = self.memory.memory_limit() + if maximum is None or maximum > 32 << 30: + raise CgroupError('A <=32 GiB aggregate container memory cgroup is required') self.initial_oom = self.oom_count() def oom_count(self): - events = dict(line.split() for line in Path('/sys/fs/cgroup/memory.events').read_text().splitlines()) - return int(events.get('oom_kill', 0)) + return self.memory.memory_events('oom_kill')[1]['oom_kill'] def check(self, disk=False): used = self.nvml.nvmlDeviceGetMemoryInfo(self.gpu).used @@ -159,7 +159,7 @@ def check(self, disk=False): def report(self): return {'peak_gpu_bytes': self.peak_gpu, 'peak_task_storage_bytes': self.peak_disk, - 'peak_cgroup_memory_bytes': int(Path('/sys/fs/cgroup/memory.peak').read_text()), + 'peak_cgroup_memory_bytes': self.memory.memory_peak(), 'gpu_name': self.nvml.nvmlDeviceGetName(self.gpu), 'gpu_uuid': self.nvml.nvmlDeviceGetUUID(self.gpu)} @@ -291,7 +291,7 @@ def run_query(query, submission, backend, output, resources): except Exception as error: report.update(status='failed', ids=[], error=str(error)) report['local_seconds'] = time.monotonic()-started-excluded - if str(error).startswith('Global resource violation:'): + if isinstance(error, CgroupError) or str(error).startswith('Global resource violation:'): raise finally: isolation.cleanup() @@ -400,7 +400,11 @@ def main(): parser.add_argument('--qrels', default='/task/data/public-qrels.json') parser.add_argument('--output', required=True) args = parser.parse_args() - report = evaluate(args.submission, args.queries, args.qrels, args.output) + try: + report = evaluate(args.submission, args.queries, args.qrels, args.output) + except CgroupError as error: + write_resource_error(args.output, error) + raise dump(Path(args.output)/'report.json', report) diff --git a/tasks/budgeted-candidate-reranking/tests/grader.py b/tasks/budgeted-candidate-reranking/tests/grader.py index 1e4b7db..4c5bf9c 100644 --- a/tasks/budgeted-candidate-reranking/tests/grader.py +++ b/tasks/budgeted-candidate-reranking/tests/grader.py @@ -8,6 +8,7 @@ import traceback sys.path.insert(0, '/tests/runtime') +from cgroup_resources import CgroupError, write_resource_error from evaluate import evaluate, dump from isolation import cleanup from submission_transfer import ARCHIVE, restore @@ -53,6 +54,8 @@ def main(): status='complete', failed_queries=sum(r['status'] != 'ok' for r in result['queries'])) status = 0 except Exception as error: + if isinstance(error, CgroupError): + write_resource_error(OUTPUT, error) report['error'] = str(error) (OUTPUT/'exception.txt').write_text(traceback.format_exc()) finally: diff --git a/tasks/budgeted-candidate-reranking/tests/runtime/cgroup_resources.py b/tasks/budgeted-candidate-reranking/tests/runtime/cgroup_resources.py new file mode 100644 index 0000000..ef60a48 --- /dev/null +++ b/tasks/budgeted-candidate-reranking/tests/runtime/cgroup_resources.py @@ -0,0 +1,205 @@ +"""Read Linux cgroup counters for the current process and its container. + +Canonical source is environments/cgroups/cgroup_resources.py. Task image +contexts contain identical copies so they can be built independently. +""" + +from dataclasses import dataclass +from functools import cache +import json +import os +from pathlib import Path +import re + + +class CgroupError(RuntimeError): + """The environment cannot establish the task's required resource budget.""" + + +def _read(path): + try: + return path.read_text(encoding="ascii") + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + +def _integer(value, path, minimum=0): + try: + result = int(value) + if result >= minimum: + return result + except ValueError: + pass + raise CgroupError(f"Invalid cgroup value in {path}: {value!r}") + + +def _mount_path(value): + return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + + +@cache +def controller(name): + """Resolve a controller using this process's membership and visible mounts.""" + return resolve_controller(name, Path("/proc/self/cgroup")) + + +def resolve_controller(name, membership_file): + """Resolve a process's membership against mounts visible to the reader.""" + memberships = {} + for line in _read(membership_file).splitlines(): + fields = line.split(":", 2) + if len(fields) != 3: + raise CgroupError(f"Invalid cgroup membership: {line!r}") + _, controllers, location = fields + for key in controllers.split(","): + memberships[key] = Path(location) + # A hybrid host can mount v2 alongside active v1 controllers. + version = 1 if name in memberships else 2 + member = memberships.get(name if version == 1 else "") + if member is None or not member.is_absolute() or ".." in member.parts: + raise CgroupError(f"No visible cgroup membership for the {name} controller") + candidates = [] + for line in _read(Path("/proc/self/mountinfo")).splitlines(): + before, _, after = line.partition(" - ") + mount, filesystem = before.split(), after.split() + if len(mount) < 6 or len(filesystem) < 3: + continue + if version == 1: + if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + continue + elif filesystem[0] != "cgroup2": + continue + root, target = map(_mount_path, mount[3:5]) + if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): + candidates.append((len(root.parts), target / member.relative_to(root))) + if not candidates: + raise CgroupError(f"No cgroup v{version} mount covers {name} membership {member}") + _, directory = max(candidates) + return Controller(version, directory) + + +@dataclass(frozen=True) +class Controller: + version: int + path: Path + + def _text(self, name): + return _read(self.path / name) + + def _number(self, name, minimum=0): + return _integer(self._text(name).strip(), self.path / name, minimum) + + def cpu_quota(self): + """Return (quota in microseconds or None for unlimited, period in microseconds).""" + if self.version == 1: + quota = self._number("cpu.cfs_quota_us", -1) + period = self._number("cpu.cfs_period_us", 1) + if quota == 0: + raise CgroupError(f"Invalid zero CPU quota in {self.path / 'cpu.cfs_quota_us'}") + return (None if quota == -1 else quota), period + path = self.path / "cpu.max" + fields = _read(path).split() + if len(fields) != 2: + raise CgroupError(f"Expected quota and period in {path}") + quota, period = fields + return (None if quota == "max" else _integer(quota, path, 1)), _integer(period, path, 1) + + def memory_limit(self): + if self.version == 2: + value = self._text("memory.max").strip() + return None if value == "max" else _integer(value, self.path / "memory.max", 1) + # memory.stat includes the tightest ancestor even in a private cgroup namespace. + path = self.path / "memory.stat" + bounds = {} + for line in _read(path).splitlines(): + fields = line.split() + if fields and fields[0] in ("hierarchical_memory_limit", "hierarchical_memsw_limit"): + if len(fields) != 2 or fields[0] in bounds: + raise CgroupError(f"Malformed {fields[0]} in {path}") + bounds[fields[0]] = _integer(fields[1], path) + if "hierarchical_memory_limit" not in bounds: + raise CgroupError(f"Missing hierarchical_memory_limit in {path}") + if "hierarchical_memsw_limit" not in bounds and (self.path / "memory.memsw.limit_in_bytes").exists(): + raise CgroupError(f"Missing hierarchical_memsw_limit in {path}") + limit = min(self._number("memory.limit_in_bytes"), *bounds.values()) + # v1 uses page-rounded LONG_MAX to represent an unbounded limit. + page_size = os.sysconf("SC_PAGE_SIZE") + unlimited = (2**63 - 1) // page_size * page_size + return None if limit >= unlimited else limit + + def swap_limit(self): + """Return a proven swap-only byte limit, or None when v2 allows 'max'.""" + if self.version == 2: + name = "memory.swap.max" + value = self._text(name).strip() + return None if value == "max" else _integer(value, self.path / name) + # The v1 combined RAM+swap limit does not prove zero swapped pages. + # Linux cannot swap when the host has no active swap devices or files. + swaps = _read(Path("/proc/swaps")).splitlines() + if swaps and swaps[0].split() == ["Filename", "Type", "Size", "Used", "Priority"] and not any( + line.strip() for line in swaps[1:]): + return 0 + raise CgroupError("cgroup v1 cannot verify zero swap with active or unreadable host swap") + + def memory_charge(self): + name = "memory.usage_in_bytes" if self.version == 1 else "memory.current" + return self._number(name) + + def memory_peak(self): + """Peak is a report-only kernel counter.""" + name = "memory.max_usage_in_bytes" if self.version == 1 else "memory.peak" + try: + return self._number(name) + except CgroupError: + return None + + def cpu_stat(self): + return self._text("cpu.stat") + + @cache + def _require_v1_leaf(self): + # A v1 counter can be local. A leaf has the same process scope as + # the hierarchical v2 counter, including forked submission processes. + try: + if any(path.is_dir() for path in self.path.iterdir()): + raise CgroupError(f"Cannot establish container-wide v1 OOM counts in {self.path} with child cgroups") + except OSError as error: + raise CgroupError(f"Cannot inspect the v1 memory cgroup {self.path}: {error}") from error + + def v1_limit_hits(self): + # v1 may reject a combined memory+swap charge before charging RAM. + # Kernels without swap accounting omit the memsw counter entirely. + try: + combined = (self.path / "memory.memsw.failcnt").read_text(encoding="ascii") + except FileNotFoundError: + combined = None + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read memory.memsw.failcnt: {error}") from error + return [self._number("memory.failcnt"), + None if combined is None else _integer(combined.strip(), self.path / "memory.memsw.failcnt")] + + def memory_events(self, *required): + name = "memory.oom_control" if self.version == 1 else "memory.events" + if self.version == 1: + if "oom" in required: + raise CgroupError(f"{self.path / name} does not expose a cumulative oom event count") + self._require_v1_leaf() + raw = self._text(name) + counters = {} + for line in raw.splitlines(): + fields = line.split() + if len(fields) != 2 or fields[0] in counters: + raise CgroupError(f"Malformed cgroup events in {self.path / name}") + key, value = fields + counters[key] = _integer(value, self.path / name) + for key in required: + if key not in counters: + raise CgroupError(f"Missing {key} counter in {self.path / name}") + return raw, counters + + +def write_resource_error(directory, error): + """Record why a verifier could not measure a resource-constrained task.""" + path = Path(directory) / "cgroup-status.json" + path.write_text(json.dumps({"measurement_valid": False, "failure_kind": "cgroup_environment", + "error": str(error)}) + "\n") diff --git a/tasks/budgeted-candidate-reranking/tests/runtime/evaluate.py b/tasks/budgeted-candidate-reranking/tests/runtime/evaluate.py index c5036dd..8d0a2b3 100644 --- a/tasks/budgeted-candidate-reranking/tests/runtime/evaluate.py +++ b/tasks/budgeted-candidate-reranking/tests/runtime/evaluate.py @@ -12,6 +12,7 @@ import sys import time +from cgroup_resources import CgroupError, controller, write_resource_error import isolation from metrics import ndcg, validate_result from model import BUDGET, MODEL, deployment_config, verify_model @@ -132,15 +133,14 @@ def __init__(self, submission, model_root=MODEL, fixed_inputs=()): seen.add(identity) self.fixed_bytes += s.st_size self.last_disk = 0 - cgroup = Path('/sys/fs/cgroup') - maximum = cgroup.joinpath('memory.max').read_text().strip() - if maximum == 'max' or int(maximum) > 32 << 30: - raise RuntimeError('A <=32 GiB aggregate container memory cgroup is required') + self.memory = controller('memory') + maximum = self.memory.memory_limit() + if maximum is None or maximum > 32 << 30: + raise CgroupError('A <=32 GiB aggregate container memory cgroup is required') self.initial_oom = self.oom_count() def oom_count(self): - events = dict(line.split() for line in Path('/sys/fs/cgroup/memory.events').read_text().splitlines()) - return int(events.get('oom_kill', 0)) + return self.memory.memory_events('oom_kill')[1]['oom_kill'] def check(self, disk=False): used = self.nvml.nvmlDeviceGetMemoryInfo(self.gpu).used @@ -159,7 +159,7 @@ def check(self, disk=False): def report(self): return {'peak_gpu_bytes': self.peak_gpu, 'peak_task_storage_bytes': self.peak_disk, - 'peak_cgroup_memory_bytes': int(Path('/sys/fs/cgroup/memory.peak').read_text()), + 'peak_cgroup_memory_bytes': self.memory.memory_peak(), 'gpu_name': self.nvml.nvmlDeviceGetName(self.gpu), 'gpu_uuid': self.nvml.nvmlDeviceGetUUID(self.gpu)} @@ -291,7 +291,7 @@ def run_query(query, submission, backend, output, resources): except Exception as error: report.update(status='failed', ids=[], error=str(error)) report['local_seconds'] = time.monotonic()-started-excluded - if str(error).startswith('Global resource violation:'): + if isinstance(error, CgroupError) or str(error).startswith('Global resource violation:'): raise finally: isolation.cleanup() @@ -400,7 +400,11 @@ def main(): parser.add_argument('--qrels', default='/task/data/public-qrels.json') parser.add_argument('--output', required=True) args = parser.parse_args() - report = evaluate(args.submission, args.queries, args.qrels, args.output) + try: + report = evaluate(args.submission, args.queries, args.qrels, args.output) + except CgroupError as error: + write_resource_error(args.output, error) + raise dump(Path(args.output)/'report.json', report) diff --git a/tasks/budgeted-candidate-reranking/tests/test.sh b/tasks/budgeted-candidate-reranking/tests/test.sh index 89fba7e..271e736 100755 --- a/tasks/budgeted-candidate-reranking/tests/test.sh +++ b/tasks/budgeted-candidate-reranking/tests/test.sh @@ -3,7 +3,7 @@ set -eu install -d -o root -g root -m 0700 /logs/verifier printf '0\n' > /logs/verifier/reward.txt printf '{"reward":0}\n' > /logs/verifier/reward.json -rm -f /logs/verifier/reward-details.json /logs/verifier/audit.json /logs/verifier/audit-status.json /logs/verifier/report.json +rm -f /logs/verifier/cgroup-status.json /logs/verifier/reward-details.json /logs/verifier/audit.json /logs/verifier/audit-status.json /logs/verifier/report.json chmod 0700 /tests /opt/conda/bin/python /tests/integrity_gate.py prepare chmod 0700 /logs/agent diff --git a/tasks/constrained-molecular-search/environment/docs/environment.md b/tasks/constrained-molecular-search/environment/docs/environment.md index 9b49984..aa62ad6 100644 --- a/tasks/constrained-molecular-search/environment/docs/environment.md +++ b/tasks/constrained-molecular-search/environment/docs/environment.md @@ -6,7 +6,7 @@ The task Python interpreter and its installed packages are available at `/opt/co The image also includes the CUDA 13.0 toolkit and runtime, Java 21 runtime, Node.js 24.16.0 with npm 11.13.0, FFmpeg, Poppler utilities, Cairo, Git, curl, `jq`, `build-essential`, `ca-certificates`, `libffi`, `libgomp`, `netbase`, `netcat`, `procps`, `tzdata`, `unzip`, and the related system runtime libraries. -The task additionally installs RDKit 2024.3.5, DuckDB 1.3.2, TensorBoard 2.20.0, LightGBM 4.6.0 and nvidia-ml-py 12.560.30. APEX and its GPU top-k extension are installed from bundled sources. One GPU is provided. Linux Landlock ABI 3 or newer, seccomp, cgroup v2 and a compatible NVIDIA runtime are required. +The task additionally installs RDKit 2024.3.5, DuckDB 1.3.2, TensorBoard 2.20.0, LightGBM 4.6.0 and nvidia-ml-py 12.560.30. APEX and its GPU top-k extension are installed from bundled sources. One GPU is provided. Linux Landlock ABI 3 or newer, seccomp, Linux cgroups and a compatible NVIDIA runtime are required. Read-only inputs are under `/task/library`, `/task/models`, `/task/train` and `/task/dev`. Public helpers are under `/task/runtime`. The `Library` helper in `library.py` provides `id_to_name`, `name_to_id`, `id_to_smiles`, `id_to_key` and `iter_smiles` for the supplied library. Property units are Da for `mol_wt`, RDKit Crippen logP for `logp`, counts for hydrogen-bond and rotatable-bond fields, and square angstroms for `tpsa`; rotatable bonds use RDKit's default strict definition. diff --git a/tasks/constrained-molecular-search/environment/runtime/baseline_model.py b/tasks/constrained-molecular-search/environment/runtime/baseline_model.py index bf5fc68..47ce787 100644 --- a/tasks/constrained-molecular-search/environment/runtime/baseline_model.py +++ b/tasks/constrained-molecular-search/environment/runtime/baseline_model.py @@ -16,6 +16,7 @@ from rdkit import RDLogger sys.path.insert(0, "/task/runtime") +from cgroup_resources import controller from library import Library from apex.data.atom_bond_features import Vocabulary from apex.data.torch_graph import PackedTorchMol, TorchMol @@ -114,14 +115,16 @@ def main(): embeddings.flush() if args.device == "cuda": torch.cuda.synchronize() + cpu, period = controller("cpu").cpu_quota() + memory = controller("memory") result = {"seconds": time.monotonic() - started, "n_products": len(dataset), "workers": args.workers, "batch_size": args.batch_size, "device": args.device, "threads": args.threads, "gpu": torch.cuda.get_device_name(0) if args.device == "cuda" else None, "peak_gpu_allocated_bytes": torch.cuda.max_memory_allocated() if args.device == "cuda" else 0, "normalized_fragments": bool(args.normalized_fragments), - "container_memory_peak_bytes": int(Path("/sys/fs/cgroup/memory.peak").read_text()), - "cpu_max": Path("/sys/fs/cgroup/cpu.max").read_text().strip(), + "container_memory_peak_bytes": memory.memory_peak(), + "cpu_max": f"{cpu if cpu is not None else 'max'} {period}", "source": "full supplied encoder+probe; public definitions only"} (output / "measurement.json").write_text(json.dumps(result, indent=2) + "\n") print(json.dumps(result), flush=True) diff --git a/tasks/constrained-molecular-search/environment/runtime/baseline_properties.py b/tasks/constrained-molecular-search/environment/runtime/baseline_properties.py index 763923a..7e84fa8 100644 --- a/tasks/constrained-molecular-search/environment/runtime/baseline_properties.py +++ b/tasks/constrained-molecular-search/environment/runtime/baseline_properties.py @@ -19,6 +19,7 @@ from rdkit import Chem, RDLogger from rdkit.Chem import Descriptors, Crippen, Lipinski, rdMolDescriptors +from cgroup_resources import controller from library import Library LIBRARY = None @@ -86,9 +87,12 @@ def main(): "seconds": time.monotonic() - started, "invalid_structures": invalid, "cpu_affinity": sorted(os.sched_getaffinity(0)), "workers": 16, "properties": ["mol_wt", "logp", "n_hbd", "n_hba", "n_rotatable_bonds", "tpsa"]} - report["container_memory_peak_bytes"] = int(Path("/sys/fs/cgroup/memory.peak").read_text()) - report["memory_max"] = Path("/sys/fs/cgroup/memory.max").read_text().strip() - report["cpu_max"] = Path("/sys/fs/cgroup/cpu.max").read_text().strip() + cpu, period = controller("cpu").cpu_quota() + memory = controller("memory") + report["container_memory_peak_bytes"] = memory.memory_peak() + maximum = memory.memory_limit() + report["memory_max"] = "max" if maximum is None else str(maximum) + report["cpu_max"] = f"{cpu if cpu is not None else 'max'} {period}" (out / "measurement.json").write_text(json.dumps(report, indent=2) + "\n") print(json.dumps(report), flush=True) diff --git a/tasks/constrained-molecular-search/environment/runtime/cgroup_resources.py b/tasks/constrained-molecular-search/environment/runtime/cgroup_resources.py new file mode 100644 index 0000000..ef60a48 --- /dev/null +++ b/tasks/constrained-molecular-search/environment/runtime/cgroup_resources.py @@ -0,0 +1,205 @@ +"""Read Linux cgroup counters for the current process and its container. + +Canonical source is environments/cgroups/cgroup_resources.py. Task image +contexts contain identical copies so they can be built independently. +""" + +from dataclasses import dataclass +from functools import cache +import json +import os +from pathlib import Path +import re + + +class CgroupError(RuntimeError): + """The environment cannot establish the task's required resource budget.""" + + +def _read(path): + try: + return path.read_text(encoding="ascii") + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + +def _integer(value, path, minimum=0): + try: + result = int(value) + if result >= minimum: + return result + except ValueError: + pass + raise CgroupError(f"Invalid cgroup value in {path}: {value!r}") + + +def _mount_path(value): + return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + + +@cache +def controller(name): + """Resolve a controller using this process's membership and visible mounts.""" + return resolve_controller(name, Path("/proc/self/cgroup")) + + +def resolve_controller(name, membership_file): + """Resolve a process's membership against mounts visible to the reader.""" + memberships = {} + for line in _read(membership_file).splitlines(): + fields = line.split(":", 2) + if len(fields) != 3: + raise CgroupError(f"Invalid cgroup membership: {line!r}") + _, controllers, location = fields + for key in controllers.split(","): + memberships[key] = Path(location) + # A hybrid host can mount v2 alongside active v1 controllers. + version = 1 if name in memberships else 2 + member = memberships.get(name if version == 1 else "") + if member is None or not member.is_absolute() or ".." in member.parts: + raise CgroupError(f"No visible cgroup membership for the {name} controller") + candidates = [] + for line in _read(Path("/proc/self/mountinfo")).splitlines(): + before, _, after = line.partition(" - ") + mount, filesystem = before.split(), after.split() + if len(mount) < 6 or len(filesystem) < 3: + continue + if version == 1: + if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + continue + elif filesystem[0] != "cgroup2": + continue + root, target = map(_mount_path, mount[3:5]) + if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): + candidates.append((len(root.parts), target / member.relative_to(root))) + if not candidates: + raise CgroupError(f"No cgroup v{version} mount covers {name} membership {member}") + _, directory = max(candidates) + return Controller(version, directory) + + +@dataclass(frozen=True) +class Controller: + version: int + path: Path + + def _text(self, name): + return _read(self.path / name) + + def _number(self, name, minimum=0): + return _integer(self._text(name).strip(), self.path / name, minimum) + + def cpu_quota(self): + """Return (quota in microseconds or None for unlimited, period in microseconds).""" + if self.version == 1: + quota = self._number("cpu.cfs_quota_us", -1) + period = self._number("cpu.cfs_period_us", 1) + if quota == 0: + raise CgroupError(f"Invalid zero CPU quota in {self.path / 'cpu.cfs_quota_us'}") + return (None if quota == -1 else quota), period + path = self.path / "cpu.max" + fields = _read(path).split() + if len(fields) != 2: + raise CgroupError(f"Expected quota and period in {path}") + quota, period = fields + return (None if quota == "max" else _integer(quota, path, 1)), _integer(period, path, 1) + + def memory_limit(self): + if self.version == 2: + value = self._text("memory.max").strip() + return None if value == "max" else _integer(value, self.path / "memory.max", 1) + # memory.stat includes the tightest ancestor even in a private cgroup namespace. + path = self.path / "memory.stat" + bounds = {} + for line in _read(path).splitlines(): + fields = line.split() + if fields and fields[0] in ("hierarchical_memory_limit", "hierarchical_memsw_limit"): + if len(fields) != 2 or fields[0] in bounds: + raise CgroupError(f"Malformed {fields[0]} in {path}") + bounds[fields[0]] = _integer(fields[1], path) + if "hierarchical_memory_limit" not in bounds: + raise CgroupError(f"Missing hierarchical_memory_limit in {path}") + if "hierarchical_memsw_limit" not in bounds and (self.path / "memory.memsw.limit_in_bytes").exists(): + raise CgroupError(f"Missing hierarchical_memsw_limit in {path}") + limit = min(self._number("memory.limit_in_bytes"), *bounds.values()) + # v1 uses page-rounded LONG_MAX to represent an unbounded limit. + page_size = os.sysconf("SC_PAGE_SIZE") + unlimited = (2**63 - 1) // page_size * page_size + return None if limit >= unlimited else limit + + def swap_limit(self): + """Return a proven swap-only byte limit, or None when v2 allows 'max'.""" + if self.version == 2: + name = "memory.swap.max" + value = self._text(name).strip() + return None if value == "max" else _integer(value, self.path / name) + # The v1 combined RAM+swap limit does not prove zero swapped pages. + # Linux cannot swap when the host has no active swap devices or files. + swaps = _read(Path("/proc/swaps")).splitlines() + if swaps and swaps[0].split() == ["Filename", "Type", "Size", "Used", "Priority"] and not any( + line.strip() for line in swaps[1:]): + return 0 + raise CgroupError("cgroup v1 cannot verify zero swap with active or unreadable host swap") + + def memory_charge(self): + name = "memory.usage_in_bytes" if self.version == 1 else "memory.current" + return self._number(name) + + def memory_peak(self): + """Peak is a report-only kernel counter.""" + name = "memory.max_usage_in_bytes" if self.version == 1 else "memory.peak" + try: + return self._number(name) + except CgroupError: + return None + + def cpu_stat(self): + return self._text("cpu.stat") + + @cache + def _require_v1_leaf(self): + # A v1 counter can be local. A leaf has the same process scope as + # the hierarchical v2 counter, including forked submission processes. + try: + if any(path.is_dir() for path in self.path.iterdir()): + raise CgroupError(f"Cannot establish container-wide v1 OOM counts in {self.path} with child cgroups") + except OSError as error: + raise CgroupError(f"Cannot inspect the v1 memory cgroup {self.path}: {error}") from error + + def v1_limit_hits(self): + # v1 may reject a combined memory+swap charge before charging RAM. + # Kernels without swap accounting omit the memsw counter entirely. + try: + combined = (self.path / "memory.memsw.failcnt").read_text(encoding="ascii") + except FileNotFoundError: + combined = None + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read memory.memsw.failcnt: {error}") from error + return [self._number("memory.failcnt"), + None if combined is None else _integer(combined.strip(), self.path / "memory.memsw.failcnt")] + + def memory_events(self, *required): + name = "memory.oom_control" if self.version == 1 else "memory.events" + if self.version == 1: + if "oom" in required: + raise CgroupError(f"{self.path / name} does not expose a cumulative oom event count") + self._require_v1_leaf() + raw = self._text(name) + counters = {} + for line in raw.splitlines(): + fields = line.split() + if len(fields) != 2 or fields[0] in counters: + raise CgroupError(f"Malformed cgroup events in {self.path / name}") + key, value = fields + counters[key] = _integer(value, self.path / name) + for key in required: + if key not in counters: + raise CgroupError(f"Missing {key} counter in {self.path / name}") + return raw, counters + + +def write_resource_error(directory, error): + """Record why a verifier could not measure a resource-constrained task.""" + path = Path(directory) / "cgroup-status.json" + path.write_text(json.dumps({"measurement_valid": False, "failure_kind": "cgroup_environment", + "error": str(error)}) + "\n") diff --git a/tasks/constrained-molecular-search/environment/runtime/evaluate.py b/tasks/constrained-molecular-search/environment/runtime/evaluate.py index 810ac6d..b718644 100644 --- a/tasks/constrained-molecular-search/environment/runtime/evaluate.py +++ b/tasks/constrained-molecular-search/environment/runtime/evaluate.py @@ -15,6 +15,7 @@ import numpy as np import pynvml +from cgroup_resources import CgroupError, controller, write_resource_error from isolation import UID, cleanup from metrics import JS, PROPERTIES, recalls, reward, summarize, validate_ids from metrics import TARGETS @@ -153,12 +154,13 @@ def __init__(self, fixed_bytes): if pynvml.nvmlDeviceGetCount() != 1: raise ResourceViolation("Exactly one mapped GPU is required") self.gpu = pynvml.nvmlDeviceGetHandleByIndex(0) - quota, period = Path("/sys/fs/cgroup/cpu.max").read_text().split() - maximum = Path("/sys/fs/cgroup/memory.max").read_text().strip() - if quota == "max" or int(quota) / int(period) > 16: - raise ResourceViolation("CPU cgroup limit must be at most 16 cores") - if maximum == "max" or int(maximum) > 32 << 30: - raise ResourceViolation("Memory cgroup limit must be at most 32 GiB") + quota, period = controller("cpu").cpu_quota() + self.memory = controller("memory") + maximum = self.memory.memory_limit() + if quota is None or quota > 16 * period: + raise CgroupError("CPU cgroup limit must be at most 16 cores") + if maximum is None or maximum > 32 << 30: + raise CgroupError("Memory cgroup limit must be at most 32 GiB") def check(self, force=False): self.peak_gpu = max(self.peak_gpu, pynvml.nvmlDeviceGetMemoryInfo(self.gpu).used) @@ -170,11 +172,10 @@ def check(self, force=False): raise ResourceViolation("64 GiB total task storage exceeded") def report(self): - path = Path("/sys/fs/cgroup/memory.peak") return {"gpu": pynvml.nvmlDeviceGetName(self.gpu), "gpu_uuid": pynvml.nvmlDeviceGetUUID(self.gpu), "peak_gpu_bytes": self.peak_gpu, "peak_task_storage_bytes": self.peak_storage, - "container_memory_peak_bytes": int(path.read_text()) if path.exists() else None} + "container_memory_peak_bytes": self.memory.memory_peak()} class Worker: @@ -387,6 +388,8 @@ def main(): report = evaluate(args.private_dir, output, args.library_dir, args.reference_dir) except Exception as error: + if isinstance(error, CgroupError): + write_resource_error(output, error) report["failure"] = type(error).__name__ + ": " + str(error) (output / "report.json").write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") return 0 if report["valid"] else 1 diff --git a/tasks/constrained-molecular-search/tests/docs/environment.md b/tasks/constrained-molecular-search/tests/docs/environment.md index a7999ba..b5ef93e 100644 --- a/tasks/constrained-molecular-search/tests/docs/environment.md +++ b/tasks/constrained-molecular-search/tests/docs/environment.md @@ -6,7 +6,7 @@ The task Python interpreter and its installed packages are available at `/opt/co The image also includes the CUDA 13.0 toolkit and runtime, Java 21 runtime, Node.js with npm, Node.js 24.16.0 with npm 11.13.0, FFmpeg, Poppler utilities, Cairo, Git, curl, `jq`, `build-essential`, `ca-certificates`, `libffi`, `libgomp`, `netbase`, `netcat`, `procps`, `tzdata`, `unzip`, and the related system runtime libraries. -The task additionally installs RDKit 2024.3.5, DuckDB 1.3.2, TensorBoard 2.20.0, LightGBM 4.6.0 and nvidia-ml-py 12.560.30. APEX and its GPU top-k extension are installed from bundled sources. One GPU is provided. Linux Landlock ABI 3 or newer, seccomp, cgroup v2 and a compatible NVIDIA runtime are required. +The task additionally installs RDKit 2024.3.5, DuckDB 1.3.2, TensorBoard 2.20.0, LightGBM 4.6.0 and nvidia-ml-py 12.560.30. APEX and its GPU top-k extension are installed from bundled sources. One GPU is provided. Linux Landlock ABI 3 or newer, seccomp, Linux cgroups and a compatible NVIDIA runtime are required. Read-only inputs are under `/task/library`, `/task/models`, `/task/train` and `/task/dev`. Public helpers are under `/task/runtime`. The `Library` helper in `library.py` provides `id_to_name`, `name_to_id`, `id_to_smiles`, `id_to_key` and `iter_smiles` for the supplied library. Property units are Da for `mol_wt`, RDKit Crippen logP for `logp`, counts for hydrogen-bond and rotatable-bond fields, and square angstroms for `tpsa`; rotatable bonds use RDKit's default strict definition. diff --git a/tasks/constrained-molecular-search/tests/runtime/baseline_model.py b/tasks/constrained-molecular-search/tests/runtime/baseline_model.py index bf5fc68..47ce787 100644 --- a/tasks/constrained-molecular-search/tests/runtime/baseline_model.py +++ b/tasks/constrained-molecular-search/tests/runtime/baseline_model.py @@ -16,6 +16,7 @@ from rdkit import RDLogger sys.path.insert(0, "/task/runtime") +from cgroup_resources import controller from library import Library from apex.data.atom_bond_features import Vocabulary from apex.data.torch_graph import PackedTorchMol, TorchMol @@ -114,14 +115,16 @@ def main(): embeddings.flush() if args.device == "cuda": torch.cuda.synchronize() + cpu, period = controller("cpu").cpu_quota() + memory = controller("memory") result = {"seconds": time.monotonic() - started, "n_products": len(dataset), "workers": args.workers, "batch_size": args.batch_size, "device": args.device, "threads": args.threads, "gpu": torch.cuda.get_device_name(0) if args.device == "cuda" else None, "peak_gpu_allocated_bytes": torch.cuda.max_memory_allocated() if args.device == "cuda" else 0, "normalized_fragments": bool(args.normalized_fragments), - "container_memory_peak_bytes": int(Path("/sys/fs/cgroup/memory.peak").read_text()), - "cpu_max": Path("/sys/fs/cgroup/cpu.max").read_text().strip(), + "container_memory_peak_bytes": memory.memory_peak(), + "cpu_max": f"{cpu if cpu is not None else 'max'} {period}", "source": "full supplied encoder+probe; public definitions only"} (output / "measurement.json").write_text(json.dumps(result, indent=2) + "\n") print(json.dumps(result), flush=True) diff --git a/tasks/constrained-molecular-search/tests/runtime/baseline_properties.py b/tasks/constrained-molecular-search/tests/runtime/baseline_properties.py index 763923a..7e84fa8 100644 --- a/tasks/constrained-molecular-search/tests/runtime/baseline_properties.py +++ b/tasks/constrained-molecular-search/tests/runtime/baseline_properties.py @@ -19,6 +19,7 @@ from rdkit import Chem, RDLogger from rdkit.Chem import Descriptors, Crippen, Lipinski, rdMolDescriptors +from cgroup_resources import controller from library import Library LIBRARY = None @@ -86,9 +87,12 @@ def main(): "seconds": time.monotonic() - started, "invalid_structures": invalid, "cpu_affinity": sorted(os.sched_getaffinity(0)), "workers": 16, "properties": ["mol_wt", "logp", "n_hbd", "n_hba", "n_rotatable_bonds", "tpsa"]} - report["container_memory_peak_bytes"] = int(Path("/sys/fs/cgroup/memory.peak").read_text()) - report["memory_max"] = Path("/sys/fs/cgroup/memory.max").read_text().strip() - report["cpu_max"] = Path("/sys/fs/cgroup/cpu.max").read_text().strip() + cpu, period = controller("cpu").cpu_quota() + memory = controller("memory") + report["container_memory_peak_bytes"] = memory.memory_peak() + maximum = memory.memory_limit() + report["memory_max"] = "max" if maximum is None else str(maximum) + report["cpu_max"] = f"{cpu if cpu is not None else 'max'} {period}" (out / "measurement.json").write_text(json.dumps(report, indent=2) + "\n") print(json.dumps(report), flush=True) diff --git a/tasks/constrained-molecular-search/tests/runtime/cgroup_resources.py b/tasks/constrained-molecular-search/tests/runtime/cgroup_resources.py new file mode 100644 index 0000000..ef60a48 --- /dev/null +++ b/tasks/constrained-molecular-search/tests/runtime/cgroup_resources.py @@ -0,0 +1,205 @@ +"""Read Linux cgroup counters for the current process and its container. + +Canonical source is environments/cgroups/cgroup_resources.py. Task image +contexts contain identical copies so they can be built independently. +""" + +from dataclasses import dataclass +from functools import cache +import json +import os +from pathlib import Path +import re + + +class CgroupError(RuntimeError): + """The environment cannot establish the task's required resource budget.""" + + +def _read(path): + try: + return path.read_text(encoding="ascii") + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + +def _integer(value, path, minimum=0): + try: + result = int(value) + if result >= minimum: + return result + except ValueError: + pass + raise CgroupError(f"Invalid cgroup value in {path}: {value!r}") + + +def _mount_path(value): + return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + + +@cache +def controller(name): + """Resolve a controller using this process's membership and visible mounts.""" + return resolve_controller(name, Path("/proc/self/cgroup")) + + +def resolve_controller(name, membership_file): + """Resolve a process's membership against mounts visible to the reader.""" + memberships = {} + for line in _read(membership_file).splitlines(): + fields = line.split(":", 2) + if len(fields) != 3: + raise CgroupError(f"Invalid cgroup membership: {line!r}") + _, controllers, location = fields + for key in controllers.split(","): + memberships[key] = Path(location) + # A hybrid host can mount v2 alongside active v1 controllers. + version = 1 if name in memberships else 2 + member = memberships.get(name if version == 1 else "") + if member is None or not member.is_absolute() or ".." in member.parts: + raise CgroupError(f"No visible cgroup membership for the {name} controller") + candidates = [] + for line in _read(Path("/proc/self/mountinfo")).splitlines(): + before, _, after = line.partition(" - ") + mount, filesystem = before.split(), after.split() + if len(mount) < 6 or len(filesystem) < 3: + continue + if version == 1: + if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + continue + elif filesystem[0] != "cgroup2": + continue + root, target = map(_mount_path, mount[3:5]) + if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): + candidates.append((len(root.parts), target / member.relative_to(root))) + if not candidates: + raise CgroupError(f"No cgroup v{version} mount covers {name} membership {member}") + _, directory = max(candidates) + return Controller(version, directory) + + +@dataclass(frozen=True) +class Controller: + version: int + path: Path + + def _text(self, name): + return _read(self.path / name) + + def _number(self, name, minimum=0): + return _integer(self._text(name).strip(), self.path / name, minimum) + + def cpu_quota(self): + """Return (quota in microseconds or None for unlimited, period in microseconds).""" + if self.version == 1: + quota = self._number("cpu.cfs_quota_us", -1) + period = self._number("cpu.cfs_period_us", 1) + if quota == 0: + raise CgroupError(f"Invalid zero CPU quota in {self.path / 'cpu.cfs_quota_us'}") + return (None if quota == -1 else quota), period + path = self.path / "cpu.max" + fields = _read(path).split() + if len(fields) != 2: + raise CgroupError(f"Expected quota and period in {path}") + quota, period = fields + return (None if quota == "max" else _integer(quota, path, 1)), _integer(period, path, 1) + + def memory_limit(self): + if self.version == 2: + value = self._text("memory.max").strip() + return None if value == "max" else _integer(value, self.path / "memory.max", 1) + # memory.stat includes the tightest ancestor even in a private cgroup namespace. + path = self.path / "memory.stat" + bounds = {} + for line in _read(path).splitlines(): + fields = line.split() + if fields and fields[0] in ("hierarchical_memory_limit", "hierarchical_memsw_limit"): + if len(fields) != 2 or fields[0] in bounds: + raise CgroupError(f"Malformed {fields[0]} in {path}") + bounds[fields[0]] = _integer(fields[1], path) + if "hierarchical_memory_limit" not in bounds: + raise CgroupError(f"Missing hierarchical_memory_limit in {path}") + if "hierarchical_memsw_limit" not in bounds and (self.path / "memory.memsw.limit_in_bytes").exists(): + raise CgroupError(f"Missing hierarchical_memsw_limit in {path}") + limit = min(self._number("memory.limit_in_bytes"), *bounds.values()) + # v1 uses page-rounded LONG_MAX to represent an unbounded limit. + page_size = os.sysconf("SC_PAGE_SIZE") + unlimited = (2**63 - 1) // page_size * page_size + return None if limit >= unlimited else limit + + def swap_limit(self): + """Return a proven swap-only byte limit, or None when v2 allows 'max'.""" + if self.version == 2: + name = "memory.swap.max" + value = self._text(name).strip() + return None if value == "max" else _integer(value, self.path / name) + # The v1 combined RAM+swap limit does not prove zero swapped pages. + # Linux cannot swap when the host has no active swap devices or files. + swaps = _read(Path("/proc/swaps")).splitlines() + if swaps and swaps[0].split() == ["Filename", "Type", "Size", "Used", "Priority"] and not any( + line.strip() for line in swaps[1:]): + return 0 + raise CgroupError("cgroup v1 cannot verify zero swap with active or unreadable host swap") + + def memory_charge(self): + name = "memory.usage_in_bytes" if self.version == 1 else "memory.current" + return self._number(name) + + def memory_peak(self): + """Peak is a report-only kernel counter.""" + name = "memory.max_usage_in_bytes" if self.version == 1 else "memory.peak" + try: + return self._number(name) + except CgroupError: + return None + + def cpu_stat(self): + return self._text("cpu.stat") + + @cache + def _require_v1_leaf(self): + # A v1 counter can be local. A leaf has the same process scope as + # the hierarchical v2 counter, including forked submission processes. + try: + if any(path.is_dir() for path in self.path.iterdir()): + raise CgroupError(f"Cannot establish container-wide v1 OOM counts in {self.path} with child cgroups") + except OSError as error: + raise CgroupError(f"Cannot inspect the v1 memory cgroup {self.path}: {error}") from error + + def v1_limit_hits(self): + # v1 may reject a combined memory+swap charge before charging RAM. + # Kernels without swap accounting omit the memsw counter entirely. + try: + combined = (self.path / "memory.memsw.failcnt").read_text(encoding="ascii") + except FileNotFoundError: + combined = None + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read memory.memsw.failcnt: {error}") from error + return [self._number("memory.failcnt"), + None if combined is None else _integer(combined.strip(), self.path / "memory.memsw.failcnt")] + + def memory_events(self, *required): + name = "memory.oom_control" if self.version == 1 else "memory.events" + if self.version == 1: + if "oom" in required: + raise CgroupError(f"{self.path / name} does not expose a cumulative oom event count") + self._require_v1_leaf() + raw = self._text(name) + counters = {} + for line in raw.splitlines(): + fields = line.split() + if len(fields) != 2 or fields[0] in counters: + raise CgroupError(f"Malformed cgroup events in {self.path / name}") + key, value = fields + counters[key] = _integer(value, self.path / name) + for key in required: + if key not in counters: + raise CgroupError(f"Missing {key} counter in {self.path / name}") + return raw, counters + + +def write_resource_error(directory, error): + """Record why a verifier could not measure a resource-constrained task.""" + path = Path(directory) / "cgroup-status.json" + path.write_text(json.dumps({"measurement_valid": False, "failure_kind": "cgroup_environment", + "error": str(error)}) + "\n") diff --git a/tasks/constrained-molecular-search/tests/runtime/evaluate.py b/tasks/constrained-molecular-search/tests/runtime/evaluate.py index 810ac6d..b718644 100644 --- a/tasks/constrained-molecular-search/tests/runtime/evaluate.py +++ b/tasks/constrained-molecular-search/tests/runtime/evaluate.py @@ -15,6 +15,7 @@ import numpy as np import pynvml +from cgroup_resources import CgroupError, controller, write_resource_error from isolation import UID, cleanup from metrics import JS, PROPERTIES, recalls, reward, summarize, validate_ids from metrics import TARGETS @@ -153,12 +154,13 @@ def __init__(self, fixed_bytes): if pynvml.nvmlDeviceGetCount() != 1: raise ResourceViolation("Exactly one mapped GPU is required") self.gpu = pynvml.nvmlDeviceGetHandleByIndex(0) - quota, period = Path("/sys/fs/cgroup/cpu.max").read_text().split() - maximum = Path("/sys/fs/cgroup/memory.max").read_text().strip() - if quota == "max" or int(quota) / int(period) > 16: - raise ResourceViolation("CPU cgroup limit must be at most 16 cores") - if maximum == "max" or int(maximum) > 32 << 30: - raise ResourceViolation("Memory cgroup limit must be at most 32 GiB") + quota, period = controller("cpu").cpu_quota() + self.memory = controller("memory") + maximum = self.memory.memory_limit() + if quota is None or quota > 16 * period: + raise CgroupError("CPU cgroup limit must be at most 16 cores") + if maximum is None or maximum > 32 << 30: + raise CgroupError("Memory cgroup limit must be at most 32 GiB") def check(self, force=False): self.peak_gpu = max(self.peak_gpu, pynvml.nvmlDeviceGetMemoryInfo(self.gpu).used) @@ -170,11 +172,10 @@ def check(self, force=False): raise ResourceViolation("64 GiB total task storage exceeded") def report(self): - path = Path("/sys/fs/cgroup/memory.peak") return {"gpu": pynvml.nvmlDeviceGetName(self.gpu), "gpu_uuid": pynvml.nvmlDeviceGetUUID(self.gpu), "peak_gpu_bytes": self.peak_gpu, "peak_task_storage_bytes": self.peak_storage, - "container_memory_peak_bytes": int(path.read_text()) if path.exists() else None} + "container_memory_peak_bytes": self.memory.memory_peak()} class Worker: @@ -387,6 +388,8 @@ def main(): report = evaluate(args.private_dir, output, args.library_dir, args.reference_dir) except Exception as error: + if isinstance(error, CgroupError): + write_resource_error(output, error) report["failure"] = type(error).__name__ + ": " + str(error) (output / "report.json").write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") return 0 if report["valid"] else 1 diff --git a/tasks/constrained-molecular-search/tests/test.sh b/tasks/constrained-molecular-search/tests/test.sh index 1df1530..98f05d0 100755 --- a/tasks/constrained-molecular-search/tests/test.sh +++ b/tasks/constrained-molecular-search/tests/test.sh @@ -3,7 +3,7 @@ set -euo pipefail install -d -o root -g root -m 0700 /logs/verifier printf '0\n' > /logs/verifier/reward.txt printf '{"reward":0.0}\n' > /logs/verifier/reward.json -rm -f /logs/verifier/audit.json /logs/verifier/audit-status.json /logs/verifier/reward-details.json /logs/verifier/report.json +rm -f /logs/verifier/cgroup-status.json /logs/verifier/audit.json /logs/verifier/audit-status.json /logs/verifier/reward-details.json /logs/verifier/report.json /opt/conda/bin/python /tests/integrity_gate.py prepare chmod 0700 /tests unset PYTHONHOME PYTHONSTARTUP diff --git a/tasks/constrained-molecular-search/tests/test_gpu_resources.py b/tasks/constrained-molecular-search/tests/test_gpu_resources.py index 5bc28f2..e937f2a 100644 --- a/tasks/constrained-molecular-search/tests/test_gpu_resources.py +++ b/tasks/constrained-molecular-search/tests/test_gpu_resources.py @@ -14,14 +14,13 @@ def nvml(self, count=1): nvmlDeviceGetName=Mock(side_effect=AssertionError('model queried')), nvmlDeviceGetMemoryInfo=Mock(side_effect=AssertionError('capacity queried'))) - def files(self, cores=16, memory=32): - def read(path, *args, **kwargs): - return f'{cores * 100000} 100000' if path.name == 'cpu.max' else str(memory << 30) - return read + def controls(self, cores=16, memory=32): + return SimpleNamespace(cpu_quota=lambda: (cores * 100000, 100000), + memory_limit=lambda: memory << 30) def test_resource_setup_accepts_the_mapped_device(self): nvml = self.nvml() - with patch.object(evaluate, 'pynvml', nvml), patch.object(Path, 'read_text', self.files()): + with patch.object(evaluate, 'pynvml', nvml), patch.object(evaluate, 'controller', return_value=self.controls()): resources = evaluate.Resources(0) nvml.nvmlDeviceGetName.assert_not_called() nvml.nvmlDeviceGetMemoryInfo.assert_not_called() @@ -30,8 +29,9 @@ def test_resource_setup_accepts_the_mapped_device(self): def test_single_device_and_host_limits_are_retained(self): for count, cores, memory in [(0, 16, 32), (2, 16, 32), (1, 17, 32), (1, 16, 33)]: with self.subTest(count=count, cores=cores, memory=memory): - with patch.object(evaluate, 'pynvml', self.nvml(count)), patch.object(Path, 'read_text', self.files(cores, memory)): - with self.assertRaises(evaluate.ResourceViolation): + with patch.object(evaluate, 'pynvml', self.nvml(count)), patch.object(evaluate, 'controller', return_value=self.controls(cores, memory)): + expected = evaluate.ResourceViolation if count != 1 else evaluate.CgroupError + with self.assertRaises(expected): evaluate.Resources(0) if __name__ == '__main__': diff --git a/tasks/large-scale-formula-retrieval/environment/Dockerfile b/tasks/large-scale-formula-retrieval/environment/Dockerfile index 364a799..379b5f6 100644 --- a/tasks/large-scale-formula-retrieval/environment/Dockerfile +++ b/tasks/large-scale-formula-retrieval/environment/Dockerfile @@ -3,7 +3,7 @@ FROM docker.io/hanhainebula/search-swe-base:cpu-py3.12-1.0.0 RUN useradd --create-home --uid 10001 --user-group submission COPY docs /task/docs COPY tools /task/tools -COPY budget_evidence.py /task/budget_evidence.py +COPY budget_evidence.py cgroup_resources.py v1_oom.py /task/ RUN chmod -R a+rX /task && mkdir -p /app ENV HOME=/home/submission CMD ["sh", "-c", "sleep infinity"] diff --git a/tasks/large-scale-formula-retrieval/environment/budget_evidence.py b/tasks/large-scale-formula-retrieval/environment/budget_evidence.py index 09cb213..31843d8 100644 --- a/tasks/large-scale-formula-retrieval/environment/budget_evidence.py +++ b/tasks/large-scale-formula-retrieval/environment/budget_evidence.py @@ -39,7 +39,30 @@ def check_log_budget(source): return total +def verify_memory(): + from cgroup_resources import CgroupError, controller + + memory = controller('memory') + if memory.memory_limit() != 4 << 30: + raise CgroupError('Formula requires a 4 GiB aggregate memory cgroup') + if memory.version == 1: + memory.swap_limit() + return memory + + +def memory_record(memory): + import v1_oom + + record = {'cgroup_memory_peak_bytes': memory.memory_peak()} + if memory.version == 1: + record['cgroup_v1_memory'] = v1_oom.snapshot(memory) + else: + record['memory_events'] = memory.memory_events('oom', 'oom_kill')[0] + return record + + def setup(): + memory_record(verify_memory()) bounded = Path(os.environ['FORMULA_DIAGNOSTICS']) assert bounded.is_dir() and os.stat(bounded).st_dev == os.stat('/app').st_dev # Sticky ownership protects root-published framework artifacts while each @@ -112,8 +135,8 @@ def freeze(): assert logs.is_symlink() and logs.readlink() == bounded # Stop all solver/background processes before snapshotting their logs. stop_solver() - record = {'memory_events': Path('/sys/fs/cgroup/memory.events').read_text(), - 'cgroup_memory_peak_bytes': int(Path('/sys/fs/cgroup/memory.peak').read_text())} + memory = verify_memory() + record = memory_record(memory) os.chown(bounded, 0, 0) bounded.chmod(0o500) snapshot = Path(tempfile.mkdtemp(prefix='.agent-logs-', dir='/logs')) diff --git a/tasks/large-scale-formula-retrieval/environment/cgroup_resources.py b/tasks/large-scale-formula-retrieval/environment/cgroup_resources.py new file mode 100644 index 0000000..ef60a48 --- /dev/null +++ b/tasks/large-scale-formula-retrieval/environment/cgroup_resources.py @@ -0,0 +1,205 @@ +"""Read Linux cgroup counters for the current process and its container. + +Canonical source is environments/cgroups/cgroup_resources.py. Task image +contexts contain identical copies so they can be built independently. +""" + +from dataclasses import dataclass +from functools import cache +import json +import os +from pathlib import Path +import re + + +class CgroupError(RuntimeError): + """The environment cannot establish the task's required resource budget.""" + + +def _read(path): + try: + return path.read_text(encoding="ascii") + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + +def _integer(value, path, minimum=0): + try: + result = int(value) + if result >= minimum: + return result + except ValueError: + pass + raise CgroupError(f"Invalid cgroup value in {path}: {value!r}") + + +def _mount_path(value): + return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + + +@cache +def controller(name): + """Resolve a controller using this process's membership and visible mounts.""" + return resolve_controller(name, Path("/proc/self/cgroup")) + + +def resolve_controller(name, membership_file): + """Resolve a process's membership against mounts visible to the reader.""" + memberships = {} + for line in _read(membership_file).splitlines(): + fields = line.split(":", 2) + if len(fields) != 3: + raise CgroupError(f"Invalid cgroup membership: {line!r}") + _, controllers, location = fields + for key in controllers.split(","): + memberships[key] = Path(location) + # A hybrid host can mount v2 alongside active v1 controllers. + version = 1 if name in memberships else 2 + member = memberships.get(name if version == 1 else "") + if member is None or not member.is_absolute() or ".." in member.parts: + raise CgroupError(f"No visible cgroup membership for the {name} controller") + candidates = [] + for line in _read(Path("/proc/self/mountinfo")).splitlines(): + before, _, after = line.partition(" - ") + mount, filesystem = before.split(), after.split() + if len(mount) < 6 or len(filesystem) < 3: + continue + if version == 1: + if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + continue + elif filesystem[0] != "cgroup2": + continue + root, target = map(_mount_path, mount[3:5]) + if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): + candidates.append((len(root.parts), target / member.relative_to(root))) + if not candidates: + raise CgroupError(f"No cgroup v{version} mount covers {name} membership {member}") + _, directory = max(candidates) + return Controller(version, directory) + + +@dataclass(frozen=True) +class Controller: + version: int + path: Path + + def _text(self, name): + return _read(self.path / name) + + def _number(self, name, minimum=0): + return _integer(self._text(name).strip(), self.path / name, minimum) + + def cpu_quota(self): + """Return (quota in microseconds or None for unlimited, period in microseconds).""" + if self.version == 1: + quota = self._number("cpu.cfs_quota_us", -1) + period = self._number("cpu.cfs_period_us", 1) + if quota == 0: + raise CgroupError(f"Invalid zero CPU quota in {self.path / 'cpu.cfs_quota_us'}") + return (None if quota == -1 else quota), period + path = self.path / "cpu.max" + fields = _read(path).split() + if len(fields) != 2: + raise CgroupError(f"Expected quota and period in {path}") + quota, period = fields + return (None if quota == "max" else _integer(quota, path, 1)), _integer(period, path, 1) + + def memory_limit(self): + if self.version == 2: + value = self._text("memory.max").strip() + return None if value == "max" else _integer(value, self.path / "memory.max", 1) + # memory.stat includes the tightest ancestor even in a private cgroup namespace. + path = self.path / "memory.stat" + bounds = {} + for line in _read(path).splitlines(): + fields = line.split() + if fields and fields[0] in ("hierarchical_memory_limit", "hierarchical_memsw_limit"): + if len(fields) != 2 or fields[0] in bounds: + raise CgroupError(f"Malformed {fields[0]} in {path}") + bounds[fields[0]] = _integer(fields[1], path) + if "hierarchical_memory_limit" not in bounds: + raise CgroupError(f"Missing hierarchical_memory_limit in {path}") + if "hierarchical_memsw_limit" not in bounds and (self.path / "memory.memsw.limit_in_bytes").exists(): + raise CgroupError(f"Missing hierarchical_memsw_limit in {path}") + limit = min(self._number("memory.limit_in_bytes"), *bounds.values()) + # v1 uses page-rounded LONG_MAX to represent an unbounded limit. + page_size = os.sysconf("SC_PAGE_SIZE") + unlimited = (2**63 - 1) // page_size * page_size + return None if limit >= unlimited else limit + + def swap_limit(self): + """Return a proven swap-only byte limit, or None when v2 allows 'max'.""" + if self.version == 2: + name = "memory.swap.max" + value = self._text(name).strip() + return None if value == "max" else _integer(value, self.path / name) + # The v1 combined RAM+swap limit does not prove zero swapped pages. + # Linux cannot swap when the host has no active swap devices or files. + swaps = _read(Path("/proc/swaps")).splitlines() + if swaps and swaps[0].split() == ["Filename", "Type", "Size", "Used", "Priority"] and not any( + line.strip() for line in swaps[1:]): + return 0 + raise CgroupError("cgroup v1 cannot verify zero swap with active or unreadable host swap") + + def memory_charge(self): + name = "memory.usage_in_bytes" if self.version == 1 else "memory.current" + return self._number(name) + + def memory_peak(self): + """Peak is a report-only kernel counter.""" + name = "memory.max_usage_in_bytes" if self.version == 1 else "memory.peak" + try: + return self._number(name) + except CgroupError: + return None + + def cpu_stat(self): + return self._text("cpu.stat") + + @cache + def _require_v1_leaf(self): + # A v1 counter can be local. A leaf has the same process scope as + # the hierarchical v2 counter, including forked submission processes. + try: + if any(path.is_dir() for path in self.path.iterdir()): + raise CgroupError(f"Cannot establish container-wide v1 OOM counts in {self.path} with child cgroups") + except OSError as error: + raise CgroupError(f"Cannot inspect the v1 memory cgroup {self.path}: {error}") from error + + def v1_limit_hits(self): + # v1 may reject a combined memory+swap charge before charging RAM. + # Kernels without swap accounting omit the memsw counter entirely. + try: + combined = (self.path / "memory.memsw.failcnt").read_text(encoding="ascii") + except FileNotFoundError: + combined = None + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read memory.memsw.failcnt: {error}") from error + return [self._number("memory.failcnt"), + None if combined is None else _integer(combined.strip(), self.path / "memory.memsw.failcnt")] + + def memory_events(self, *required): + name = "memory.oom_control" if self.version == 1 else "memory.events" + if self.version == 1: + if "oom" in required: + raise CgroupError(f"{self.path / name} does not expose a cumulative oom event count") + self._require_v1_leaf() + raw = self._text(name) + counters = {} + for line in raw.splitlines(): + fields = line.split() + if len(fields) != 2 or fields[0] in counters: + raise CgroupError(f"Malformed cgroup events in {self.path / name}") + key, value = fields + counters[key] = _integer(value, self.path / name) + for key in required: + if key not in counters: + raise CgroupError(f"Missing {key} counter in {self.path / name}") + return raw, counters + + +def write_resource_error(directory, error): + """Record why a verifier could not measure a resource-constrained task.""" + path = Path(directory) / "cgroup-status.json" + path.write_text(json.dumps({"measurement_valid": False, "failure_kind": "cgroup_environment", + "error": str(error)}) + "\n") diff --git a/tasks/large-scale-formula-retrieval/environment/docker-compose.yaml b/tasks/large-scale-formula-retrieval/environment/docker-compose.yaml index b38bb64..85fdb63 100644 --- a/tasks/large-scale-formula-retrieval/environment/docker-compose.yaml +++ b/tasks/large-scale-formula-retrieval/environment/docker-compose.yaml @@ -1,4 +1,5 @@ x-search-swe: + v1_oom_notifications: true bounded_agent_logs: finalize: ["/opt/conda/bin/python", "/task/budget_evidence.py", "freeze"] diff --git a/tasks/large-scale-formula-retrieval/environment/v1_oom.py b/tasks/large-scale-formula-retrieval/environment/v1_oom.py new file mode 100644 index 0000000..da33623 --- /dev/null +++ b/tasks/large-scale-formula-retrieval/environment/v1_oom.py @@ -0,0 +1,91 @@ +"""Container-side OOM evidence shared by Formula and Bimetric on cgroup v1.""" + +import json +import socket + +if __package__: + from .cgroup_resources import CgroupError +else: # Independent task image. + from cgroup_resources import CgroupError + + +SOCKET = "/run/search-swe-memory/oom.sock" + + +def snapshot(memory): + """Sample the limit counters first so the notification check covers that interval.""" + hits = memory.v1_limit_hits() + _, counters = memory.memory_events("oom_kill") + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(5) + client.connect(SOCKET) + with client.makefile("rb") as stream: + watch = json.loads(stream.readline(4096)) + except (FileNotFoundError, PermissionError, ConnectionError, TimeoutError): + watch = None # Raw cgroup counters can still establish some outcomes. + except (OSError, ValueError) as error: + raise CgroupError(f"Cannot read v1 OOM observer: {error}") from error + if watch is not None: + try: + info = memory.path.stat() + except OSError as error: + raise CgroupError(f"Cannot identify the task memory cgroup: {error}") from error + if not isinstance(watch, dict) or watch.get("cgroup") != [info.st_dev, info.st_ino]: + raise CgroupError("Host OOM observer and task see different memory cgroups") + return {"limit_hits": hits, "oom_kill": counters["oom_kill"], "observer": watch} + + +def _counter(value): + return type(value) is int and value >= 0 + + +def _limit_hits(value): + return (isinstance(value, list) and len(value) == 2 and _counter(value[0]) + and (value[1] is None or _counter(value[1]))) + + +def _validate(sample): + if (not isinstance(sample, dict) or not _limit_hits(sample.get("limit_hits")) + or not _counter(sample.get("oom_kill")) or "observer" not in sample): + raise CgroupError("Invalid v1 memory counters") + watch = sample["observer"] + if watch is not None and (not isinstance(watch, dict) or not isinstance(watch.get("id"), str) + or not watch["id"] or not _counter(watch.get("events")) or not _counter(watch.get("parent_events")) + or not _limit_hits(watch.get("initial_hits"))): + raise CgroupError("Invalid v1 OOM notification evidence") + + +def violation(sample, baseline=None, *, include_kills): + """Formula checks the container lifetime. Bimetric supplies a verifier baseline.""" + _validate(sample) + hits = sample["limit_hits"] + old_hits = [0, None if hits[1] is None else 0] + old_kills = 0 + old_watch = None + if baseline is not None: + _validate(baseline) + old_hits, old_kills, old_watch = baseline["limit_hits"], baseline["oom_kill"], baseline["observer"] + if (sample["oom_kill"] < old_kills + or any((a is None) != (b is None) or (a is not None and a < b) for a, b in zip(hits, old_hits))): + raise CgroupError("Reset v1 memory counters") + if include_kills and sample["oom_kill"] > old_kills: + return True + watch = sample["observer"] + if watch and (baseline is None or (old_watch and old_watch["id"] == watch["id"])): + start = old_watch["events"] if old_watch else 0 + if watch["events"] < start or (old_watch and watch["parent_events"] < old_watch["parent_events"]): + raise CgroupError("Reset v1 OOM notifications") + # Ancestor OOMs notify descendants too. A plain eventfd cannot + # attribute a notification if an ancestor was also under pressure. + if watch["parent_events"] == 0: + if watch["events"] > start: + return True + if old_watch is not None or not any(watch["initial_hits"]): + return False + # With stable caps, local memcg OOM first requires a failed RAM or memsw + # charge. Successful reclaim also increments these counters. + if hits == old_hits: + return False + raise CgroupError("The v1 memory limit was reached, but local OOM could not be distinguished " + "from reclaim or ancestor pressure in this measurement") diff --git a/tasks/large-scale-formula-retrieval/tests/cgroup_resources.py b/tasks/large-scale-formula-retrieval/tests/cgroup_resources.py new file mode 100644 index 0000000..ef60a48 --- /dev/null +++ b/tasks/large-scale-formula-retrieval/tests/cgroup_resources.py @@ -0,0 +1,205 @@ +"""Read Linux cgroup counters for the current process and its container. + +Canonical source is environments/cgroups/cgroup_resources.py. Task image +contexts contain identical copies so they can be built independently. +""" + +from dataclasses import dataclass +from functools import cache +import json +import os +from pathlib import Path +import re + + +class CgroupError(RuntimeError): + """The environment cannot establish the task's required resource budget.""" + + +def _read(path): + try: + return path.read_text(encoding="ascii") + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + +def _integer(value, path, minimum=0): + try: + result = int(value) + if result >= minimum: + return result + except ValueError: + pass + raise CgroupError(f"Invalid cgroup value in {path}: {value!r}") + + +def _mount_path(value): + return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + + +@cache +def controller(name): + """Resolve a controller using this process's membership and visible mounts.""" + return resolve_controller(name, Path("/proc/self/cgroup")) + + +def resolve_controller(name, membership_file): + """Resolve a process's membership against mounts visible to the reader.""" + memberships = {} + for line in _read(membership_file).splitlines(): + fields = line.split(":", 2) + if len(fields) != 3: + raise CgroupError(f"Invalid cgroup membership: {line!r}") + _, controllers, location = fields + for key in controllers.split(","): + memberships[key] = Path(location) + # A hybrid host can mount v2 alongside active v1 controllers. + version = 1 if name in memberships else 2 + member = memberships.get(name if version == 1 else "") + if member is None or not member.is_absolute() or ".." in member.parts: + raise CgroupError(f"No visible cgroup membership for the {name} controller") + candidates = [] + for line in _read(Path("/proc/self/mountinfo")).splitlines(): + before, _, after = line.partition(" - ") + mount, filesystem = before.split(), after.split() + if len(mount) < 6 or len(filesystem) < 3: + continue + if version == 1: + if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + continue + elif filesystem[0] != "cgroup2": + continue + root, target = map(_mount_path, mount[3:5]) + if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): + candidates.append((len(root.parts), target / member.relative_to(root))) + if not candidates: + raise CgroupError(f"No cgroup v{version} mount covers {name} membership {member}") + _, directory = max(candidates) + return Controller(version, directory) + + +@dataclass(frozen=True) +class Controller: + version: int + path: Path + + def _text(self, name): + return _read(self.path / name) + + def _number(self, name, minimum=0): + return _integer(self._text(name).strip(), self.path / name, minimum) + + def cpu_quota(self): + """Return (quota in microseconds or None for unlimited, period in microseconds).""" + if self.version == 1: + quota = self._number("cpu.cfs_quota_us", -1) + period = self._number("cpu.cfs_period_us", 1) + if quota == 0: + raise CgroupError(f"Invalid zero CPU quota in {self.path / 'cpu.cfs_quota_us'}") + return (None if quota == -1 else quota), period + path = self.path / "cpu.max" + fields = _read(path).split() + if len(fields) != 2: + raise CgroupError(f"Expected quota and period in {path}") + quota, period = fields + return (None if quota == "max" else _integer(quota, path, 1)), _integer(period, path, 1) + + def memory_limit(self): + if self.version == 2: + value = self._text("memory.max").strip() + return None if value == "max" else _integer(value, self.path / "memory.max", 1) + # memory.stat includes the tightest ancestor even in a private cgroup namespace. + path = self.path / "memory.stat" + bounds = {} + for line in _read(path).splitlines(): + fields = line.split() + if fields and fields[0] in ("hierarchical_memory_limit", "hierarchical_memsw_limit"): + if len(fields) != 2 or fields[0] in bounds: + raise CgroupError(f"Malformed {fields[0]} in {path}") + bounds[fields[0]] = _integer(fields[1], path) + if "hierarchical_memory_limit" not in bounds: + raise CgroupError(f"Missing hierarchical_memory_limit in {path}") + if "hierarchical_memsw_limit" not in bounds and (self.path / "memory.memsw.limit_in_bytes").exists(): + raise CgroupError(f"Missing hierarchical_memsw_limit in {path}") + limit = min(self._number("memory.limit_in_bytes"), *bounds.values()) + # v1 uses page-rounded LONG_MAX to represent an unbounded limit. + page_size = os.sysconf("SC_PAGE_SIZE") + unlimited = (2**63 - 1) // page_size * page_size + return None if limit >= unlimited else limit + + def swap_limit(self): + """Return a proven swap-only byte limit, or None when v2 allows 'max'.""" + if self.version == 2: + name = "memory.swap.max" + value = self._text(name).strip() + return None if value == "max" else _integer(value, self.path / name) + # The v1 combined RAM+swap limit does not prove zero swapped pages. + # Linux cannot swap when the host has no active swap devices or files. + swaps = _read(Path("/proc/swaps")).splitlines() + if swaps and swaps[0].split() == ["Filename", "Type", "Size", "Used", "Priority"] and not any( + line.strip() for line in swaps[1:]): + return 0 + raise CgroupError("cgroup v1 cannot verify zero swap with active or unreadable host swap") + + def memory_charge(self): + name = "memory.usage_in_bytes" if self.version == 1 else "memory.current" + return self._number(name) + + def memory_peak(self): + """Peak is a report-only kernel counter.""" + name = "memory.max_usage_in_bytes" if self.version == 1 else "memory.peak" + try: + return self._number(name) + except CgroupError: + return None + + def cpu_stat(self): + return self._text("cpu.stat") + + @cache + def _require_v1_leaf(self): + # A v1 counter can be local. A leaf has the same process scope as + # the hierarchical v2 counter, including forked submission processes. + try: + if any(path.is_dir() for path in self.path.iterdir()): + raise CgroupError(f"Cannot establish container-wide v1 OOM counts in {self.path} with child cgroups") + except OSError as error: + raise CgroupError(f"Cannot inspect the v1 memory cgroup {self.path}: {error}") from error + + def v1_limit_hits(self): + # v1 may reject a combined memory+swap charge before charging RAM. + # Kernels without swap accounting omit the memsw counter entirely. + try: + combined = (self.path / "memory.memsw.failcnt").read_text(encoding="ascii") + except FileNotFoundError: + combined = None + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read memory.memsw.failcnt: {error}") from error + return [self._number("memory.failcnt"), + None if combined is None else _integer(combined.strip(), self.path / "memory.memsw.failcnt")] + + def memory_events(self, *required): + name = "memory.oom_control" if self.version == 1 else "memory.events" + if self.version == 1: + if "oom" in required: + raise CgroupError(f"{self.path / name} does not expose a cumulative oom event count") + self._require_v1_leaf() + raw = self._text(name) + counters = {} + for line in raw.splitlines(): + fields = line.split() + if len(fields) != 2 or fields[0] in counters: + raise CgroupError(f"Malformed cgroup events in {self.path / name}") + key, value = fields + counters[key] = _integer(value, self.path / name) + for key in required: + if key not in counters: + raise CgroupError(f"Missing {key} counter in {self.path / name}") + return raw, counters + + +def write_resource_error(directory, error): + """Record why a verifier could not measure a resource-constrained task.""" + path = Path(directory) / "cgroup-status.json" + path.write_text(json.dumps({"measurement_valid": False, "failure_kind": "cgroup_environment", + "error": str(error)}) + "\n") diff --git a/tasks/large-scale-formula-retrieval/tests/development.py b/tasks/large-scale-formula-retrieval/tests/development.py index 736deea..58a1c32 100644 --- a/tasks/large-scale-formula-retrieval/tests/development.py +++ b/tasks/large-scale-formula-retrieval/tests/development.py @@ -7,11 +7,19 @@ import json from pathlib import Path +from cgroup_resources import CgroupError +import v1_oom + def check_development(evidence=Path('/task-evidence')): if evidence.stat().st_mode & 0o077: raise RuntimeError('Development evidence must be private to the host/verifier owner') record = json.loads((evidence / 'agent.json').read_text()) + if 'cgroup_v1_memory' in record: + if 'memory_events' in record: + raise CgroupError('Conflicting development memory evidence for v1 and v2') + exceeded = v1_oom.violation(record['cgroup_v1_memory'], include_kills=True) + return record, 'development memory limit' if exceeded else None events = dict(line.split() for line in record['memory_events'].splitlines()) counts = [int(events[name]) for name in ['oom', 'oom_kill']] if any(count < 0 for count in counts): diff --git a/tasks/large-scale-formula-retrieval/tests/docker-compose.yaml b/tasks/large-scale-formula-retrieval/tests/docker-compose.yaml index 7365cf7..8961b14 100644 --- a/tasks/large-scale-formula-retrieval/tests/docker-compose.yaml +++ b/tasks/large-scale-formula-retrieval/tests/docker-compose.yaml @@ -1,3 +1,6 @@ +x-search-swe: + v1_oom_notifications: true + services: main: ipc: private diff --git a/tasks/large-scale-formula-retrieval/tests/grader.py b/tasks/large-scale-formula-retrieval/tests/grader.py index dc2b5fd..379f6ab 100644 --- a/tasks/large-scale-formula-retrieval/tests/grader.py +++ b/tasks/large-scale-formula-retrieval/tests/grader.py @@ -6,6 +6,8 @@ import time import numpy as np +from cgroup_resources import CgroupError, controller, write_resource_error +import v1_oom from development import check_development from runtime import Worker, Violation from scoring import read_qrels, evaluate @@ -24,10 +26,17 @@ def write(name, obj): def finish_report(report): - events = Path('/sys/fs/cgroup/memory.events').read_text() - report['cgroup_memory_events'] = events - report['cgroup_memory_peak_bytes'] = int(Path('/sys/fs/cgroup/memory.peak').read_text()) - if any(int(r.split()[1]) for r in events.splitlines() if r.split()[0] in ['oom', 'oom_kill']): + memory = controller('memory') + report['cgroup_memory_peak_bytes'] = memory.memory_peak() + if memory.version == 1: + sample = v1_oom.snapshot(memory) + report['cgroup_v1_memory'] = sample + exceeded = v1_oom.violation(sample, include_kills=True) + else: + events, counters = memory.memory_events('oom', 'oom_kill') + report['cgroup_memory_events'] = events + exceeded = counters['oom'] or counters['oom_kill'] + if exceeded: report['failure'] = 'memory limit' if report.get('failure'): report['raw_reward'] = 0.0 @@ -37,6 +46,15 @@ def finish_report(report): def main(): started = time.monotonic() + memory = controller('memory') + if memory.memory_limit() != 4 << 30: + raise CgroupError('Formula requires a 4 GiB aggregate memory cgroup') + if memory.version == 1: + memory.swap_limit() + v1_oom.snapshot(memory) + else: + memory.memory_events('oom', 'oom_kill') + memory.memory_charge() queries = [json.loads(s) for s in Path('/tests/data/queries.jsonl').read_text().splitlines()] labels = read_qrels(Path('/tests/data/qrels.tsv')) if len(queries) != 76 or set(labels) != {q['query_id'] for q in queries}: @@ -179,5 +197,7 @@ def load_worker(): try: main() except Exception as exc: + if isinstance(exc, CgroupError): + write_resource_error(LOG, exc) write('infrastructure-error.json', {'error': f'{type(exc).__name__}: {exc}'}) raise diff --git a/tasks/large-scale-formula-retrieval/tests/runtime.py b/tasks/large-scale-formula-retrieval/tests/runtime.py index c08c7cd..89a55ef 100644 --- a/tasks/large-scale-formula-retrieval/tests/runtime.py +++ b/tasks/large-scale-formula-retrieval/tests/runtime.py @@ -11,6 +11,7 @@ import time import psutil +from cgroup_resources import controller UID = 10001 MEMORY = 4 << 30 @@ -149,6 +150,7 @@ def __init__(self, command, work, log, cpu_ids, fixed_bytes=0, *, allow_api=Fals os.set_blocking(self.process.stderr.fileno(), False) os.set_blocking(self.process.stdin.fileno(), False) self.buffer = bytearray() + self.memory = controller('memory') self.last_sample = self.last_disk = 0 self.peak = self.disk_peak = self.lifetime_peak = 0 self.phase_start = time.monotonic() @@ -172,7 +174,7 @@ def sample(self, force=False): # Aggregate RSS double-counts shared pages in multiprocess indexes. # Keep it as a diagnostic, but enforce the container's actual charge, # including page cache and descendants, consistent with Docker. - if int(Path('/sys/fs/cgroup/memory.current').read_text()) > MEMORY: + if self.memory.memory_charge() > MEMORY: raise Violation('memory limit') if force or now - self.last_disk >= .5: size = self.fixed_bytes + os.fstat(self.stderr.fileno()).st_size diff --git a/tasks/large-scale-formula-retrieval/tests/test.sh b/tasks/large-scale-formula-retrieval/tests/test.sh index ba270ec..6f8ad18 100755 --- a/tasks/large-scale-formula-retrieval/tests/test.sh +++ b/tasks/large-scale-formula-retrieval/tests/test.sh @@ -6,7 +6,7 @@ cd /tests install -d -m 0700 /logs /logs/verifier printf '0\n' > /logs/verifier/reward.txt printf '{"reward":0}\n' > /logs/verifier/reward.json -rm -f /logs/verifier/audit.json /logs/verifier/audit-status.json /logs/verifier/reward-details.json /logs/verifier/evaluation.json +rm -f /logs/verifier/cgroup-status.json /logs/verifier/audit.json /logs/verifier/audit-status.json /logs/verifier/reward-details.json /logs/verifier/evaluation.json rm -f /audit/output/audit.json /audit/output/reward-details.json /opt/conda/bin/python /tests/integrity_gate.py prepare cleanup() { diff --git a/tasks/large-scale-formula-retrieval/tests/test_development.py b/tasks/large-scale-formula-retrieval/tests/test_development.py index dd6806e..931b6ea 100644 --- a/tasks/large-scale-formula-retrieval/tests/test_development.py +++ b/tasks/large-scale-formula-retrieval/tests/test_development.py @@ -3,6 +3,7 @@ import tempfile import unittest +from cgroup_resources import CgroupError from development import check_development @@ -39,6 +40,24 @@ def test_development_oom(self): self.memory(oom, oom_kill) self.assertEqual(check_development(self.evidence)[1], 'development memory limit') + def test_v1_frozen_evidence(self): + path = self.evidence / 'agent.json' + watch = {'id': 'test', 'initial_hits': [0, 0], 'events': 0, 'parent_events': 0} + sample = {'limit_hits': [1, 7], 'oom_kill': 0, 'observer': watch} + record = {'cgroup_v1_memory': sample} + path.write_text(json.dumps(record)) + self.assertIsNone(check_development(self.evidence)[1]) + watch['events'] = 1 + path.write_text(json.dumps(record)) + self.assertEqual(check_development(self.evidence)[1], 'development memory limit') + sample['observer'] = None + path.write_text(json.dumps(record)) + with self.assertRaises(CgroupError): + check_development(self.evidence) + path.write_text(json.dumps({**record, 'memory_events': 'oom 0\noom_kill 0\n'})) + with self.assertRaisesRegex(CgroupError, 'Conflicting'): + check_development(self.evidence) + def test_missing_memory_evidence_is_an_error(self): (self.evidence / 'agent.json').unlink() with self.assertRaises(FileNotFoundError): diff --git a/tasks/large-scale-formula-retrieval/tests/test_runtime.py b/tasks/large-scale-formula-retrieval/tests/test_runtime.py index 87c3cfd..433cb5e 100644 --- a/tasks/large-scale-formula-retrieval/tests/test_runtime.py +++ b/tasks/large-scale-formula-retrieval/tests/test_runtime.py @@ -149,7 +149,7 @@ def test_exit(self): def test_actual_memory_excess(self): previous = runtime.MEMORY try: - runtime.MEMORY = int(Path('/sys/fs/cgroup/memory.current').read_text()) + (64 << 20) + runtime.MEMORY = self.worker.memory.memory_charge() + (64 << 20) self.worker.begin(5) with self.assertRaisesRegex(Violation, 'memory limit'): self.worker.request({'op': 'memory'}) @@ -160,7 +160,7 @@ def test_actual_memory_excess(self): def test_shared_pages_are_not_charged_per_process(self): previous = runtime.MEMORY try: - runtime.MEMORY = int(Path('/sys/fs/cgroup/memory.current').read_text()) + (192 << 20) + runtime.MEMORY = self.worker.memory.memory_charge() + (192 << 20) self.worker.begin(5) self.assertEqual(self.worker.request({'op': 'shared_memory'}), {'ok': True}) result = self.worker.finish() diff --git a/tasks/large-scale-formula-retrieval/tests/v1_oom.py b/tasks/large-scale-formula-retrieval/tests/v1_oom.py new file mode 100644 index 0000000..da33623 --- /dev/null +++ b/tasks/large-scale-formula-retrieval/tests/v1_oom.py @@ -0,0 +1,91 @@ +"""Container-side OOM evidence shared by Formula and Bimetric on cgroup v1.""" + +import json +import socket + +if __package__: + from .cgroup_resources import CgroupError +else: # Independent task image. + from cgroup_resources import CgroupError + + +SOCKET = "/run/search-swe-memory/oom.sock" + + +def snapshot(memory): + """Sample the limit counters first so the notification check covers that interval.""" + hits = memory.v1_limit_hits() + _, counters = memory.memory_events("oom_kill") + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(5) + client.connect(SOCKET) + with client.makefile("rb") as stream: + watch = json.loads(stream.readline(4096)) + except (FileNotFoundError, PermissionError, ConnectionError, TimeoutError): + watch = None # Raw cgroup counters can still establish some outcomes. + except (OSError, ValueError) as error: + raise CgroupError(f"Cannot read v1 OOM observer: {error}") from error + if watch is not None: + try: + info = memory.path.stat() + except OSError as error: + raise CgroupError(f"Cannot identify the task memory cgroup: {error}") from error + if not isinstance(watch, dict) or watch.get("cgroup") != [info.st_dev, info.st_ino]: + raise CgroupError("Host OOM observer and task see different memory cgroups") + return {"limit_hits": hits, "oom_kill": counters["oom_kill"], "observer": watch} + + +def _counter(value): + return type(value) is int and value >= 0 + + +def _limit_hits(value): + return (isinstance(value, list) and len(value) == 2 and _counter(value[0]) + and (value[1] is None or _counter(value[1]))) + + +def _validate(sample): + if (not isinstance(sample, dict) or not _limit_hits(sample.get("limit_hits")) + or not _counter(sample.get("oom_kill")) or "observer" not in sample): + raise CgroupError("Invalid v1 memory counters") + watch = sample["observer"] + if watch is not None and (not isinstance(watch, dict) or not isinstance(watch.get("id"), str) + or not watch["id"] or not _counter(watch.get("events")) or not _counter(watch.get("parent_events")) + or not _limit_hits(watch.get("initial_hits"))): + raise CgroupError("Invalid v1 OOM notification evidence") + + +def violation(sample, baseline=None, *, include_kills): + """Formula checks the container lifetime. Bimetric supplies a verifier baseline.""" + _validate(sample) + hits = sample["limit_hits"] + old_hits = [0, None if hits[1] is None else 0] + old_kills = 0 + old_watch = None + if baseline is not None: + _validate(baseline) + old_hits, old_kills, old_watch = baseline["limit_hits"], baseline["oom_kill"], baseline["observer"] + if (sample["oom_kill"] < old_kills + or any((a is None) != (b is None) or (a is not None and a < b) for a, b in zip(hits, old_hits))): + raise CgroupError("Reset v1 memory counters") + if include_kills and sample["oom_kill"] > old_kills: + return True + watch = sample["observer"] + if watch and (baseline is None or (old_watch and old_watch["id"] == watch["id"])): + start = old_watch["events"] if old_watch else 0 + if watch["events"] < start or (old_watch and watch["parent_events"] < old_watch["parent_events"]): + raise CgroupError("Reset v1 OOM notifications") + # Ancestor OOMs notify descendants too. A plain eventfd cannot + # attribute a notification if an ancestor was also under pressure. + if watch["parent_events"] == 0: + if watch["events"] > start: + return True + if old_watch is not None or not any(watch["initial_hits"]): + return False + # With stable caps, local memcg OOM first requires a failed RAM or memsw + # charge. Successful reclaim also increments these counters. + if hits == old_hits: + return False + raise CgroupError("The v1 memory limit was reached, but local OOM could not be distinguished " + "from reclaim or ancestor pressure in this measurement") diff --git a/tasks/memory-constrained-dense-retrieval/README.md b/tasks/memory-constrained-dense-retrieval/README.md index b9c0936..fadd80c 100644 --- a/tasks/memory-constrained-dense-retrieval/README.md +++ b/tasks/memory-constrained-dense-retrieval/README.md @@ -169,7 +169,7 @@ assets, credentials, or hardware. ### Local regression checks -Run `python -m unittest discover -s tasks/memory-constrained-dense-retrieval/tests -p "test_*.py"`. Runtime grading requires Linux blocking process waits and a cgroup-v2 container with memory.max=536870912 and memory.swap.max=0. The grader records final-verifier container memory peak/events; any OOM kill in that container fails the evaluation. The two Compose overlays set identical 16-CPU, 512-MiB/no-swap container limits. The native query launcher enforces one logical CPU for all online submission processes. +Run `python -m unittest discover -s tasks/memory-constrained-dense-retrieval/tests -p "test_*.py"`. Runtime grading requires Linux blocking process waits and a container with a verified 512 MiB cgroup memory limit and no swap. The grader records final-verifier container memory peak/events; any OOM kill in that container fails the evaluation. The two Compose overlays set identical 16-CPU, 512-MiB/no-swap container limits. The native query launcher enforces one logical CPU for all online submission processes. The verifier snapshots each output immediately after observing client exit, in a root-only directory. Query timing ends after capture completes, so every scored byte was received within the measured interval. Validation uses only that protected snapshot; writes through retained descriptors after capture cannot change it. The same capture-inclusive duration controls the 0.5-second deadline and P95. diff --git a/tasks/memory-constrained-dense-retrieval/tests/cgroup_resources.py b/tasks/memory-constrained-dense-retrieval/tests/cgroup_resources.py new file mode 100644 index 0000000..ef60a48 --- /dev/null +++ b/tasks/memory-constrained-dense-retrieval/tests/cgroup_resources.py @@ -0,0 +1,205 @@ +"""Read Linux cgroup counters for the current process and its container. + +Canonical source is environments/cgroups/cgroup_resources.py. Task image +contexts contain identical copies so they can be built independently. +""" + +from dataclasses import dataclass +from functools import cache +import json +import os +from pathlib import Path +import re + + +class CgroupError(RuntimeError): + """The environment cannot establish the task's required resource budget.""" + + +def _read(path): + try: + return path.read_text(encoding="ascii") + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + +def _integer(value, path, minimum=0): + try: + result = int(value) + if result >= minimum: + return result + except ValueError: + pass + raise CgroupError(f"Invalid cgroup value in {path}: {value!r}") + + +def _mount_path(value): + return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + + +@cache +def controller(name): + """Resolve a controller using this process's membership and visible mounts.""" + return resolve_controller(name, Path("/proc/self/cgroup")) + + +def resolve_controller(name, membership_file): + """Resolve a process's membership against mounts visible to the reader.""" + memberships = {} + for line in _read(membership_file).splitlines(): + fields = line.split(":", 2) + if len(fields) != 3: + raise CgroupError(f"Invalid cgroup membership: {line!r}") + _, controllers, location = fields + for key in controllers.split(","): + memberships[key] = Path(location) + # A hybrid host can mount v2 alongside active v1 controllers. + version = 1 if name in memberships else 2 + member = memberships.get(name if version == 1 else "") + if member is None or not member.is_absolute() or ".." in member.parts: + raise CgroupError(f"No visible cgroup membership for the {name} controller") + candidates = [] + for line in _read(Path("/proc/self/mountinfo")).splitlines(): + before, _, after = line.partition(" - ") + mount, filesystem = before.split(), after.split() + if len(mount) < 6 or len(filesystem) < 3: + continue + if version == 1: + if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + continue + elif filesystem[0] != "cgroup2": + continue + root, target = map(_mount_path, mount[3:5]) + if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): + candidates.append((len(root.parts), target / member.relative_to(root))) + if not candidates: + raise CgroupError(f"No cgroup v{version} mount covers {name} membership {member}") + _, directory = max(candidates) + return Controller(version, directory) + + +@dataclass(frozen=True) +class Controller: + version: int + path: Path + + def _text(self, name): + return _read(self.path / name) + + def _number(self, name, minimum=0): + return _integer(self._text(name).strip(), self.path / name, minimum) + + def cpu_quota(self): + """Return (quota in microseconds or None for unlimited, period in microseconds).""" + if self.version == 1: + quota = self._number("cpu.cfs_quota_us", -1) + period = self._number("cpu.cfs_period_us", 1) + if quota == 0: + raise CgroupError(f"Invalid zero CPU quota in {self.path / 'cpu.cfs_quota_us'}") + return (None if quota == -1 else quota), period + path = self.path / "cpu.max" + fields = _read(path).split() + if len(fields) != 2: + raise CgroupError(f"Expected quota and period in {path}") + quota, period = fields + return (None if quota == "max" else _integer(quota, path, 1)), _integer(period, path, 1) + + def memory_limit(self): + if self.version == 2: + value = self._text("memory.max").strip() + return None if value == "max" else _integer(value, self.path / "memory.max", 1) + # memory.stat includes the tightest ancestor even in a private cgroup namespace. + path = self.path / "memory.stat" + bounds = {} + for line in _read(path).splitlines(): + fields = line.split() + if fields and fields[0] in ("hierarchical_memory_limit", "hierarchical_memsw_limit"): + if len(fields) != 2 or fields[0] in bounds: + raise CgroupError(f"Malformed {fields[0]} in {path}") + bounds[fields[0]] = _integer(fields[1], path) + if "hierarchical_memory_limit" not in bounds: + raise CgroupError(f"Missing hierarchical_memory_limit in {path}") + if "hierarchical_memsw_limit" not in bounds and (self.path / "memory.memsw.limit_in_bytes").exists(): + raise CgroupError(f"Missing hierarchical_memsw_limit in {path}") + limit = min(self._number("memory.limit_in_bytes"), *bounds.values()) + # v1 uses page-rounded LONG_MAX to represent an unbounded limit. + page_size = os.sysconf("SC_PAGE_SIZE") + unlimited = (2**63 - 1) // page_size * page_size + return None if limit >= unlimited else limit + + def swap_limit(self): + """Return a proven swap-only byte limit, or None when v2 allows 'max'.""" + if self.version == 2: + name = "memory.swap.max" + value = self._text(name).strip() + return None if value == "max" else _integer(value, self.path / name) + # The v1 combined RAM+swap limit does not prove zero swapped pages. + # Linux cannot swap when the host has no active swap devices or files. + swaps = _read(Path("/proc/swaps")).splitlines() + if swaps and swaps[0].split() == ["Filename", "Type", "Size", "Used", "Priority"] and not any( + line.strip() for line in swaps[1:]): + return 0 + raise CgroupError("cgroup v1 cannot verify zero swap with active or unreadable host swap") + + def memory_charge(self): + name = "memory.usage_in_bytes" if self.version == 1 else "memory.current" + return self._number(name) + + def memory_peak(self): + """Peak is a report-only kernel counter.""" + name = "memory.max_usage_in_bytes" if self.version == 1 else "memory.peak" + try: + return self._number(name) + except CgroupError: + return None + + def cpu_stat(self): + return self._text("cpu.stat") + + @cache + def _require_v1_leaf(self): + # A v1 counter can be local. A leaf has the same process scope as + # the hierarchical v2 counter, including forked submission processes. + try: + if any(path.is_dir() for path in self.path.iterdir()): + raise CgroupError(f"Cannot establish container-wide v1 OOM counts in {self.path} with child cgroups") + except OSError as error: + raise CgroupError(f"Cannot inspect the v1 memory cgroup {self.path}: {error}") from error + + def v1_limit_hits(self): + # v1 may reject a combined memory+swap charge before charging RAM. + # Kernels without swap accounting omit the memsw counter entirely. + try: + combined = (self.path / "memory.memsw.failcnt").read_text(encoding="ascii") + except FileNotFoundError: + combined = None + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read memory.memsw.failcnt: {error}") from error + return [self._number("memory.failcnt"), + None if combined is None else _integer(combined.strip(), self.path / "memory.memsw.failcnt")] + + def memory_events(self, *required): + name = "memory.oom_control" if self.version == 1 else "memory.events" + if self.version == 1: + if "oom" in required: + raise CgroupError(f"{self.path / name} does not expose a cumulative oom event count") + self._require_v1_leaf() + raw = self._text(name) + counters = {} + for line in raw.splitlines(): + fields = line.split() + if len(fields) != 2 or fields[0] in counters: + raise CgroupError(f"Malformed cgroup events in {self.path / name}") + key, value = fields + counters[key] = _integer(value, self.path / name) + for key in required: + if key not in counters: + raise CgroupError(f"Missing {key} counter in {self.path / name}") + return raw, counters + + +def write_resource_error(directory, error): + """Record why a verifier could not measure a resource-constrained task.""" + path = Path(directory) / "cgroup-status.json" + path.write_text(json.dumps({"measurement_valid": False, "failure_kind": "cgroup_environment", + "error": str(error)}) + "\n") diff --git a/tasks/memory-constrained-dense-retrieval/tests/grader.py b/tasks/memory-constrained-dense-retrieval/tests/grader.py index abea04e..aeea081 100644 --- a/tasks/memory-constrained-dense-retrieval/tests/grader.py +++ b/tasks/memory-constrained-dense-retrieval/tests/grader.py @@ -16,6 +16,8 @@ from pathlib import Path from typing import Any +from cgroup_resources import CgroupError, controller, write_resource_error + TASK_DATA = Path("/task/data") PRIVATE_DATA = Path("/tests/data") @@ -458,11 +460,20 @@ def reap(): def resource_snapshot() -> dict[str, Any]: - """Container-wide cgroup v2 accounting includes service and descendants.""" - root = Path("/sys/fs/cgroup") - snapshot = {} - for name in ("memory.max", "memory.swap.max", "memory.peak", "memory.events", "cpu.max"): - snapshot[name] = (root / name).read_text().strip() + """Container-wide accounting includes the service and its descendants.""" + memory = controller("memory") + quota, period = controller("cpu").cpu_quota() + maximum, swap, peak = memory.memory_limit(), memory.swap_limit(), memory.memory_peak() + events, _ = memory.memory_events("oom_kill") + snapshot = { + "memory.max": "max" if maximum is None else str(maximum), + "memory.swap.max": "max" if swap is None else str(swap), + "memory.peak": None if peak is None else str(peak), + "memory.events": events.strip(), + "cpu.max": f"{quota if quota is not None else 'max'} {period}", + } + if memory.version == 1: + snapshot.update(cgroup_version=1, memory_events_source="memory.oom_control") return snapshot @@ -872,7 +883,7 @@ def main() -> int: resources_before = resource_snapshot() report["resources_before"] = resources_before if resources_before["memory.max"] != str(MEMORY_LIMIT_BYTES) or resources_before["memory.swap.max"] != "0": - raise VerificationError("verifier requires a 512 MiB cgroup memory limit and no swap") + raise CgroupError("verifier requires a 512 MiB cgroup memory limit and no swap") prepare_workdirs() check_submission_boundary() report['query_cpu_boundary'] = validate_query_cpu_boundary() @@ -989,6 +1000,9 @@ def main() -> int: "queries": results, }) except Exception as error: + if isinstance(error, CgroupError): + write_resource_error(RESULTS_DIR.parent, error) + report.update(measurement_valid=False, failure_kind="cgroup_environment") report["errors"].append(str(error)) finally: try: @@ -998,7 +1012,9 @@ def main() -> int: report["errors"].append(f"cannot confirm submission cleanup: {error}") try: report["resources_final"] = resource_snapshot() - except OSError as error: + except CgroupError as error: + write_resource_error(RESULTS_DIR.parent, error) + report.update(valid=False, score=0.0, reward=0.0, measurement_valid=False, failure_kind="cgroup_environment") report["errors"].append(f"cannot record final cgroup resources: {error}") write_json(REPORT_PATH, report) diff --git a/tasks/memory-constrained-dense-retrieval/tests/test.sh b/tasks/memory-constrained-dense-retrieval/tests/test.sh index 187cdc9..d2aae26 100755 --- a/tasks/memory-constrained-dense-retrieval/tests/test.sh +++ b/tasks/memory-constrained-dense-retrieval/tests/test.sh @@ -66,7 +66,7 @@ if [[ -d /logs/agent && ! -L /logs/agent ]]; then fi fi -rm -f /logs/verifier/reward.json \ +rm -f /logs/verifier/cgroup-status.json /logs/verifier/reward.json \ /logs/verifier/reward.txt \ /logs/verifier/reward-details.json \ "$RESULTS_DIR/evaluation.json" diff --git a/tasks/training-error-tracing/authoring/tests/test_runtime_limits.py b/tasks/training-error-tracing/authoring/tests/test_runtime_limits.py index d9830ee..26d5f60 100644 --- a/tasks/training-error-tracing/authoring/tests/test_runtime_limits.py +++ b/tasks/training-error-tracing/authoring/tests/test_runtime_limits.py @@ -12,14 +12,16 @@ from unittest.mock import patch import grade +from cgroup_resources import Controller class RuntimeLimits(unittest.TestCase): def profile(self, gpu="GPU-assigned, NVIDIA A100-SXM4-40GB, 40960\n", **overrides): limits = {"cpu.max": "1600000 100000", "memory.max": str(16 * 1024**3), - "memory.swap.max": "0"} + "memory.swap.max": "0", "memory.events": "oom_kill 0"} limits.update(overrides) - with patch.object(Path, "read_text", lambda path: limits[path.name]), \ + with patch.object(grade, "controller", return_value=Controller(2, Path("/sys/fs/cgroup"))), \ + patch.object(Path, "read_text", lambda path, **_: limits[path.name]), \ patch.object(grade.subprocess, "check_output", return_value=gpu): return grade.verify_runtime_limits() @@ -55,13 +57,14 @@ def test_preserves_cpu_memory_and_swap_limits(self): for limits in ({"cpu.max": "max 100000"}, {"cpu.max": "800000 100000"}, {"memory.max": "max"}, {"memory.max": str(32 * 1024**3)}, {"memory.swap.max": "1"}): - with self.subTest(limits=limits), self.assertRaises(ValueError): + with self.subTest(limits=limits), self.assertRaises(grade.CgroupError): self.profile(**limits) def test_gpu_probe_failure_is_an_infrastructure_error(self): limits = {"cpu.max": "1600000 100000", "memory.max": str(16 * 1024**3), - "memory.swap.max": "0"} - with patch.object(Path, "read_text", lambda path: limits[path.name]), \ + "memory.swap.max": "0", "memory.events": "oom_kill 0"} + with patch.object(grade, "controller", return_value=Controller(2, Path("/sys/fs/cgroup"))), \ + patch.object(Path, "read_text", lambda path, **_: limits[path.name]), \ patch.object(grade.subprocess, "check_output", side_effect=FileNotFoundError), \ self.assertRaises(FileNotFoundError): grade.verify_runtime_limits() diff --git a/tasks/training-error-tracing/tests/cgroup_resources.py b/tasks/training-error-tracing/tests/cgroup_resources.py new file mode 100644 index 0000000..ef60a48 --- /dev/null +++ b/tasks/training-error-tracing/tests/cgroup_resources.py @@ -0,0 +1,205 @@ +"""Read Linux cgroup counters for the current process and its container. + +Canonical source is environments/cgroups/cgroup_resources.py. Task image +contexts contain identical copies so they can be built independently. +""" + +from dataclasses import dataclass +from functools import cache +import json +import os +from pathlib import Path +import re + + +class CgroupError(RuntimeError): + """The environment cannot establish the task's required resource budget.""" + + +def _read(path): + try: + return path.read_text(encoding="ascii") + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + +def _integer(value, path, minimum=0): + try: + result = int(value) + if result >= minimum: + return result + except ValueError: + pass + raise CgroupError(f"Invalid cgroup value in {path}: {value!r}") + + +def _mount_path(value): + return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + + +@cache +def controller(name): + """Resolve a controller using this process's membership and visible mounts.""" + return resolve_controller(name, Path("/proc/self/cgroup")) + + +def resolve_controller(name, membership_file): + """Resolve a process's membership against mounts visible to the reader.""" + memberships = {} + for line in _read(membership_file).splitlines(): + fields = line.split(":", 2) + if len(fields) != 3: + raise CgroupError(f"Invalid cgroup membership: {line!r}") + _, controllers, location = fields + for key in controllers.split(","): + memberships[key] = Path(location) + # A hybrid host can mount v2 alongside active v1 controllers. + version = 1 if name in memberships else 2 + member = memberships.get(name if version == 1 else "") + if member is None or not member.is_absolute() or ".." in member.parts: + raise CgroupError(f"No visible cgroup membership for the {name} controller") + candidates = [] + for line in _read(Path("/proc/self/mountinfo")).splitlines(): + before, _, after = line.partition(" - ") + mount, filesystem = before.split(), after.split() + if len(mount) < 6 or len(filesystem) < 3: + continue + if version == 1: + if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + continue + elif filesystem[0] != "cgroup2": + continue + root, target = map(_mount_path, mount[3:5]) + if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): + candidates.append((len(root.parts), target / member.relative_to(root))) + if not candidates: + raise CgroupError(f"No cgroup v{version} mount covers {name} membership {member}") + _, directory = max(candidates) + return Controller(version, directory) + + +@dataclass(frozen=True) +class Controller: + version: int + path: Path + + def _text(self, name): + return _read(self.path / name) + + def _number(self, name, minimum=0): + return _integer(self._text(name).strip(), self.path / name, minimum) + + def cpu_quota(self): + """Return (quota in microseconds or None for unlimited, period in microseconds).""" + if self.version == 1: + quota = self._number("cpu.cfs_quota_us", -1) + period = self._number("cpu.cfs_period_us", 1) + if quota == 0: + raise CgroupError(f"Invalid zero CPU quota in {self.path / 'cpu.cfs_quota_us'}") + return (None if quota == -1 else quota), period + path = self.path / "cpu.max" + fields = _read(path).split() + if len(fields) != 2: + raise CgroupError(f"Expected quota and period in {path}") + quota, period = fields + return (None if quota == "max" else _integer(quota, path, 1)), _integer(period, path, 1) + + def memory_limit(self): + if self.version == 2: + value = self._text("memory.max").strip() + return None if value == "max" else _integer(value, self.path / "memory.max", 1) + # memory.stat includes the tightest ancestor even in a private cgroup namespace. + path = self.path / "memory.stat" + bounds = {} + for line in _read(path).splitlines(): + fields = line.split() + if fields and fields[0] in ("hierarchical_memory_limit", "hierarchical_memsw_limit"): + if len(fields) != 2 or fields[0] in bounds: + raise CgroupError(f"Malformed {fields[0]} in {path}") + bounds[fields[0]] = _integer(fields[1], path) + if "hierarchical_memory_limit" not in bounds: + raise CgroupError(f"Missing hierarchical_memory_limit in {path}") + if "hierarchical_memsw_limit" not in bounds and (self.path / "memory.memsw.limit_in_bytes").exists(): + raise CgroupError(f"Missing hierarchical_memsw_limit in {path}") + limit = min(self._number("memory.limit_in_bytes"), *bounds.values()) + # v1 uses page-rounded LONG_MAX to represent an unbounded limit. + page_size = os.sysconf("SC_PAGE_SIZE") + unlimited = (2**63 - 1) // page_size * page_size + return None if limit >= unlimited else limit + + def swap_limit(self): + """Return a proven swap-only byte limit, or None when v2 allows 'max'.""" + if self.version == 2: + name = "memory.swap.max" + value = self._text(name).strip() + return None if value == "max" else _integer(value, self.path / name) + # The v1 combined RAM+swap limit does not prove zero swapped pages. + # Linux cannot swap when the host has no active swap devices or files. + swaps = _read(Path("/proc/swaps")).splitlines() + if swaps and swaps[0].split() == ["Filename", "Type", "Size", "Used", "Priority"] and not any( + line.strip() for line in swaps[1:]): + return 0 + raise CgroupError("cgroup v1 cannot verify zero swap with active or unreadable host swap") + + def memory_charge(self): + name = "memory.usage_in_bytes" if self.version == 1 else "memory.current" + return self._number(name) + + def memory_peak(self): + """Peak is a report-only kernel counter.""" + name = "memory.max_usage_in_bytes" if self.version == 1 else "memory.peak" + try: + return self._number(name) + except CgroupError: + return None + + def cpu_stat(self): + return self._text("cpu.stat") + + @cache + def _require_v1_leaf(self): + # A v1 counter can be local. A leaf has the same process scope as + # the hierarchical v2 counter, including forked submission processes. + try: + if any(path.is_dir() for path in self.path.iterdir()): + raise CgroupError(f"Cannot establish container-wide v1 OOM counts in {self.path} with child cgroups") + except OSError as error: + raise CgroupError(f"Cannot inspect the v1 memory cgroup {self.path}: {error}") from error + + def v1_limit_hits(self): + # v1 may reject a combined memory+swap charge before charging RAM. + # Kernels without swap accounting omit the memsw counter entirely. + try: + combined = (self.path / "memory.memsw.failcnt").read_text(encoding="ascii") + except FileNotFoundError: + combined = None + except (OSError, UnicodeError) as error: + raise CgroupError(f"Cannot read memory.memsw.failcnt: {error}") from error + return [self._number("memory.failcnt"), + None if combined is None else _integer(combined.strip(), self.path / "memory.memsw.failcnt")] + + def memory_events(self, *required): + name = "memory.oom_control" if self.version == 1 else "memory.events" + if self.version == 1: + if "oom" in required: + raise CgroupError(f"{self.path / name} does not expose a cumulative oom event count") + self._require_v1_leaf() + raw = self._text(name) + counters = {} + for line in raw.splitlines(): + fields = line.split() + if len(fields) != 2 or fields[0] in counters: + raise CgroupError(f"Malformed cgroup events in {self.path / name}") + key, value = fields + counters[key] = _integer(value, self.path / name) + for key in required: + if key not in counters: + raise CgroupError(f"Missing {key} counter in {self.path / name}") + return raw, counters + + +def write_resource_error(directory, error): + """Record why a verifier could not measure a resource-constrained task.""" + path = Path(directory) / "cgroup-status.json" + path.write_text(json.dumps({"measurement_valid": False, "failure_kind": "cgroup_environment", + "error": str(error)}) + "\n") diff --git a/tasks/training-error-tracing/tests/efficiency_grade.py b/tasks/training-error-tracing/tests/efficiency_grade.py index c28b97c..307f94c 100644 --- a/tasks/training-error-tracing/tests/efficiency_grade.py +++ b/tasks/training-error-tracing/tests/efficiency_grade.py @@ -8,6 +8,7 @@ from types import SimpleNamespace import grade +from cgroup_resources import CgroupError, write_resource_error from efficiency_metrics import efficiency_score, latency_metric, quality_gate @@ -26,6 +27,8 @@ def run_phase(args, config, name, submission, *, control=False): phase.update(prepare_timeout_sec=300, query_timeout_sec=5) status = grade.main(SimpleNamespace(**options), phase) report = json.loads((options["logs"] / "evaluation.json").read_text()) + if report.get("infrastructure_exception") == CgroupError.__name__: + raise CgroupError(f"{name}: {report['infrastructure_message']}") if status or report["infrastructure_error"]: raise RuntimeError(f"{name} infrastructure failure: {report.get('infrastructure_message')}") return report @@ -86,6 +89,8 @@ def main(): control_time, control_memory, reference) result.update(**score, normalized_score_defined=True, reward=score["score"]/100) except Exception as error: + if isinstance(error, CgroupError): + write_resource_error(args.logs, error) result.update(infrastructure_error=True, reward=0., infrastructure_exception=type(error).__name__, infrastructure_message=str(error)) finally: diff --git a/tasks/training-error-tracing/tests/grade.py b/tasks/training-error-tracing/tests/grade.py index df771c6..da4f7e3 100644 --- a/tasks/training-error-tracing/tests/grade.py +++ b/tasks/training-error-tracing/tests/grade.py @@ -19,6 +19,7 @@ import time from pathlib import Path +from cgroup_resources import CgroupError, controller, write_resource_error from metrics import macro_precision, normalized_score, query_precision from storage_accounting import inventory, total_bytes, require_inspection_capabilities @@ -29,14 +30,14 @@ def verify_runtime_limits(): """Reject an incorrectly provisioned verifier instead of scoring its solver.""" - cgroup = Path("/sys/fs/cgroup") - quota, period = (cgroup / "cpu.max").read_text().split() - if quota == "max" or int(quota)/int(period) != 16: - raise ValueError("Verifier must enforce exactly 16 CPU cores of quota") - memory = (cgroup / "memory.max").read_text().strip() - swap = (cgroup / "memory.swap.max").read_text().strip() - if memory == "max" or int(memory) != 16*1024**3 or swap != "0": - raise ValueError("Verifier must enforce 16 GiB RAM and no swap") + quota, period = controller("cpu").cpu_quota() + if quota != 16 * period: + raise CgroupError("Verifier must enforce exactly 16 CPU cores of quota") + cgroup = controller("memory") + memory = cgroup.memory_limit() + if memory != 16 << 30 or cgroup.swap_limit() != 0: + raise CgroupError("Verifier must enforce 16 GiB RAM and no swap") + cgroup.memory_events("oom_kill") raw = subprocess.check_output(["nvidia-smi", "--query-gpu=uuid,name,memory.total", "--format=csv,noheader,nounits"], text=True, timeout=10) rows = list(csv.reader(raw.splitlines())) @@ -231,10 +232,7 @@ def bounded_log(pipe, path): def memory_oom_count(): - path = Path("/sys/fs/cgroup/memory.events") - if not path.exists(): - return 0 - return int(dict(line.split() for line in path.read_text().splitlines()).get("oom_kill", 0)) + return controller("memory").memory_events("oom_kill")[1]["oom_kill"] def run_command(command, *, cwd, index, timeout, log, storage_roots, fixed_files, budget): @@ -488,6 +486,8 @@ def main(phase_args=None, phase_config=None): except InvalidSubmission as error: report["submission_error"] = str(error) except Exception as error: + if isinstance(error, CgroupError): + write_resource_error(args.logs, error) report.update(infrastructure_error=True, infrastructure_exception=type(error).__name__, infrastructure_message=str(error)) finally: @@ -498,10 +498,19 @@ def main(phase_args=None, phase_config=None): except Exception as error: report.update(infrastructure_error=True, reward=0.0, infrastructure_exception=type(error).__name__, infrastructure_message=str(error)) - for name in ("memory.peak", "cpu.stat", "memory.events"): - path = Path("/sys/fs/cgroup") / name - if path.is_file(): - report.setdefault("container_cgroup", {})[name] = path.read_text().strip() + try: + memory = controller("memory") + peak = memory.memory_peak() + report["container_cgroup"] = { + "memory.peak": None if peak is None else str(peak), + "memory.events" if memory.version == 2 else "memory.oom_control": + memory.memory_events("oom_kill")[0].strip(), + } + report["container_cgroup"]["cpu.stat"] = controller("cpu").cpu_stat().strip() + except CgroupError: + # The required resources were checked before execution and after + # each workload. A missing diagnostic must not mask that outcome. + pass write_json(args.logs / "evaluation.json", report) # An independent integrity judge must accept the trajectory before a # production finalizer can turn this intermediate result into a reward. diff --git a/tasks/training-error-tracing/tests/test.sh b/tasks/training-error-tracing/tests/test.sh index 91048d1..fbef97f 100755 --- a/tasks/training-error-tracing/tests/test.sh +++ b/tasks/training-error-tracing/tests/test.sh @@ -5,7 +5,7 @@ mkdir -p /logs/verifier chmod 0700 /logs/verifier printf '0\n' > /logs/verifier/reward.txt printf '{"tracing_improvement":0,"jailbreak_judge":0,"reward":0}\n' > /logs/verifier/reward.json -rm -f /logs/verifier/evaluation.json /logs/verifier/reward-details.json +rm -f /logs/verifier/cgroup-status.json /logs/verifier/evaluation.json /logs/verifier/reward-details.json # Stage the small public sources for the submission UID without chmod on host binds. prepare_public_inputs() { diff --git a/tests/unit/test_cgroup_resources.py b/tests/unit/test_cgroup_resources.py new file mode 100644 index 0000000..d2d4448 --- /dev/null +++ b/tests/unit/test_cgroup_resources.py @@ -0,0 +1,266 @@ +"""Exercise cgroup layouts, evidence requirements and image-context copies.""" + +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from environments.cgroups import cgroup_resources as cg, v1_oom + + +ROOT = Path(__file__).resolve().parents[2] + + +class CgroupResourcesTests(unittest.TestCase): + def setUp(self): + self.path = Path(self.enterContext(tempfile.TemporaryDirectory())) + cg.controller.cache_clear() + cg.Controller._require_v1_leaf.cache_clear() + self.addCleanup(cg.controller.cache_clear) + self.addCleanup(cg.Controller._require_v1_leaf.cache_clear) + + def put(self, **files): + for name, value in files.items(): + (self.path / name).write_text(value + "\n") + + def resolve(self, membership, mountinfo, *names): + original = cg._read + + def read(path): + if path == Path("/proc/self/cgroup"): + return membership + if path == Path("/proc/self/mountinfo"): + return mountinfo + return original(path) + + with patch.object(cg, "_read", side_effect=read) as reads: + result = [cg.controller(name) for name in names] + self.assertIs(cg.controller(names[0]), result[0]) + self.assertEqual(reads.call_count, 2 * len(names)) + return result + + def test_v2_resolves_current_member_and_prefers_more_specific_mount(self): + mounts = ("24 1 0:18 / /sys/fs/cgroup ro - cgroup2 cgroup ro\n" + f"35 1 0:18 /slice {self.path} ro - cgroup2 cgroup ro") + memory, = self.resolve("0::/slice/task.scope", mounts, "memory") + self.assertEqual(memory, cg.Controller(2, self.path / "task.scope")) + + def test_v1_resolves_legacy_and_hybrid_memberships_per_controller(self): + mounts = ("17 1 0:12 / /sys/fs/cgroup/unified ro - cgroup2 cgroup rw\n" + "21 1 0:16 / /sys/fs/cgroup/cpu,cpuacct ro - cgroup cgroup rw,cpu,cpuacct\n" + "22 1 0:17 /docker/isolated /mnt/mem\\040space ro - cgroup cgroup rw,memory") + cpu, memory = self.resolve("4:cpu,cpuacct:/docker/isolated\n" + "5:memory:/docker/isolated\n0::/unified", mounts, "cpu", "memory") + self.assertEqual(cpu, cg.Controller(1, Path("/sys/fs/cgroup/cpu,cpuacct/docker/isolated"))) + self.assertEqual(memory, cg.Controller(1, Path("/mnt/mem space"))) + + def test_mount_for_a_different_group_is_not_a_fallback(self): + mounts = "22 1 0:17 /docker/other /sys/fs/cgroup/memory ro - cgroup cgroup rw,memory" + with self.assertRaisesRegex(cg.CgroupError, "No cgroup v1 mount covers memory"): + self.resolve("5:memory:/docker/isolated", mounts, "memory") + + def test_v2_budget_current_peak_and_oom_are_separate(self): + self.put(**{"cpu.max": "1600000 100000", "memory.max": str(16 << 30), + "memory.swap.max": "0", "memory.current": str(120 << 20), + "memory.peak": str(200 << 20), "memory.events": "oom 4\noom_kill 1"}) + controller = cg.Controller(2, self.path) + self.assertEqual(controller.cpu_quota(), (1600000, 100000)) + self.assertEqual(controller.memory_limit(), 16 << 30) + self.assertEqual(controller.swap_limit(), 0) + self.assertEqual(controller.memory_charge(), 120 << 20) + self.assertEqual(controller.memory_peak(), 200 << 20) + self.assertEqual(controller.memory_events("oom", "oom_kill")[1], {"oom": 4, "oom_kill": 1}) + + def test_v1_aggregate_counters_and_no_swap_evidence(self): + self.put(**{"cpu.cfs_quota_us": "1600000", "cpu.cfs_period_us": "100000", + "memory.limit_in_bytes": str(16 << 30), "memory.memsw.limit_in_bytes": str(16 << 30), + "memory.stat": f"hierarchical_memory_limit {16 << 30}\nhierarchical_memsw_limit {16 << 30}", + "memory.usage_in_bytes": str(120 << 20), "memory.max_usage_in_bytes": str(200 << 20), + "memory.oom_control": "oom_kill_disable 0\nunder_oom 0\noom_kill 1", + "memory.failcnt": "4"}) + controller = cg.Controller(1, self.path) + self.assertEqual(controller.cpu_quota(), (1600000, 100000)) + self.assertEqual(controller.memory_limit(), 16 << 30) + self.assertEqual(controller.memory_charge(), 120 << 20) + self.assertEqual(controller.memory_peak(), 200 << 20) + self.assertEqual(controller.memory_events("oom_kill")[1]["oom_kill"], 1) + with self.assertRaisesRegex(cg.CgroupError, "cumulative oom event count"): + controller.memory_events("oom") + no_swap = "Filename Type Size Used Priority\n" + with patch.object(cg, "_read", return_value=no_swap) as read: + self.assertEqual(controller.swap_limit(), 0) + read.assert_called_once_with(Path("/proc/swaps")) + with patch.object(cg, "_read", return_value=no_swap + "/swap file 512 0 -2\n"): + with self.assertRaisesRegex(cg.CgroupError, "zero swap"): + controller.swap_limit() + + def test_v1_local_kill_count_cannot_cover_child_cgroups(self): + (self.path / "nested").mkdir() + self.put(**{"memory.oom_control": "oom_kill 0"}) + with self.assertRaisesRegex(cg.CgroupError, "child cgroups"): + cg.Controller(1, self.path).memory_events("oom_kill") + + def test_missing_diagnostics_and_unverifiable_limits(self): + controller = cg.Controller(2, self.path) + self.assertIsNone(controller.memory_peak()) + with self.assertRaisesRegex(cg.CgroupError, "memory.events"): + controller.memory_events("oom_kill") + self.put(**{"memory.events": "oom 1", "cpu.max": "max 100000", + "memory.max": "max", "memory.swap.max": "max"}) + self.assertEqual(controller.cpu_quota(), (None, 100000)) + self.assertIsNone(controller.memory_limit()) + self.assertIsNone(controller.swap_limit()) + with self.assertRaisesRegex(cg.CgroupError, "Missing oom_kill"): + controller.memory_events("oom_kill") + self.put(**{"cpu.max": "1600000 0", "memory.max": "broken", "memory.events": "oom_kill -1"}) + for read in (controller.cpu_quota, controller.memory_limit, controller.memory_events): + with self.subTest(read=read), self.assertRaises(cg.CgroupError): + read() + with patch.object(Path, "read_text", side_effect=PermissionError("denied")): + self.assertIsNone(controller.memory_peak()) + with self.assertRaisesRegex(cg.CgroupError, "denied"): + controller.memory_limit() + + def test_v1_unlimited_sentinels(self): + page = os.sysconf("SC_PAGE_SIZE") + unlimited = (2**63 - 1) // page * page + self.put(**{"memory.limit_in_bytes": str(unlimited), + "memory.stat": f"hierarchical_memory_limit {unlimited}", + "cpu.cfs_quota_us": "-1", "cpu.cfs_period_us": "100000"}) + controller = cg.Controller(1, self.path) + self.assertIsNone(controller.memory_limit()) + self.assertEqual(controller.cpu_quota(), (None, 100000)) + + def test_v1_effective_limit_includes_ancestors_and_combined_memory(self): + memory = cg.Controller(1, self.path) + self.put(**{"memory.limit_in_bytes": str(32 << 30)}) + for ram, combined, effective in ((16, None, 16), (32, 24, 24), (32, 64, 32)): + with self.subTest(ram=ram, combined=combined): + counters = [f"hierarchical_memory_limit {ram << 30}"] + if combined is not None: + self.put(**{"memory.memsw.limit_in_bytes": str(64 << 30)}) + counters.append(f"hierarchical_memsw_limit {combined << 30}") + self.put(**{"memory.stat": "\n".join(counters)}) + self.assertEqual(memory.memory_limit(), effective << 30) + for stats in ("rss 123", "hierarchical_memory_limit bad", "hierarchical_memory_limit 1", + "hierarchical_memory_limit 1\nhierarchical_memsw_limit 2\nhierarchical_memsw_limit 2"): + with self.subTest(stats=stats), self.assertRaises(cg.CgroupError): + self.put(**{"memory.stat": stats}) + memory.memory_limit() + + def test_environment_failure_marker(self): + cg.write_resource_error(self.path, cg.CgroupError("Missing memory evidence")) + status = json.loads((self.path / "cgroup-status.json").read_text()) + self.assertIs(status["measurement_valid"], False) + self.assertEqual(status["failure_kind"], "cgroup_environment") + self.assertEqual(status["error"], "Missing memory evidence") + + def test_v1_oom_interval_distinguishes_reclaim_local_oom_and_global_kill(self): + def sample(hits, kills=0, events=0, parent=0, initial=0): + return {"limit_hits": [0, hits], "oom_kill": kills, "observer": { + "id": "watch", "events": events, "parent_events": parent, "initial_hits": [0, initial]}} + start = sample(1, events=1, initial=1) + # An earlier local OOM is outside Bimetric's interval but counts for Formula. + self.assertFalse(v1_oom.violation(sample(5, events=1, initial=1), start, include_kills=False)) + self.assertTrue(v1_oom.violation(start, include_kills=True)) + self.assertTrue(v1_oom.violation(sample(5, events=2), start, include_kills=False)) + global_kill = sample(0, kills=1) + self.assertFalse(v1_oom.violation(global_kill, sample(0), include_kills=False)) + self.assertTrue(v1_oom.violation(global_kill, include_kills=True)) + with self.assertRaises(cg.CgroupError): + v1_oom.violation(sample(5, events=1, parent=1), sample(0), include_kills=False) + with self.assertRaises(cg.CgroupError): + v1_oom.violation(sample(5, initial=1), include_kills=True) + + def test_v1_without_observer_uses_only_provable_outcomes(self): + memory = cg.Controller(1, self.path) + self.put(**{"memory.failcnt": "0", "memory.oom_control": "oom_kill 0"}) + with patch.object(v1_oom, "SOCKET", str(self.path / "missing.sock")): + start = v1_oom.snapshot(memory) + self.assertEqual(start, {"limit_hits": [0, None], "oom_kill": 0, "observer": None}) + self.assertFalse(v1_oom.violation(start, include_kills=True)) + self.put(**{"memory.failcnt": "12"}) + with self.assertRaisesRegex(cg.CgroupError, "reclaim"): + v1_oom.violation(v1_oom.snapshot(memory), start, include_kills=False) + self.put(**{"memory.oom_control": "oom_kill 1"}) + self.assertTrue(v1_oom.violation(v1_oom.snapshot(memory), include_kills=True)) + self.put(**{"memory.memsw.failcnt": "invalid"}) + with self.assertRaises(cg.CgroupError): + v1_oom.snapshot(memory) + + def test_v1_invalid_evidence_and_discontinuous_watch(self): + watch = {"id": "first", "events": 0, "parent_events": 0, "initial_hits": [0, 0]} + sample = {"limit_hits": [2, 0], "oom_kill": 0, "observer": watch} + invalid = [None, {}, {**sample, "oom_kill": -1}, {**sample, "limit_hits": [None, None]}, + {**sample, "observer": {"id": "truncated"}}] + for record in invalid: + with self.subTest(record=record), self.assertRaises(cg.CgroupError): + v1_oom.violation(record, include_kills=True) + with self.assertRaises(cg.CgroupError): + v1_oom.violation({**sample, "limit_hits": [3, 0], "observer": {**watch, "id": "second"}}, + sample, include_kills=False) + with self.assertRaises(cg.CgroupError): + v1_oom.violation({**sample, "limit_hits": [1, 0]}, sample, include_kills=False) + + def test_v1_monitor_packaged_for_both_required_task_phases(self): + import yaml + root = ROOT / "tasks" + for context in ("budgeted-bimetric-retrieval/tests", "large-scale-formula-retrieval/environment", + "large-scale-formula-retrieval/tests"): + with self.subTest(context=context): + package = root / context + self.assertEqual((package / "v1_oom.py").read_bytes(), Path(v1_oom.__file__).read_bytes()) + compose = yaml.safe_load((package / "docker-compose.yaml").read_text()) + self.assertIs(compose["x-search-swe"]["v1_oom_notifications"], True) + + def test_docker_context_copies_match_canonical_source(self): + canonical = Path(cg.__file__).read_bytes() + copies = [ + f"{task}/{phase}/runtime" for task in ("budgeted-candidate-reranking", "constrained-molecular-search") + for phase in ("environment", "tests") + ] + [ + "large-scale-formula-retrieval/environment", "large-scale-formula-retrieval/tests", + "budgeted-bimetric-retrieval/tests", "memory-constrained-dense-retrieval/tests", + "training-error-tracing/tests", + ] + for context in copies: + with self.subTest(context=context): + self.assertEqual((ROOT / "tasks" / context / "cgroup_resources.py").read_bytes(), canonical) + + +class V1OomLauncherTests(unittest.IsolatedAsyncioTestCase): + async def test_mount_lifetime_and_v2_native_path(self): + from unittest.mock import AsyncMock + from scripts.evaluation import harbor_environments as envs + + for version, fail, opt_in in [(1, False, True), (1, True, True), (2, False, True), (1, False, False)]: + with self.subTest(version=version, fail=fail, opt_in=opt_in), tempfile.TemporaryDirectory() as temp: + env = object.__new__(envs.GpuDocker) + env.environment_dir = Path(temp) + env._mounts = [] + config = "x-search-swe: {v1_oom_notifications: true}" if opt_in else "services: {}" + (env.environment_dir / "docker-compose.yaml").write_text(config) + async def start(_force_build): + self.assertEqual(len(env._mounts), int(version == 1 and opt_in)) + if env._mounts: + mount, = env._mounts + self.assertTrue(mount["read_only"]) + path = Path(mount["source"]) + self.assertTrue((path / "oom.sock").is_socket()) + self.socket_path = path + if fail: + raise RuntimeError("build failed") + with patch.object(envs, "controller", return_value=cg.Controller(version, env.environment_dir)), \ + patch.object(envs.DockerEnvironment, "start", new=AsyncMock(side_effect=start)), \ + patch.object(envs.DockerEnvironment, "stop", new=AsyncMock()): + if fail: + with self.assertRaisesRegex(RuntimeError, "build failed"): + await env.start(False) + else: + await env.start(False) + await env.stop(True) + self.assertFalse(env._mounts) + if version == 1 and opt_in: + self.assertFalse(self.socket_path.exists()) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index cff37ce..c4873e1 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -21,7 +21,7 @@ from search_swe.configuration import Setting, read_env, save_env, check_config_destination, saved_setting, resolve_setting from search_swe.repository import discover, find_repo from search_swe.resources import Pool -from search_swe.runner import doctor, run_batch +from search_swe.runner import doctor, run_batch, trial_outcome class CliTests(unittest.TestCase): @@ -220,6 +220,18 @@ def test_explicit_task_cpu_binding_is_respected(self): with self.assertRaises(ValueError): pool.validate([task], {"FIXED_CPUSET": "6-7"}) + def harbor_result(self): + name = self.tasks[0].name + task = {"path": str(self.tasks[0].path)} + return { + "task_name": name, "trial_name": "trial", "trial_uri": "file:///trial", + "task_id": task, "task_checksum": "test", "config": {"task": task, "agent": {"name": "codex"}}, + "agent_info": {"name": "codex", "version": "test"}, + "agent_execution": {"started_at": "2026-11-18T11:00:00Z", "finished_at": "2026-11-18T12:00:00Z"}, + "verifier": {"started_at": "2026-11-18T12:01:00Z", "finished_at": "2026-11-18T12:02:00Z"}, + "finished_at": "2026-11-18T12:03:00Z", "verifier_result": {"rewards": {"reward": 0}}, + } + def fake_launcher(self, *, fail=False, cancel=False): (self.root / "scripts/evaluation/run_task.py").write_text( 'import json, os, signal, sys, time\nfrom pathlib import Path\n' @@ -230,6 +242,9 @@ def fake_launcher(self, *, fail=False, cancel=False): 'start=time.time(); time.sleep(0.4)\n' + ('os.kill(os.getppid(), signal.SIGINT); time.sleep(5)\n' if cancel else '') + '(output/"observed.json").write_text(json.dumps({"start":start,"end":time.time(),"cpus":cpus}))\n' + + 'trial=output/"job"/(task+"__fixture"); trial.mkdir(parents=True)\n' + + f'result={self.harbor_result()!r}; result.update(task_name=task, trial_name=trial.name)\n' + + '(trial/"result.json").write_text(json.dumps(result))\n' + ('raise SystemExit(7 if task=="first-search" else 0)\n' if fail else '')) def test_parallel_batch_overlaps_and_queues_without_cpu_overlap(self): @@ -263,6 +278,75 @@ def test_sequential_failure_is_recorded_and_other_tasks_continue(self): second = json.loads((output / "second-search/observed.json").read_text()) self.assertGreaterEqual(second["start"], first["end"]) + def test_cgroup_evidence_failure_marks_batch_invalid_even_when_harbor_exits_zero(self): + self.fake_launcher() + launcher = self.root / "scripts/evaluation/run_task.py" + with launcher.open("a") as script: + script.write( + 'verifier=trial/"verifier"; verifier.mkdir()\n' + '(verifier/"cgroup-status.json").write_text(json.dumps({"measurement_valid": False}))\n' + ) + output = self.root / "batch" + pool = Pool({0, 1}, 8192, []) + with redirect_stdout(io.StringIO()): + code = run_batch(self.root, self.tasks[:1], Setting(), dict(os.environ), pool, 1, output) + row = json.loads((output / "run.json").read_text())["tasks"]["first-search"] + self.assertEqual(code, 1) + self.assertEqual(row["returncode"], 0) + self.assertEqual(row["status"], "invalid-measurement") + self.assertEqual(row["failure_kind"], "cgroup_environment") + + def test_harbor_healthcheck_failure_before_verifier_fails_batch(self): + self.fake_launcher() + launcher = self.root / "scripts/evaluation/run_task.py" + with launcher.open("a") as script: + script.write( + 'result.update(verifier=None, verifier_result=None, agent_execution=None, exception_info={' + '"exception_type":"HealthcheckError","exception_message":"private diagnostics",' + '"exception_traceback":"trace","occurred_at":result["finished_at"]})\n' + '(trial/"result.json").write_text(json.dumps(result))\n' + ) + output = self.root / "batch" + with redirect_stdout(io.StringIO()): + code = run_batch(self.root, self.tasks[:1], Setting(), dict(os.environ), Pool({0, 1}, 8192, []), 1, output) + summary = (output / "run.json").read_text() + row = json.loads(summary)["tasks"][self.tasks[0].name] + self.assertEqual((code, row["returncode"], row["status"]), (1, 0, "failed")) + self.assertEqual(row["exception_type"], "HealthcheckError") + self.assertNotIn("private diagnostics", summary) + + def test_harbor_trial_requires_finished_verification_and_preserves_graded_agent_timeouts(self): + trial = self.root / "job/trial" + trial.mkdir(parents=True) + path = trial / "result.json" + (trial.parent / "result.json").write_text('{"stats": {"n_errored_trials": 1}}') + self.assertEqual(trial_outcome(self.root, 0)["status"], "failed") + path.write_text("{") + self.assertEqual(trial_outcome(self.root, 0)["status"], "failed") + result = self.harbor_result() + for override in ({"finished_at": None}, {"verifier": None}, {"verifier_result": None}, + {"verifier_result": {"rewards": {}}}, {"verifier_result": {"rewards": {"reward": True}}}, + {"verifier_result": {"rewards": {"reward": float("nan")}}}, + {"verifier_result": {"rewards": {"reward": -1}}}, {"verifier_result": {"rewards": {"reward": 5}}}, + {"verifier_result": {"rewards": {"latency_ms": 4.3}}}): + with self.subTest(override=override): + path.write_text(json.dumps({**result, **override})) + self.assertEqual(trial_outcome(self.root, 0)["status"], "failed") + exception = {"exception_type": "AgentTimeoutError", "exception_traceback": "", "exception_message": "", + "occurred_at": "2026-11-18T12:00:01Z"} + for info, expected in ((None, "completed"), (exception, "completed"), + ({**exception, "exception_type": "VerifierTimeoutError", + "occurred_at": result["finished_at"]}, "failed")): + with self.subTest(info=info): + path.write_text(json.dumps({**result, "exception_info": info})) + self.assertEqual(trial_outcome(self.root, 0)["status"], expected) + marker = trial / "verifier/cgroup-status.json" + marker.parent.mkdir() + marker.write_text("truncated") + self.assertEqual(trial_outcome(self.root, 0)["status"], "invalid-measurement") + path.unlink() + self.assertEqual(trial_outcome(self.root, 0)["status"], "invalid-measurement") + def test_cancellation_releases_allocations_and_records_not_started_tasks(self): self.fake_launcher(cancel=True) output = self.root / "batch" diff --git a/tests/unit/test_submission_java.py b/tests/unit/test_submission_java.py index be4794c..5acf69f 100644 --- a/tests/unit/test_submission_java.py +++ b/tests/unit/test_submission_java.py @@ -9,6 +9,8 @@ import unittest from unittest.mock import Mock, patch +from environments.cgroups.cgroup_resources import CgroupError + ROOT = Path(__file__).resolve().parents[2] JAVA_HOME = "/opt/java/openjdk" @@ -88,7 +90,7 @@ def test_budgeted_reranking_public_and_verifier_query_environments(self): if isinstance(node, ast.FunctionDef) and node.name == "run_query") namespace = {"Path": Mock(), "shutil": Mock(), "isolation": Mock(UID=10001), "os": Mock(sched_getaffinity=Mock(return_value={0, 1})), - "time": time, "subprocess": Mock(PIPE=-1)} + "time": time, "subprocess": Mock(PIPE=-1), "CgroupError": CgroupError} spawn = namespace["subprocess"].Popen spawn.side_effect = RuntimeError("fixture stopped before launch") # Exercise the actual query launcher without /work mutations, UID @@ -108,6 +110,9 @@ def test_budgeted_reranking_public_and_verifier_query_environments(self): self.assertEqual(environment["HOME"], "/work") self.assertNotIn("OPENAI_API_KEY", environment) self.assertNotIn("VERIFIER_OPENAI_API_KEY", environment) + spawn.side_effect = CgroupError("Missing aggregate memory evidence") + with self.assertRaises(CgroupError): + namespace["run_query"]({"query_id": "fixture"}, output, None, output, Mock()) if __name__ == "__main__": From 09cc69447f3c08126e8d3a02dee4291f2447b2a6 Mon Sep 17 00:00:00 2001 From: hanhainebula <2512674094@qq.com> Date: Thu, 19 Nov 2026 19:38:10 +0800 Subject: [PATCH 2/5] Reject invalid verifier measurements and scope v1 OOM checks to the trial --- docs/evaluation/cli.md | 5 +-- environments/cgroups/v1_oom.py | 5 +-- src/search_swe/regrade.py | 12 ++----- src/search_swe/runner.py | 15 ++++++++ .../tests/v1_oom.py | 5 +-- .../environment/v1_oom.py | 5 +-- .../tests/development.py | 34 +++++++++++++------ .../tests/test_development.py | 9 ++--- .../tests/v1_oom.py | 5 +-- tests/unit/test_cgroup_resources.py | 7 ++++ tests/unit/test_cli.py | 23 +++++++++++++ 11 files changed, 91 insertions(+), 34 deletions(-) diff --git a/docs/evaluation/cli.md b/docs/evaluation/cli.md index 1918900..87ecd9c 100644 --- a/docs/evaluation/cli.md +++ b/docs/evaluation/cli.md @@ -237,8 +237,9 @@ Each batch creates `jobs/evaluations//` or a new `--output` director `completed` requires a successful launch, a finished Harbor trial and verifier, and a primary `reward` between 0 and 1 inclusive. A graded zero counts as completed, including when the agent timed out and the verifier scored its work. A setup failure or missing -trial result marks the task `failed`. Invalid cgroup evidence marks it -`invalid-measurement`. Both statuses give the batch a nonzero exit code. +trial result marks the task `failed`. Invalid resource evidence or exhausted +answer-judge retries mark it `invalid-measurement`. Both statuses give the +batch a nonzero exit code. Ctrl-C/SIGTERM cancels active runs and lets storage teardown finish. Cleanup failure stops the batch, cancels other active runs, diff --git a/environments/cgroups/v1_oom.py b/environments/cgroups/v1_oom.py index da33623..3a387ef 100644 --- a/environments/cgroups/v1_oom.py +++ b/environments/cgroups/v1_oom.py @@ -74,11 +74,12 @@ def violation(sample, baseline=None, *, include_kills): watch = sample["observer"] if watch and (baseline is None or (old_watch and old_watch["id"] == watch["id"])): start = old_watch["events"] if old_watch else 0 - if watch["events"] < start or (old_watch and watch["parent_events"] < old_watch["parent_events"]): + parent_start = old_watch["parent_events"] if old_watch else 0 + if watch["events"] < start or watch["parent_events"] < parent_start: raise CgroupError("Reset v1 OOM notifications") # Ancestor OOMs notify descendants too. A plain eventfd cannot # attribute a notification if an ancestor was also under pressure. - if watch["parent_events"] == 0: + if watch["parent_events"] == parent_start: if watch["events"] > start: return True if old_watch is not None or not any(watch["initial_hits"]): diff --git a/src/search_swe/regrade.py b/src/search_swe/regrade.py index 6497d30..82d8fe8 100644 --- a/src/search_swe/regrade.py +++ b/src/search_swe/regrade.py @@ -94,24 +94,16 @@ def walk_error(error): def measurement_valid(verifier, task_name): cgroup_valid = runner.cgroup_measurement_valid(verifier / "cgroup-status.json") - if cgroup_valid is False: + if cgroup_valid is False or runner.answer_judging_failed(verifier, task_name): return False - evaluation = f"{task_name.rsplit('/', 1)[-1]}-eval/evaluation.json" health = [] if cgroup_valid is None else [cgroup_valid] - for name in ("reward-details.json", "judge-status.json", evaluation): + for name in ("reward-details.json", "judge-status.json"): try: data = json.loads((verifier / name).read_text()) except (OSError, ValueError): continue if not isinstance(data, dict): continue - if name == evaluation: - # The primary grader records failed answer judgments separately - # from the final gate's trajectory judge status. - errors = data.get("judge_error_count") - if data.get("valid") is False and type(errors) is int and errors > 0: - return False - continue guard = data.get("timeout_guard") phase = guard.get("failed_phase") if isinstance(guard, dict) else None if isinstance(phase, str) and phase.strip(): diff --git a/src/search_swe/runner.py b/src/search_swe/runner.py index 55e1ed2..205b54b 100644 --- a/src/search_swe/runner.py +++ b/src/search_swe/runner.py @@ -141,6 +141,19 @@ def cgroup_measurement_valid(path): return isinstance(status, dict) and status.get("measurement_valid") is True +def answer_judging_failed(verifier, task_name): + name = task_name.rsplit("/", 1)[-1] + evaluation = verifier / f"{name}-eval/evaluation.json" + try: + report = json.loads(evaluation.read_text()) + except (OSError, ValueError): + return False + if not isinstance(report, dict): + return False + errors = report.get("judge_error_count") + return report.get("valid") is False and type(errors) is int and errors > 0 + + def trial_outcome(output, returncode): from harbor.models.trial.result import TrialResult @@ -183,6 +196,8 @@ def trial_outcome(output, returncode): if not (rewards and "reward" in rewards and 0 <= rewards["reward"] <= 1 and all(isinstance(value, int) or math.isfinite(value) for value in rewards.values())): return {**failure, "failure_kind": "invalid_reward"} + if answer_judging_failed(path.parent / "verifier", trial.task_name): + return {**failure, "status": "invalid-measurement", "failure_kind": "answer_judging"} return {"status": "completed", "trial_result": str(path)} diff --git a/tasks/budgeted-bimetric-retrieval/tests/v1_oom.py b/tasks/budgeted-bimetric-retrieval/tests/v1_oom.py index da33623..3a387ef 100644 --- a/tasks/budgeted-bimetric-retrieval/tests/v1_oom.py +++ b/tasks/budgeted-bimetric-retrieval/tests/v1_oom.py @@ -74,11 +74,12 @@ def violation(sample, baseline=None, *, include_kills): watch = sample["observer"] if watch and (baseline is None or (old_watch and old_watch["id"] == watch["id"])): start = old_watch["events"] if old_watch else 0 - if watch["events"] < start or (old_watch and watch["parent_events"] < old_watch["parent_events"]): + parent_start = old_watch["parent_events"] if old_watch else 0 + if watch["events"] < start or watch["parent_events"] < parent_start: raise CgroupError("Reset v1 OOM notifications") # Ancestor OOMs notify descendants too. A plain eventfd cannot # attribute a notification if an ancestor was also under pressure. - if watch["parent_events"] == 0: + if watch["parent_events"] == parent_start: if watch["events"] > start: return True if old_watch is not None or not any(watch["initial_hits"]): diff --git a/tasks/large-scale-formula-retrieval/environment/v1_oom.py b/tasks/large-scale-formula-retrieval/environment/v1_oom.py index da33623..3a387ef 100644 --- a/tasks/large-scale-formula-retrieval/environment/v1_oom.py +++ b/tasks/large-scale-formula-retrieval/environment/v1_oom.py @@ -74,11 +74,12 @@ def violation(sample, baseline=None, *, include_kills): watch = sample["observer"] if watch and (baseline is None or (old_watch and old_watch["id"] == watch["id"])): start = old_watch["events"] if old_watch else 0 - if watch["events"] < start or (old_watch and watch["parent_events"] < old_watch["parent_events"]): + parent_start = old_watch["parent_events"] if old_watch else 0 + if watch["events"] < start or watch["parent_events"] < parent_start: raise CgroupError("Reset v1 OOM notifications") # Ancestor OOMs notify descendants too. A plain eventfd cannot # attribute a notification if an ancestor was also under pressure. - if watch["parent_events"] == 0: + if watch["parent_events"] == parent_start: if watch["events"] > start: return True if old_watch is not None or not any(watch["initial_hits"]): diff --git a/tasks/large-scale-formula-retrieval/tests/development.py b/tasks/large-scale-formula-retrieval/tests/development.py index 58a1c32..c53e9f9 100644 --- a/tasks/large-scale-formula-retrieval/tests/development.py +++ b/tasks/large-scale-formula-retrieval/tests/development.py @@ -12,18 +12,32 @@ def check_development(evidence=Path('/task-evidence')): - if evidence.stat().st_mode & 0o077: - raise RuntimeError('Development evidence must be private to the host/verifier owner') - record = json.loads((evidence / 'agent.json').read_text()) + try: + if evidence.stat().st_mode & 0o077: + raise CgroupError('Development evidence must be private to the host/verifier owner') + record = json.loads((evidence / 'agent.json').read_text()) + if not isinstance(record, dict): + raise ValueError('expected a JSON object') + except (OSError, ValueError) as exc: + raise CgroupError(f'Cannot read development memory evidence: {exc}') from exc if 'cgroup_v1_memory' in record: if 'memory_events' in record: raise CgroupError('Conflicting development memory evidence for v1 and v2') exceeded = v1_oom.violation(record['cgroup_v1_memory'], include_kills=True) return record, 'development memory limit' if exceeded else None - events = dict(line.split() for line in record['memory_events'].splitlines()) - counts = [int(events[name]) for name in ['oom', 'oom_kill']] - if any(count < 0 for count in counts): - raise ValueError('Invalid development memory event count') - if any(counts): - return record, 'development memory limit' - return record, None + raw = record.get('memory_events') + if not isinstance(raw, str): + raise CgroupError('Development evidence must include memory_events') + events = {} + try: + for line in raw.splitlines(): + key, value = line.split() + count = int(value) + if key in events or count < 0: + raise ValueError(f'duplicate or negative {key} counter') + events[key] = count + exceeded = events['oom'] > 0 + killed = events['oom_kill'] > 0 + except (KeyError, ValueError) as exc: + raise CgroupError(f'Invalid development memory events: {exc}') from exc + return record, 'development memory limit' if exceeded or killed else None diff --git a/tasks/large-scale-formula-retrieval/tests/test_development.py b/tasks/large-scale-formula-retrieval/tests/test_development.py index 931b6ea..b3b1628 100644 --- a/tasks/large-scale-formula-retrieval/tests/test_development.py +++ b/tasks/large-scale-formula-retrieval/tests/test_development.py @@ -60,7 +60,7 @@ def test_v1_frozen_evidence(self): def test_missing_memory_evidence_is_an_error(self): (self.evidence / 'agent.json').unlink() - with self.assertRaises(FileNotFoundError): + with self.assertRaises(CgroupError): check_development(self.evidence) def test_malformed_memory_evidence_is_an_error(self): @@ -68,10 +68,11 @@ def test_malformed_memory_evidence_is_an_error(self): '{"memory_events": ""}', '{"memory_events": "oom 0"}', '{"memory_events": "oom_kill 0"}', '{"memory_events": "oom unknown\\noom_kill 0"}', - '{"memory_events": "oom -1\\noom_kill 0"}']: + '{"memory_events": "oom -1\\noom_kill 0"}', + '{"memory_events": "oom 0\\noom 1\\noom_kill 0"}', + '[]', 'null']: (self.evidence / 'agent.json').write_text(content) - with self.subTest(content=content), self.assertRaises( - (ValueError, KeyError, AttributeError)): + with self.subTest(content=content), self.assertRaises(CgroupError): check_development(self.evidence) def test_evidence_privacy(self): diff --git a/tasks/large-scale-formula-retrieval/tests/v1_oom.py b/tasks/large-scale-formula-retrieval/tests/v1_oom.py index da33623..3a387ef 100644 --- a/tasks/large-scale-formula-retrieval/tests/v1_oom.py +++ b/tasks/large-scale-formula-retrieval/tests/v1_oom.py @@ -74,11 +74,12 @@ def violation(sample, baseline=None, *, include_kills): watch = sample["observer"] if watch and (baseline is None or (old_watch and old_watch["id"] == watch["id"])): start = old_watch["events"] if old_watch else 0 - if watch["events"] < start or (old_watch and watch["parent_events"] < old_watch["parent_events"]): + parent_start = old_watch["parent_events"] if old_watch else 0 + if watch["events"] < start or watch["parent_events"] < parent_start: raise CgroupError("Reset v1 OOM notifications") # Ancestor OOMs notify descendants too. A plain eventfd cannot # attribute a notification if an ancestor was also under pressure. - if watch["parent_events"] == 0: + if watch["parent_events"] == parent_start: if watch["events"] > start: return True if old_watch is not None or not any(watch["initial_hits"]): diff --git a/tests/unit/test_cgroup_resources.py b/tests/unit/test_cgroup_resources.py index d2d4448..32a632a 100644 --- a/tests/unit/test_cgroup_resources.py +++ b/tests/unit/test_cgroup_resources.py @@ -164,6 +164,13 @@ def sample(hits, kills=0, events=0, parent=0, initial=0): start = sample(1, events=1, initial=1) # An earlier local OOM is outside Bimetric's interval but counts for Formula. self.assertFalse(v1_oom.violation(sample(5, events=1, initial=1), start, include_kills=False)) + before = sample(0, events=1, parent=1) + self.assertFalse(v1_oom.violation(sample(3, events=1, parent=1), before, include_kills=False)) + self.assertTrue(v1_oom.violation(sample(3, events=2, parent=1), before, include_kills=False)) + with self.assertRaises(cg.CgroupError): + v1_oom.violation(sample(3, events=2, parent=2), before, include_kills=False) + with self.assertRaises(cg.CgroupError): + v1_oom.violation(sample(3, events=1, parent=1), include_kills=True) self.assertTrue(v1_oom.violation(start, include_kills=True)) self.assertTrue(v1_oom.violation(sample(5, events=2), start, include_kills=False)) global_kill = sample(0, kills=1) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index c4873e1..8c7a5a4 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -315,6 +315,29 @@ def test_harbor_healthcheck_failure_before_verifier_fails_batch(self): self.assertEqual(row["exception_type"], "HealthcheckError") self.assertNotIn("private diagnostics", summary) + def test_batch_distinguishes_exhausted_answer_judging_from_a_graded_zero(self): + for details, expected in (({"valid": False, "judge_error_count": 1}, "invalid-measurement"), + ({"valid": False, "judge_error_count": 0}, "completed"), + ({"valid": True, "judge_error_count": 0}, "completed")): + with self.subTest(details=details), tempfile.TemporaryDirectory() as temp: + self.fake_launcher() + launcher = self.root / "scripts/evaluation/run_task.py" + with launcher.open("a") as script: + script.write( + 'result["task_name"]="search-swe/scientific-paper-qa"\n' + '(trial/"result.json").write_text(json.dumps(result))\n' + 'evaluation=trial/"verifier/scientific-paper-qa-eval"\n' + 'evaluation.mkdir(parents=True)\n' + f'(evaluation/"evaluation.json").write_text(json.dumps({details!r}))\n' + ) + output = Path(temp) / "batch" + with redirect_stdout(io.StringIO()): + code = run_batch(self.root, self.tasks[:1], Setting(), dict(os.environ), + Pool({0, 1}, 8192, []), 1, output) + row = json.loads((output / "run.json").read_text())["tasks"][self.tasks[0].name] + self.assertEqual(row["status"], expected) + self.assertEqual(code, 0 if expected == "completed" else 1) + def test_harbor_trial_requires_finished_verification_and_preserves_graded_agent_timeouts(self): trial = self.root / "job/trial" trial.mkdir(parents=True) From 12119e1849df6beefbec294a8ee72a00aa6e1860 Mon Sep 17 00:00:00 2001 From: hanhainebula <2512674094@qq.com> Date: Thu, 19 Nov 2026 20:19:17 +0800 Subject: [PATCH 3/5] Share verifier measurement checks between batch runs and regrades --- docs/evaluation/cli.md | 8 ++-- src/search_swe/measurement.py | 76 +++++++++++++++++++++++++++++++++++ src/search_swe/regrade.py | 30 +------------- src/search_swe/runner.py | 30 ++------------ tests/unit/test_cli.py | 57 ++++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 57 deletions(-) create mode 100644 src/search_swe/measurement.py diff --git a/docs/evaluation/cli.md b/docs/evaluation/cli.md index 87ecd9c..17e2773 100644 --- a/docs/evaluation/cli.md +++ b/docs/evaluation/cli.md @@ -237,9 +237,11 @@ Each batch creates `jobs/evaluations//` or a new `--output` director `completed` requires a successful launch, a finished Harbor trial and verifier, and a primary `reward` between 0 and 1 inclusive. A graded zero counts as completed, including when the agent timed out and the verifier scored its work. A setup failure or missing -trial result marks the task `failed`. Invalid resource evidence or exhausted -answer-judge retries mark it `invalid-measurement`. Both statuses give the -batch a nonzero exit code. +trial result marks the task `failed`. Batch runs and regrades use the same verifier +diagnostics. A verifier timeout, failed answer or trajectory judging, or invalid +resource evidence marks the task `invalid-measurement`. A completed audit may +reject the submission and award a valid zero. Failed and invalid measurements +give the batch a nonzero exit code. Ctrl-C/SIGTERM cancels active runs and lets storage teardown finish. Cleanup failure stops the batch, cancels other active runs, diff --git a/src/search_swe/measurement.py b/src/search_swe/measurement.py new file mode 100644 index 0000000..004ba3e --- /dev/null +++ b/src/search_swe/measurement.py @@ -0,0 +1,76 @@ +"""Interpret verifier diagnostics for both batch runs and regrades.""" + +import json + + +def _report(path): + try: + value = json.loads(path.read_text()) + except (OSError, ValueError): + return {} + return value if isinstance(value, dict) else {} + + +def cgroup_measurement_valid(path): + """Return None for no diagnostic and False for invalid or unreadable evidence.""" + try: + status = json.loads(path.read_text()) + except FileNotFoundError: + return None + except (OSError, ValueError): + return False + return isinstance(status, dict) and status.get("measurement_valid") is True + + +def _multilingual_submission_zero(verifier, status, reward): + # This finalizer marks every nonzero primary exit invalid, including the + # collector rejecting a submission that violates the task's file contract. + if (reward != 0 or status.get("healthy") is not True or status.get("exit_status") != "0" + or status.get("primary_exit_status") != "1"): + return False + report = _report(verifier / "report.json") + if report.get("valid") is not False or report.get("reward") != 0: + return False + # Match the verifier's complete cause chain. A failed copy, missing host + # metadata or an unrecognized error still represents a failed measurement. + prefix = ("ContractViolation: Invalid frozen submission: Collected artifact restoration failed: " + "Invalid frozen submission: ") + failure = report.get("failure") + return isinstance(failure, str) and failure.startswith(prefix) and failure[len(prefix):] in { + "Submission root must be a real directory", "Linked or non-directory submission entry", + "Linked or non-regular submission entry", "Missing transform.py", + "Submission exceeds the total storage limit", "100 GiB total task storage exceeded", + } + + +def verifier_measurement(verifier, task_name, reward): + """Return (valid, failure kind). None means the verifier gave no health signal.""" + valid = cgroup_measurement_valid(verifier / "cgroup-status.json") + if valid is False: + return False, "cgroup_environment" + task = task_name.rsplit("/", 1)[-1] + evaluation = _report(verifier / f"{task}-eval/evaluation.json") + errors = evaluation.get("judge_error_count") + if evaluation.get("valid") is False and type(errors) is int and errors > 0: + return False, "answer_judging" + + for filename in ("reward-details.json", "judge-status.json"): + data = _report(verifier / filename) + guard = data.get("timeout_guard") + phase = guard.get("failed_phase") if isinstance(guard, dict) else None + if isinstance(phase, str) and phase.strip(): + return False, "verifier_timeout" + for details in (data, data.get("final_gate")): + if not isinstance(details, dict): + continue + flag = details.get("measurement_valid") + if flag is False: + if not (task == "multilingual-vector-postprocessing" and filename == "judge-status.json" + and details is data and _multilingual_submission_zero(verifier, data, reward)): + return False, "verifier_measurement" + valid = True + elif flag is True: + valid = True + elif details.get("evaluation_valid") is True and details.get("judge_succeeded") is False: + return False, "judge_infrastructure" + return valid, None diff --git a/src/search_swe/regrade.py b/src/search_swe/regrade.py index 82d8fe8..9586994 100644 --- a/src/search_swe/regrade.py +++ b/src/search_swe/regrade.py @@ -11,6 +11,7 @@ from . import runner, updates from .configuration import DEFAULTS, read_env +from .measurement import verifier_measurement from .repository import VARIABLE, check_compatibility, discover from .resources import Pool, cpu_ids from .submission_checks import verify_submission_keys @@ -92,33 +93,6 @@ def walk_error(error): + ", ".join(missing)) -def measurement_valid(verifier, task_name): - cgroup_valid = runner.cgroup_measurement_valid(verifier / "cgroup-status.json") - if cgroup_valid is False or runner.answer_judging_failed(verifier, task_name): - return False - health = [] if cgroup_valid is None else [cgroup_valid] - for name in ("reward-details.json", "judge-status.json"): - try: - data = json.loads((verifier / name).read_text()) - except (OSError, ValueError): - continue - if not isinstance(data, dict): - continue - guard = data.get("timeout_guard") - phase = guard.get("failed_phase") if isinstance(guard, dict) else None - if isinstance(phase, str) and phase.strip(): - return False - for details in (data, data.get("final_gate")): - if not isinstance(details, dict): - continue - if type(details.get("measurement_valid")) is bool: - health.append(details["measurement_valid"]) - elif details.get("evaluation_valid") is True and details.get("judge_succeeded") is False: - # Some finalizers write zero when the audit could not complete. - health.append(False) - return all(health) if health else None - - def result_summary(trial, report): from harbor.models.trial.result import TrialResult @@ -132,7 +106,7 @@ def result_summary(trial, report): report.update(measurement_valid=False, status="verifier-error", exception_type=result.exception_info.exception_type) elif rewards: - valid = measurement_valid(trial / "verifier", result.task_name) + valid, _ = verifier_measurement(trial / "verifier", result.task_name, rewards.get("reward")) report.update(measurement_valid=valid, status="invalid-measurement" if valid is False else "completed") diff --git a/src/search_swe/runner.py b/src/search_swe/runner.py index 205b54b..8a0c441 100644 --- a/src/search_swe/runner.py +++ b/src/search_swe/runner.py @@ -15,6 +15,7 @@ import uuid from . import __version__ +from .measurement import cgroup_measurement_valid, verifier_measurement from .resources import gpu_inventory @@ -130,30 +131,6 @@ def wait_cancelled(processes): process.wait() -def cgroup_measurement_valid(path): - """Return None for no diagnostic and False for invalid or unreadable evidence.""" - try: - status = json.loads(path.read_text()) - except FileNotFoundError: - return None - except (OSError, ValueError): - return False - return isinstance(status, dict) and status.get("measurement_valid") is True - - -def answer_judging_failed(verifier, task_name): - name = task_name.rsplit("/", 1)[-1] - evaluation = verifier / f"{name}-eval/evaluation.json" - try: - report = json.loads(evaluation.read_text()) - except (OSError, ValueError): - return False - if not isinstance(report, dict): - return False - errors = report.get("judge_error_count") - return report.get("valid") is False and type(errors) is int and errors > 0 - - def trial_outcome(output, returncode): from harbor.models.trial.result import TrialResult @@ -196,8 +173,9 @@ def trial_outcome(output, returncode): if not (rewards and "reward" in rewards and 0 <= rewards["reward"] <= 1 and all(isinstance(value, int) or math.isfinite(value) for value in rewards.values())): return {**failure, "failure_kind": "invalid_reward"} - if answer_judging_failed(path.parent / "verifier", trial.task_name): - return {**failure, "status": "invalid-measurement", "failure_kind": "answer_judging"} + valid, failure_kind = verifier_measurement(path.parent / "verifier", trial.task_name, rewards["reward"]) + if valid is False: + return {**failure, "status": "invalid-measurement", "failure_kind": failure_kind} return {"status": "completed", "trial_result": str(path)} diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 8c7a5a4..cabf38c 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -20,6 +20,7 @@ from search_swe.cli import main, parser, configure_run from search_swe.configuration import Setting, read_env, save_env, check_config_destination, saved_setting, resolve_setting from search_swe.repository import discover, find_repo +from search_swe.regrade import result_summary from search_swe.resources import Pool from search_swe.runner import doctor, run_batch, trial_outcome @@ -370,6 +371,62 @@ def test_harbor_trial_requires_finished_verification_and_preserves_graded_agent_ path.unlink() self.assertEqual(trial_outcome(self.root, 0)["status"], "invalid-measurement") + def test_batch_and_regrade_agree_on_verifier_diagnostics(self): + timeout = {"timeout_guard": {"failed_phase": "primary_evaluation", "exit_status": "124"}} + rejected = {"healthy": True, "accepted": False, "measurement_valid": True} + failed = {"healthy": False, "accepted": False, "measurement_valid": False} + for task, files, valid in ( + ("query-encoder-alignment", {"reward-details.json": timeout}, False), + ("join-aware-table-retrieval", {"judge-status.json": failed}, False), + ("join-aware-table-retrieval", {"judge-status.json": rejected}, True), + ("query-encoder-alignment", {"reward-details.json": timeout, "judge-status.json": rejected}, False), + ("fixture", {"reward-details.json": {"final_gate": {"measurement_valid": False}}}, False), + ("fixture", {"reward-details.json": {"final_gate": {"evaluation_valid": True, + "judge_succeeded": False}}}, False), + ): + with self.subTest(task=task, files=files), tempfile.TemporaryDirectory() as temp: + self.fake_launcher() + with (self.root / "scripts/evaluation/run_task.py").open("a") as script: + script.write( + f'result["task_name"]="search-swe/{task}"\n' + '(trial/"result.json").write_text(json.dumps(result))\n' + '(trial/"verifier").mkdir()\n' + ) + for name, content in files.items(): + script.write(f'(trial/"verifier"/{name!r}).write_text({json.dumps(content)!r})\n') + output = Path(temp) / "batch" + with redirect_stdout(io.StringIO()): + code = run_batch(self.root, self.tasks[:1], Setting(), dict(os.environ), + Pool({0, 1}, 8192, []), 1, output) + row = json.loads((output / "run.json").read_text())["tasks"][self.tasks[0].name] + regrade = {} + result_summary(Path(row["trial_result"]).parent, regrade) + self.assertIs(regrade["measurement_valid"], valid) + self.assertEqual(row["status"], regrade["status"]) + self.assertEqual(row["status"], "completed" if valid else "invalid-measurement") + self.assertEqual(code, 0 if valid else 1) + + def test_multilingual_collector_rejection_requires_a_healthy_audit_and_a_submission_error(self): + trial = self.root / "job/trial" + verifier = trial / "verifier" + verifier.mkdir(parents=True) + result = self.harbor_result() + result["task_name"] = "search-swe/multilingual-vector-postprocessing" + (trial / "result.json").write_text(json.dumps(result)) + prefix = ("ContractViolation: Invalid frozen submission: Collected artifact restoration failed: " + "Invalid frozen submission: ") + for reason, healthy, valid in (("Missing transform.py", True, True), ("Missing transform.py", False, False), + ("Cannot inspect submission open files or mappings", True, False)): + with self.subTest(reason=reason, healthy=healthy): + (verifier / "report.json").write_text(json.dumps({"valid": False, "reward": 0, "failure": prefix + reason})) + (verifier / "judge-status.json").write_text(json.dumps({ + "measurement_valid": False, "healthy": healthy, "primary_exit_status": "1", + "exit_status": "0" if healthy else "78"})) + regrade = {} + result_summary(trial, regrade) + self.assertIs(regrade["measurement_valid"], valid) + self.assertEqual(trial_outcome(self.root, 0)["status"], regrade["status"]) + def test_cancellation_releases_allocations_and_records_not_started_tasks(self): self.fake_launcher(cancel=True) output = self.root / "batch" From 557570f5c0e094fd01524b939956f044d2c19341 Mon Sep 17 00:00:00 2001 From: hanhainebula <2512674094@qq.com> Date: Thu, 19 Nov 2026 20:57:54 +0800 Subject: [PATCH 4/5] Distinguish verifier failures from measured submission zeros --- src/search_swe/measurement.py | 27 +++++++--- .../environment/runtime/evaluate.py | 54 +++++++++++++------ .../environment/runtime/failures.py | 20 +++++++ .../environment/runtime/freeze_submission.py | 14 ++--- .../environment/runtime/process.py | 5 +- .../environment/runtime/restore_submission.py | 4 +- .../environment/runtime/storage.py | 3 +- .../environment/runtime/worker.py | 12 +++-- .../tests/finalize.py | 5 +- .../tests/grade.py | 10 ++-- .../tests/runtime/evaluate.py | 54 +++++++++++++------ .../tests/runtime/failures.py | 20 +++++++ .../tests/runtime/freeze_submission.py | 14 ++--- .../tests/runtime/process.py | 5 +- .../tests/runtime/restore_submission.py | 4 +- .../tests/runtime/storage.py | 3 +- .../tests/runtime/worker.py | 12 +++-- .../tests/test_finalizer.py | 22 ++++++++ .../tests/test_grader_import.py | 3 ++ .../tests/test_runtime_controls.py | 18 +++++-- tests/unit/test_cli.py | 28 +++++++--- 21 files changed, 254 insertions(+), 83 deletions(-) create mode 100644 tasks/multilingual-vector-postprocessing/environment/runtime/failures.py create mode 100644 tasks/multilingual-vector-postprocessing/tests/runtime/failures.py diff --git a/src/search_swe/measurement.py b/src/search_swe/measurement.py index 004ba3e..225e4f9 100644 --- a/src/search_swe/measurement.py +++ b/src/search_swe/measurement.py @@ -22,14 +22,13 @@ def cgroup_measurement_valid(path): return isinstance(status, dict) and status.get("measurement_valid") is True -def _multilingual_submission_zero(verifier, status, reward): - # This finalizer marks every nonzero primary exit invalid, including the - # collector rejecting a submission that violates the task's file contract. +def _legacy_multilingual_submission_zero(verifier, status, reward): + # Older finalizers marked collector rejections invalid despite a healthy audit. if (reward != 0 or status.get("healthy") is not True or status.get("exit_status") != "0" or status.get("primary_exit_status") != "1"): return False report = _report(verifier / "report.json") - if report.get("valid") is not False or report.get("reward") != 0: + if "status" in report or report.get("valid") is not False or report.get("reward") != 0: return False # Match the verifier's complete cause chain. A failed copy, missing host # metadata or an unrecognized error still represents a failed measurement. @@ -54,19 +53,33 @@ def verifier_measurement(verifier, task_name, reward): if evaluation.get("valid") is False and type(errors) is int and errors > 0: return False, "answer_judging" - for filename in ("reward-details.json", "judge-status.json"): + # Some verifiers publish a standalone final gate result. + gate = _report(verifier / "gate-status.json") + jailbreak = _report(verifier / "jailbreak-status.json") + if gate.get("status") == "infrastructure_error" or jailbreak.get("status") == "error": + return False, "verifier_measurement" + if gate.get("status") == "complete" or jailbreak.get("status") == "ok": + valid = True + + for filename in ("reward-details.json", "judge-status.json", "audit-status.json"): data = _report(verifier / filename) guard = data.get("timeout_guard") phase = guard.get("failed_phase") if isinstance(guard, dict) else None if isinstance(phase, str) and phase.strip(): - return False, "verifier_timeout" + exit_status = guard.get("exit_status") + return False, "verifier_timeout" if exit_status in ("124", "137", 124, 137) else "verifier_failure" + if filename in ("judge-status.json", "audit-status.json"): + if data.get("infrastructure_error") is True or data.get("healthy") is False: + return False, "verifier_measurement" if data.get("audit_valid") is True else "judge_infrastructure" + if data.get("healthy") is True: + valid = True for details in (data, data.get("final_gate")): if not isinstance(details, dict): continue flag = details.get("measurement_valid") if flag is False: if not (task == "multilingual-vector-postprocessing" and filename == "judge-status.json" - and details is data and _multilingual_submission_zero(verifier, data, reward)): + and details is data and _legacy_multilingual_submission_zero(verifier, data, reward)): return False, "verifier_measurement" valid = True elif flag is True: diff --git a/tasks/multilingual-vector-postprocessing/environment/runtime/evaluate.py b/tasks/multilingual-vector-postprocessing/environment/runtime/evaluate.py index 087de11..5b171f2 100644 --- a/tasks/multilingual-vector-postprocessing/environment/runtime/evaluate.py +++ b/tasks/multilingual-vector-postprocessing/environment/runtime/evaluate.py @@ -10,6 +10,7 @@ import numpy as np from threadpoolctl import threadpool_limits +from failures import ContractViolation, LEGACY_COLLECTION_REJECTIONS, SubmissionViolation from metrics import aggregate, normalize, quality from process import TransformError, TransformProcess from storage import StorageBudget @@ -19,10 +20,6 @@ AUDIT_SEED = 0 -class ContractViolation(RuntimeError): - pass - - def rows(path): result = [json.loads(line) for line in Path(path).read_text().splitlines() if line.strip()] if not result: @@ -45,7 +42,13 @@ def inputs(directory, kind): def verify_snapshot(submission, status_path, *, make_read_only=True): status = json.loads(Path(status_path).read_text()) if status.get("valid") is not True: - raise ContractViolation("Invalid frozen submission: " + str(status.get("reason"))) + reason = status.get("reason") + rejected = (status.get("valid") is False and status.get("files") == [] + and (status.get("failure_kind") == "submission" + or ("failure_kind" not in status and isinstance(reason, str) + and reason in LEGACY_COLLECTION_REJECTIONS))) + error = SubmissionViolation if rejected else ContractViolation + raise error("Invalid frozen submission: " + str(reason)) submission = Path(submission) if submission.is_symlink() or not submission.is_dir(): raise ContractViolation("Linked submission root") @@ -124,6 +127,8 @@ def sample(self): def check(self): if self.storage_error is not None: + if isinstance(self.storage_error, SubmissionViolation): + raise self.storage_error raise ContractViolation(str(self.storage_error)) from self.storage_error def close(self): @@ -136,7 +141,11 @@ def close(self): def equivalent(a, b): # Permit floating-point kernel differences across batch shapes while # rejecting meaningful changes in the effective transformation. - return np.max(np.linalg.norm(normalize(a) - normalize(b), axis=1)) <= .01 + try: + actual = normalize(a) + except ValueError as error: + raise SubmissionViolation(f"Invalid vector in a consistency probe: {error}") from error + return np.max(np.linalg.norm(actual - normalize(b), axis=1)) <= .01 def evaluate_once(submission, corpus, query_root, development, output, *, sandbox=False, @@ -153,31 +162,42 @@ def evaluate_once(submission, corpus, query_root, development, output, *, sandbo audit_rng = np.random.default_rng(AUDIT_SEED) memory_events = Path("/sys/fs/cgroup/memory.events") before_events = memory_events.read_text() if memory_events.exists() else "" + + def prepare_vectors(vectors, languages, *, normalized=False): + try: + transformed = worker.transform(vectors, languages, prepare_deadline) + return normalize(transformed) if normalized else transformed + except (ValueError, TimeoutError) as error: + raise SubmissionViolation(f"Document preparation failed: {error}") from error + try: - worker = TransformProcess(submission, temporary.name, output / "worker.stderr.log", - prepare_deadline, sandbox=sandbox, require_gpu=require_gpu) + try: + worker = TransformProcess(submission, temporary.name, output / "worker.stderr.log", + prepare_deadline, sandbox=sandbox, require_gpu=require_gpu) + except (TransformError, TimeoutError) as error: + raise SubmissionViolation(f"Document preparation failed: {error}") from error for dataset in ("belebele", "xquad"): metadata, raw = inputs(Path(corpus) / dataset, "documents") converted = [] for begin in range(0, len(raw), 128): meter.check() - converted.append(normalize(worker.transform(raw[begin:begin + 128], - [r["language"] for r in metadata[begin:begin + 128]], prepare_deadline))) + converted.append(prepare_vectors(raw[begin:begin + 128], + [r["language"] for r in metadata[begin:begin + 128]], normalized=True)) converted = np.concatenate(converted) caches[dataset] = dict(metadata=metadata, raw=raw, vectors=converted, ids=np.array([r["id"] for r in metadata])) # Repeated document inputs after reordering must preserve f. selected = audit_rng.choice(len(raw), size=min(12, len(raw)), replace=False) - changed = worker.transform(raw[selected], [metadata[i]["language"] for i in selected], prepare_deadline) + changed = prepare_vectors(raw[selected], [metadata[i]["language"] for i in selected]) if not equivalent(changed, converted[selected]): - raise ContractViolation("Transform depends on document order or batch shape") + raise SubmissionViolation("Transform depends on document order or batch shape") # Warm only with public development queries. Hidden queries are not sent # to the candidate during loading or preparation. public_metadata, public_vectors = inputs(Path(development) / "belebele", "queries") for i in range(min(4, len(public_vectors))): - worker.transform(public_vectors[i:i + 1], [public_metadata[i]["language"]], prepare_deadline) + prepare_vectors(public_vectors[i:i + 1], [public_metadata[i]["language"]]) if time.monotonic() > prepare_deadline: - raise ContractViolation("Document preparation exceeded its limit") + raise SubmissionViolation("Document preparation exceeded its limit") prepared = time.monotonic() - start meter.check() queued = [] @@ -218,7 +238,7 @@ def evaluate_once(submission, corpus, query_root, development, output, *, sandbo valid = True except TransformError as error: if str(error) in {"OutOfMemoryError", "MemoryError"}: - raise ContractViolation("Submission exhausted its memory budget") from error + raise SubmissionViolation("Submission exhausted its memory budget") from error reason = "transform-error:" + str(error) except (ValueError, TimeoutError, RuntimeError, OSError) as error: reason = type(error).__name__ @@ -247,14 +267,14 @@ def evaluate_once(submission, corpus, query_root, development, output, *, sandbo worker = None else: if not equivalent(check, cache["vectors"][i:i + 1]): - raise ContractViolation("The frozen function changed during query evaluation") + raise SubmissionViolation("The frozen function changed during query evaluation") meter.check() meter.storage.check(force=True) if memory_events.exists(): def oom(text): return int(dict(line.split() for line in text.splitlines()).get("oom_kill", "0")) if oom(memory_events.read_text()) > oom(before_events): - raise ContractViolation("Container memory limit was exceeded") + raise SubmissionViolation("Container memory limit was exceeded") report = {**aggregate(records), "valid_queries": len(records) - failed, "total_queries": len(records), "consistency_probe_failures": probe_failures, "preparation_seconds": prepared, "max_query_seconds": longest, diff --git a/tasks/multilingual-vector-postprocessing/environment/runtime/failures.py b/tasks/multilingual-vector-postprocessing/environment/runtime/failures.py new file mode 100644 index 0000000..8d752d2 --- /dev/null +++ b/tasks/multilingual-vector-postprocessing/environment/runtime/failures.py @@ -0,0 +1,20 @@ +"""Distinguish submission violations from unverifiable measurements.""" + + +class ContractViolation(RuntimeError): + """A verifier input or invariant did not satisfy its contract.""" + + +class SubmissionViolation(ContractViolation): + """The submission violated a rule that awards a measured zero.""" + + +# Older collectors recorded these fixed rejection reasons without a failure kind. +LEGACY_COLLECTION_REJECTIONS = frozenset({ + "Submission root must be a real directory", + "Linked or non-directory submission entry", + "Linked or non-regular submission entry", + "Missing transform.py", + "Submission exceeds the total storage limit", + "100 GiB total task storage exceeded", +}) diff --git a/tasks/multilingual-vector-postprocessing/environment/runtime/freeze_submission.py b/tasks/multilingual-vector-postprocessing/environment/runtime/freeze_submission.py index df51b8b..8131d0d 100644 --- a/tasks/multilingual-vector-postprocessing/environment/runtime/freeze_submission.py +++ b/tasks/multilingual-vector-postprocessing/environment/runtime/freeze_submission.py @@ -12,6 +12,7 @@ import psutil sys.path.insert(0, "/task/runtime") +from failures import SubmissionViolation from storage import StorageBudget @@ -71,24 +72,24 @@ def main(): storage = StorageBudget() status["task_storage_bytes"] = storage.check(force=True) if root.is_symlink() or not root.is_dir(): - raise ValueError("Submission root must be a real directory") + raise SubmissionViolation("Submission root must be a real directory") paths = [] for parent, directories, files in os.walk(root, followlinks=False): for name in directories: path = Path(parent) / name if not stat.S_ISDIR(path.lstat().st_mode): - raise ValueError("Linked or non-directory submission entry") + raise SubmissionViolation("Linked or non-directory submission entry") for name in files: path = Path(parent) / name mode = path.lstat().st_mode if not stat.S_ISREG(mode): - raise ValueError("Linked or non-regular submission entry") + raise SubmissionViolation("Linked or non-regular submission entry") paths.append((path, mode)) if not (root / "transform.py").is_file(): - raise ValueError("Missing transform.py") + raise SubmissionViolation("Missing transform.py") total = sum(path.stat().st_size for path, mode in paths) if total > 100 * 1024**3: - raise ValueError("Submission exceeds the total storage limit") + raise SubmissionViolation("Submission exceeds the total storage limit") for path, mode in sorted(paths): with path.open("rb") as handle: digest = hashlib.file_digest(handle, "sha256").hexdigest() @@ -100,7 +101,8 @@ def main(): Path(parent).chmod(0o555) status.update(valid=True, reason=None, submitted_bytes=total) except Exception as error: - status.update(valid=False, reason=str(error), files=[]) + status.update(valid=False, reason=str(error), files=[], + failure_kind="submission" if isinstance(error, SubmissionViolation) else "infrastructure") # Still supply a real, harmless artifact to the framework, so a bad # submission is a measured zero rather than a collection-path error. if root.exists() or root.is_symlink(): diff --git a/tasks/multilingual-vector-postprocessing/environment/runtime/process.py b/tasks/multilingual-vector-postprocessing/environment/runtime/process.py index e55e3f8..afac025 100644 --- a/tasks/multilingual-vector-postprocessing/environment/runtime/process.py +++ b/tasks/multilingual-vector-postprocessing/environment/runtime/process.py @@ -53,7 +53,10 @@ def drain(): self.drainer = threading.Thread(target=drain, daemon=True) self.drainer.start() try: - if self.read(deadline) != {"ready": True}: + response = self.read(deadline) + if "error" in response: + raise TransformError(str(response["error"])) + if response != {"ready": True}: raise ValueError("Transform did not become ready") except BaseException: self.close() diff --git a/tasks/multilingual-vector-postprocessing/environment/runtime/restore_submission.py b/tasks/multilingual-vector-postprocessing/environment/runtime/restore_submission.py index e011669..aad98f6 100644 --- a/tasks/multilingual-vector-postprocessing/environment/runtime/restore_submission.py +++ b/tasks/multilingual-vector-postprocessing/environment/runtime/restore_submission.py @@ -7,6 +7,7 @@ sys.path.insert(0, "/task/runtime") from evaluate import verify_snapshot +from failures import SubmissionViolation def main(): @@ -29,7 +30,8 @@ def main(): shutil.rmtree(destination) shutil.copytree(source, destination) except Exception as error: - status = dict(valid=False, files=[], reason="Collected artifact restoration failed: " + str(error)) + status = dict(valid=False, files=[], reason="Collected artifact restoration failed: " + str(error), + failure_kind="submission" if isinstance(error, SubmissionViolation) else "infrastructure") if destination.is_symlink(): destination.unlink() elif destination.exists(): diff --git a/tasks/multilingual-vector-postprocessing/environment/runtime/storage.py b/tasks/multilingual-vector-postprocessing/environment/runtime/storage.py index e22e496..d416666 100644 --- a/tasks/multilingual-vector-postprocessing/environment/runtime/storage.py +++ b/tasks/multilingual-vector-postprocessing/environment/runtime/storage.py @@ -5,6 +5,7 @@ import stat import time +from failures import SubmissionViolation from isolation import UID @@ -156,5 +157,5 @@ def check(self, *, force=False): used = self.fixed_bytes + logical_bytes(self.work_roots, include_retained=True) self.peak_bytes = max(self.peak_bytes, used) if self.peak_bytes > self.limit: - raise RuntimeError("100 GiB total task storage exceeded") + raise SubmissionViolation("100 GiB total task storage exceeded") return self.peak_bytes diff --git a/tasks/multilingual-vector-postprocessing/environment/runtime/worker.py b/tasks/multilingual-vector-postprocessing/environment/runtime/worker.py index ceff34f..4fbe3c2 100644 --- a/tasks/multilingual-vector-postprocessing/environment/runtime/worker.py +++ b/tasks/multilingual-vector-postprocessing/environment/runtime/worker.py @@ -52,10 +52,14 @@ def main(): sys.path.insert(0, str(submission)) spec = importlib.util.spec_from_file_location("submission_transform", submission / "transform.py") module = importlib.util.module_from_spec(spec) - with contextlib.redirect_stdout(sys.stderr), torch.inference_mode(): - spec.loader.exec_module(module) - state = module.load(str(submission)) - synchronize() + try: + with contextlib.redirect_stdout(sys.stderr), torch.inference_mode(): + spec.loader.exec_module(module) + state = module.load(str(submission)) + except Exception as error: + print(json.dumps({"error": type(error).__name__}), flush=True) + raise + synchronize() print('{"ready":true}', flush=True) for line in sys.stdin: try: diff --git a/tasks/multilingual-vector-postprocessing/tests/finalize.py b/tasks/multilingual-vector-postprocessing/tests/finalize.py index 01f4229..5cef03c 100644 --- a/tasks/multilingual-vector-postprocessing/tests/finalize.py +++ b/tasks/multilingual-vector-postprocessing/tests/finalize.py @@ -64,7 +64,10 @@ def main(): TRAJECTORY.is_file() and TRAJECTORY.stat().st_size > 0, read_json(OUT/'reward-details.json')) primary_status = os.environ.get('PRIMARY_STATUS') - measurement_valid = primary_status == '0' and report.get('valid') is True and health['healthy'] + submission_failed = (report.get('status') == 'submission_failed' and report.get('valid') is False + and type(report.get('reward')) in (int, float) and report['reward'] == 0) + primary_valid = report.get('valid') is True or submission_failed + measurement_valid = primary_status == '0' and primary_valid and health['healthy'] if not measurement_valid: result['reward'] = 0. health.update(primary_exit_status=primary_status, measurement_valid=measurement_valid) diff --git a/tasks/multilingual-vector-postprocessing/tests/grade.py b/tasks/multilingual-vector-postprocessing/tests/grade.py index 698e9ff..3f3e468 100644 --- a/tasks/multilingual-vector-postprocessing/tests/grade.py +++ b/tasks/multilingual-vector-postprocessing/tests/grade.py @@ -9,6 +9,7 @@ sys.path.insert(0, "/task/runtime") from evaluate import evaluate +from failures import SubmissionViolation from metrics import reward from isolation import cleanup @@ -17,7 +18,6 @@ def main(): OUT.mkdir(parents=True, exist_ok=True) - report = {"valid": False, "reward": 0.} try: anchors = SCORING reward(0., anchors["baseline"], anchors["reference"]) @@ -26,16 +26,18 @@ def main(): frozen_status="/var/tmp/submission-freeze-status.json") if report["total_queries"] != 13002: raise RuntimeError("Fixed hidden query population is incomplete") - report.update(valid=True, baseline=anchors["baseline"], reference=anchors["reference"], + report.update(status="ok", valid=True, baseline=anchors["baseline"], reference=anchors["reference"], reward=reward(report["Q"], anchors["baseline"], anchors["reference"])) report["score_100"] = report["reward"] * 100 except Exception as error: - report = {"valid": False, "reward": 0., "failure": type(error).__name__ + ": " + str(error)} + status = "submission_failed" if isinstance(error, SubmissionViolation) else "infrastructure_error" + report = {"status": status, "valid": False, "reward": 0., + "failure": type(error).__name__ + ": " + str(error)} print(report["failure"], file=sys.stderr) finally: cleanup() (OUT / "report.json").write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") - return 0 if report["valid"] else 1 + return int(report["status"] == "infrastructure_error") if __name__ == "__main__": diff --git a/tasks/multilingual-vector-postprocessing/tests/runtime/evaluate.py b/tasks/multilingual-vector-postprocessing/tests/runtime/evaluate.py index 087de11..5b171f2 100644 --- a/tasks/multilingual-vector-postprocessing/tests/runtime/evaluate.py +++ b/tasks/multilingual-vector-postprocessing/tests/runtime/evaluate.py @@ -10,6 +10,7 @@ import numpy as np from threadpoolctl import threadpool_limits +from failures import ContractViolation, LEGACY_COLLECTION_REJECTIONS, SubmissionViolation from metrics import aggregate, normalize, quality from process import TransformError, TransformProcess from storage import StorageBudget @@ -19,10 +20,6 @@ AUDIT_SEED = 0 -class ContractViolation(RuntimeError): - pass - - def rows(path): result = [json.loads(line) for line in Path(path).read_text().splitlines() if line.strip()] if not result: @@ -45,7 +42,13 @@ def inputs(directory, kind): def verify_snapshot(submission, status_path, *, make_read_only=True): status = json.loads(Path(status_path).read_text()) if status.get("valid") is not True: - raise ContractViolation("Invalid frozen submission: " + str(status.get("reason"))) + reason = status.get("reason") + rejected = (status.get("valid") is False and status.get("files") == [] + and (status.get("failure_kind") == "submission" + or ("failure_kind" not in status and isinstance(reason, str) + and reason in LEGACY_COLLECTION_REJECTIONS))) + error = SubmissionViolation if rejected else ContractViolation + raise error("Invalid frozen submission: " + str(reason)) submission = Path(submission) if submission.is_symlink() or not submission.is_dir(): raise ContractViolation("Linked submission root") @@ -124,6 +127,8 @@ def sample(self): def check(self): if self.storage_error is not None: + if isinstance(self.storage_error, SubmissionViolation): + raise self.storage_error raise ContractViolation(str(self.storage_error)) from self.storage_error def close(self): @@ -136,7 +141,11 @@ def close(self): def equivalent(a, b): # Permit floating-point kernel differences across batch shapes while # rejecting meaningful changes in the effective transformation. - return np.max(np.linalg.norm(normalize(a) - normalize(b), axis=1)) <= .01 + try: + actual = normalize(a) + except ValueError as error: + raise SubmissionViolation(f"Invalid vector in a consistency probe: {error}") from error + return np.max(np.linalg.norm(actual - normalize(b), axis=1)) <= .01 def evaluate_once(submission, corpus, query_root, development, output, *, sandbox=False, @@ -153,31 +162,42 @@ def evaluate_once(submission, corpus, query_root, development, output, *, sandbo audit_rng = np.random.default_rng(AUDIT_SEED) memory_events = Path("/sys/fs/cgroup/memory.events") before_events = memory_events.read_text() if memory_events.exists() else "" + + def prepare_vectors(vectors, languages, *, normalized=False): + try: + transformed = worker.transform(vectors, languages, prepare_deadline) + return normalize(transformed) if normalized else transformed + except (ValueError, TimeoutError) as error: + raise SubmissionViolation(f"Document preparation failed: {error}") from error + try: - worker = TransformProcess(submission, temporary.name, output / "worker.stderr.log", - prepare_deadline, sandbox=sandbox, require_gpu=require_gpu) + try: + worker = TransformProcess(submission, temporary.name, output / "worker.stderr.log", + prepare_deadline, sandbox=sandbox, require_gpu=require_gpu) + except (TransformError, TimeoutError) as error: + raise SubmissionViolation(f"Document preparation failed: {error}") from error for dataset in ("belebele", "xquad"): metadata, raw = inputs(Path(corpus) / dataset, "documents") converted = [] for begin in range(0, len(raw), 128): meter.check() - converted.append(normalize(worker.transform(raw[begin:begin + 128], - [r["language"] for r in metadata[begin:begin + 128]], prepare_deadline))) + converted.append(prepare_vectors(raw[begin:begin + 128], + [r["language"] for r in metadata[begin:begin + 128]], normalized=True)) converted = np.concatenate(converted) caches[dataset] = dict(metadata=metadata, raw=raw, vectors=converted, ids=np.array([r["id"] for r in metadata])) # Repeated document inputs after reordering must preserve f. selected = audit_rng.choice(len(raw), size=min(12, len(raw)), replace=False) - changed = worker.transform(raw[selected], [metadata[i]["language"] for i in selected], prepare_deadline) + changed = prepare_vectors(raw[selected], [metadata[i]["language"] for i in selected]) if not equivalent(changed, converted[selected]): - raise ContractViolation("Transform depends on document order or batch shape") + raise SubmissionViolation("Transform depends on document order or batch shape") # Warm only with public development queries. Hidden queries are not sent # to the candidate during loading or preparation. public_metadata, public_vectors = inputs(Path(development) / "belebele", "queries") for i in range(min(4, len(public_vectors))): - worker.transform(public_vectors[i:i + 1], [public_metadata[i]["language"]], prepare_deadline) + prepare_vectors(public_vectors[i:i + 1], [public_metadata[i]["language"]]) if time.monotonic() > prepare_deadline: - raise ContractViolation("Document preparation exceeded its limit") + raise SubmissionViolation("Document preparation exceeded its limit") prepared = time.monotonic() - start meter.check() queued = [] @@ -218,7 +238,7 @@ def evaluate_once(submission, corpus, query_root, development, output, *, sandbo valid = True except TransformError as error: if str(error) in {"OutOfMemoryError", "MemoryError"}: - raise ContractViolation("Submission exhausted its memory budget") from error + raise SubmissionViolation("Submission exhausted its memory budget") from error reason = "transform-error:" + str(error) except (ValueError, TimeoutError, RuntimeError, OSError) as error: reason = type(error).__name__ @@ -247,14 +267,14 @@ def evaluate_once(submission, corpus, query_root, development, output, *, sandbo worker = None else: if not equivalent(check, cache["vectors"][i:i + 1]): - raise ContractViolation("The frozen function changed during query evaluation") + raise SubmissionViolation("The frozen function changed during query evaluation") meter.check() meter.storage.check(force=True) if memory_events.exists(): def oom(text): return int(dict(line.split() for line in text.splitlines()).get("oom_kill", "0")) if oom(memory_events.read_text()) > oom(before_events): - raise ContractViolation("Container memory limit was exceeded") + raise SubmissionViolation("Container memory limit was exceeded") report = {**aggregate(records), "valid_queries": len(records) - failed, "total_queries": len(records), "consistency_probe_failures": probe_failures, "preparation_seconds": prepared, "max_query_seconds": longest, diff --git a/tasks/multilingual-vector-postprocessing/tests/runtime/failures.py b/tasks/multilingual-vector-postprocessing/tests/runtime/failures.py new file mode 100644 index 0000000..8d752d2 --- /dev/null +++ b/tasks/multilingual-vector-postprocessing/tests/runtime/failures.py @@ -0,0 +1,20 @@ +"""Distinguish submission violations from unverifiable measurements.""" + + +class ContractViolation(RuntimeError): + """A verifier input or invariant did not satisfy its contract.""" + + +class SubmissionViolation(ContractViolation): + """The submission violated a rule that awards a measured zero.""" + + +# Older collectors recorded these fixed rejection reasons without a failure kind. +LEGACY_COLLECTION_REJECTIONS = frozenset({ + "Submission root must be a real directory", + "Linked or non-directory submission entry", + "Linked or non-regular submission entry", + "Missing transform.py", + "Submission exceeds the total storage limit", + "100 GiB total task storage exceeded", +}) diff --git a/tasks/multilingual-vector-postprocessing/tests/runtime/freeze_submission.py b/tasks/multilingual-vector-postprocessing/tests/runtime/freeze_submission.py index df51b8b..8131d0d 100644 --- a/tasks/multilingual-vector-postprocessing/tests/runtime/freeze_submission.py +++ b/tasks/multilingual-vector-postprocessing/tests/runtime/freeze_submission.py @@ -12,6 +12,7 @@ import psutil sys.path.insert(0, "/task/runtime") +from failures import SubmissionViolation from storage import StorageBudget @@ -71,24 +72,24 @@ def main(): storage = StorageBudget() status["task_storage_bytes"] = storage.check(force=True) if root.is_symlink() or not root.is_dir(): - raise ValueError("Submission root must be a real directory") + raise SubmissionViolation("Submission root must be a real directory") paths = [] for parent, directories, files in os.walk(root, followlinks=False): for name in directories: path = Path(parent) / name if not stat.S_ISDIR(path.lstat().st_mode): - raise ValueError("Linked or non-directory submission entry") + raise SubmissionViolation("Linked or non-directory submission entry") for name in files: path = Path(parent) / name mode = path.lstat().st_mode if not stat.S_ISREG(mode): - raise ValueError("Linked or non-regular submission entry") + raise SubmissionViolation("Linked or non-regular submission entry") paths.append((path, mode)) if not (root / "transform.py").is_file(): - raise ValueError("Missing transform.py") + raise SubmissionViolation("Missing transform.py") total = sum(path.stat().st_size for path, mode in paths) if total > 100 * 1024**3: - raise ValueError("Submission exceeds the total storage limit") + raise SubmissionViolation("Submission exceeds the total storage limit") for path, mode in sorted(paths): with path.open("rb") as handle: digest = hashlib.file_digest(handle, "sha256").hexdigest() @@ -100,7 +101,8 @@ def main(): Path(parent).chmod(0o555) status.update(valid=True, reason=None, submitted_bytes=total) except Exception as error: - status.update(valid=False, reason=str(error), files=[]) + status.update(valid=False, reason=str(error), files=[], + failure_kind="submission" if isinstance(error, SubmissionViolation) else "infrastructure") # Still supply a real, harmless artifact to the framework, so a bad # submission is a measured zero rather than a collection-path error. if root.exists() or root.is_symlink(): diff --git a/tasks/multilingual-vector-postprocessing/tests/runtime/process.py b/tasks/multilingual-vector-postprocessing/tests/runtime/process.py index e55e3f8..afac025 100644 --- a/tasks/multilingual-vector-postprocessing/tests/runtime/process.py +++ b/tasks/multilingual-vector-postprocessing/tests/runtime/process.py @@ -53,7 +53,10 @@ def drain(): self.drainer = threading.Thread(target=drain, daemon=True) self.drainer.start() try: - if self.read(deadline) != {"ready": True}: + response = self.read(deadline) + if "error" in response: + raise TransformError(str(response["error"])) + if response != {"ready": True}: raise ValueError("Transform did not become ready") except BaseException: self.close() diff --git a/tasks/multilingual-vector-postprocessing/tests/runtime/restore_submission.py b/tasks/multilingual-vector-postprocessing/tests/runtime/restore_submission.py index e011669..aad98f6 100644 --- a/tasks/multilingual-vector-postprocessing/tests/runtime/restore_submission.py +++ b/tasks/multilingual-vector-postprocessing/tests/runtime/restore_submission.py @@ -7,6 +7,7 @@ sys.path.insert(0, "/task/runtime") from evaluate import verify_snapshot +from failures import SubmissionViolation def main(): @@ -29,7 +30,8 @@ def main(): shutil.rmtree(destination) shutil.copytree(source, destination) except Exception as error: - status = dict(valid=False, files=[], reason="Collected artifact restoration failed: " + str(error)) + status = dict(valid=False, files=[], reason="Collected artifact restoration failed: " + str(error), + failure_kind="submission" if isinstance(error, SubmissionViolation) else "infrastructure") if destination.is_symlink(): destination.unlink() elif destination.exists(): diff --git a/tasks/multilingual-vector-postprocessing/tests/runtime/storage.py b/tasks/multilingual-vector-postprocessing/tests/runtime/storage.py index e22e496..d416666 100644 --- a/tasks/multilingual-vector-postprocessing/tests/runtime/storage.py +++ b/tasks/multilingual-vector-postprocessing/tests/runtime/storage.py @@ -5,6 +5,7 @@ import stat import time +from failures import SubmissionViolation from isolation import UID @@ -156,5 +157,5 @@ def check(self, *, force=False): used = self.fixed_bytes + logical_bytes(self.work_roots, include_retained=True) self.peak_bytes = max(self.peak_bytes, used) if self.peak_bytes > self.limit: - raise RuntimeError("100 GiB total task storage exceeded") + raise SubmissionViolation("100 GiB total task storage exceeded") return self.peak_bytes diff --git a/tasks/multilingual-vector-postprocessing/tests/runtime/worker.py b/tasks/multilingual-vector-postprocessing/tests/runtime/worker.py index ceff34f..4fbe3c2 100644 --- a/tasks/multilingual-vector-postprocessing/tests/runtime/worker.py +++ b/tasks/multilingual-vector-postprocessing/tests/runtime/worker.py @@ -52,10 +52,14 @@ def main(): sys.path.insert(0, str(submission)) spec = importlib.util.spec_from_file_location("submission_transform", submission / "transform.py") module = importlib.util.module_from_spec(spec) - with contextlib.redirect_stdout(sys.stderr), torch.inference_mode(): - spec.loader.exec_module(module) - state = module.load(str(submission)) - synchronize() + try: + with contextlib.redirect_stdout(sys.stderr), torch.inference_mode(): + spec.loader.exec_module(module) + state = module.load(str(submission)) + except Exception as error: + print(json.dumps({"error": type(error).__name__}), flush=True) + raise + synchronize() print('{"ready":true}', flush=True) for line in sys.stdin: try: diff --git a/tasks/multilingual-vector-postprocessing/tests/test_finalizer.py b/tasks/multilingual-vector-postprocessing/tests/test_finalizer.py index 494b014..71b73c7 100644 --- a/tasks/multilingual-vector-postprocessing/tests/test_finalizer.py +++ b/tasks/multilingual-vector-postprocessing/tests/test_finalizer.py @@ -102,6 +102,28 @@ def test_process_status_distinguishes_rejection_and_infrastructure_failure(self) self.assertEqual(finalize.main(), expected_status) self.assertEqual(float((out / 'reward.txt').read_text()), expected_reward) + def test_submission_failure_is_measured_only_with_a_healthy_audit(self): + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) + trajectory = out / 'trajectory.json' + trajectory.write_text('{"steps":[{}]}') + for name, value in [('judge.json', {'jailbreak_judge': 1}), + ('reward-details.json', details())]: + (out / name).write_text(json.dumps(value)) + for primary, judge, report_status, valid in ( + ('0', '0', 'submission_failed', True), + ('0', '78', 'submission_failed', False), + ('124', '0', 'submission_failed', False), + ('1', '0', 'infrastructure_error', False)): + with self.subTest(primary=primary, judge=judge, report_status=report_status): + (out / 'report.json').write_text(json.dumps( + {'valid': False, 'reward': 0, 'status': report_status})) + with patch.object(finalize, 'OUT', out), patch.object(finalize, 'TRAJECTORY', trajectory), \ + patch.dict(os.environ, {'PRIMARY_STATUS': primary, 'JUDGE_STATUS': judge}): + self.assertEqual(finalize.main(), int(not valid)) + self.assertIs(json.loads((out / 'judge-status.json').read_text())['measurement_valid'], valid) + self.assertEqual(float((out / 'reward.txt').read_text()), 0) + if __name__ == '__main__': unittest.main() diff --git a/tasks/multilingual-vector-postprocessing/tests/test_grader_import.py b/tasks/multilingual-vector-postprocessing/tests/test_grader_import.py index 9e03b70..229dd2d 100644 --- a/tasks/multilingual-vector-postprocessing/tests/test_grader_import.py +++ b/tasks/multilingual-vector-postprocessing/tests/test_grader_import.py @@ -18,6 +18,9 @@ def forbidden(*args, **kwargs): module = types.ModuleType(name) setattr(module, attribute, forbidden) sys.modules[name] = module +failures = types.ModuleType('failures') +failures.SubmissionViolation = type('SubmissionViolation', (RuntimeError,), {}) +sys.modules['failures'] = failures runpy.run_path(sys.argv[1], run_name='grade_import_probe') assert pathlib.Path(sys.modules['scoring_parameters'].__file__).resolve() == pathlib.Path(sys.argv[1]).with_name('scoring_parameters.py') """ diff --git a/tasks/multilingual-vector-postprocessing/tests/test_runtime_controls.py b/tasks/multilingual-vector-postprocessing/tests/test_runtime_controls.py index 5754ebd..56cdd1b 100644 --- a/tasks/multilingual-vector-postprocessing/tests/test_runtime_controls.py +++ b/tasks/multilingual-vector-postprocessing/tests/test_runtime_controls.py @@ -13,6 +13,7 @@ import psutil sys.path.insert(0, "/task/runtime") from evaluate import evaluate, verify_snapshot +from failures import SubmissionViolation ROOT = Path("/app/control-private") @@ -291,12 +292,17 @@ def test_consistency_probe_timeout_does_not_abort_all_measurements(self): def test_nan_and_zero_document_outputs_rejected(self): for expression in ("return np.full_like(vectors, np.nan)", "return np.zeros_like(vectors)"): self.module(transform=expression) - with self.assertRaises(ValueError): + with self.assertRaises(SubmissionViolation): self.run_control() def test_batch_dependent_function_rejected(self): self.module(transform="return np.roll(vectors, 1, axis=1) if len(vectors) == 1 else vectors.copy()") - with self.assertRaisesRegex(RuntimeError, "frozen function changed"): + with self.assertRaisesRegex(SubmissionViolation, "frozen function changed"): + self.run_control() + + def test_worker_load_exception_is_a_submission_failure(self): + self.module(load="raise ValueError('invalid submission weights')") + with self.assertRaisesRegex(SubmissionViolation, "Document preparation failed: ValueError"): self.run_control() def test_labels_rewards_network_and_privilege_protected(self): @@ -338,7 +344,7 @@ def test_orphan_processes_cleaned(self): def test_preparation_timeout_stops_worker(self): self.module(load="import time\ntime.sleep(60)\nreturn None") began = time.monotonic() - with self.assertRaises(TimeoutError): + with self.assertRaisesRegex(SubmissionViolation, "timed out"): evaluate("/app/submission", ROOT / "corpus", ROOT / "hidden", ROOT / "development", ROOT / "logs/preparation-timeout", sandbox=True, require_gpu=False, preparation_seconds=.5, frozen_status=ROOT / "frozen-status.json") @@ -406,7 +412,11 @@ def restore(): self.assertTrue(restore()["valid"]) self.assertEqual(self.run_control()["Q"], 1) (collected / "transform.py").write_text("tampered") - self.assertFalse(restore()["valid"]) + self.assertEqual(restore()["failure_kind"], "infrastructure") + self.assertFalse(any(Path("/app/submission").iterdir())) + (parent / "submission-freeze-status.json").write_text(json.dumps( + {"valid": False, "files": [], "reason": "Missing transform.py", "failure_kind": "submission"})) + self.assertEqual(restore()["failure_kind"], "submission") self.assertFalse(any(Path("/app/submission").iterdir())) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index cabf38c..7da2722 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -373,16 +373,28 @@ def test_harbor_trial_requires_finished_verification_and_preserves_graded_agent_ def test_batch_and_regrade_agree_on_verifier_diagnostics(self): timeout = {"timeout_guard": {"failed_phase": "primary_evaluation", "exit_status": "124"}} + primary_error = {"timeout_guard": {"failed_phase": "primary_evaluation", "exit_status": "1"}} rejected = {"healthy": True, "accepted": False, "measurement_valid": True} failed = {"healthy": False, "accepted": False, "measurement_valid": False} - for task, files, valid in ( - ("query-encoder-alignment", {"reward-details.json": timeout}, False), - ("join-aware-table-retrieval", {"judge-status.json": failed}, False), - ("join-aware-table-retrieval", {"judge-status.json": rejected}, True), - ("query-encoder-alignment", {"reward-details.json": timeout, "judge-status.json": rejected}, False), - ("fixture", {"reward-details.json": {"final_gate": {"measurement_valid": False}}}, False), + audit_failed = {"primary_valid": True, "audit_valid": False, "infrastructure_error": True, + "healthy": False, "integrity": 0} + audit_rejected = {"primary_valid": True, "audit_valid": True, "infrastructure_error": False, + "healthy": True, "integrity": 0} + for task, files, failure_kind in ( + ("query-encoder-alignment", {"reward-details.json": timeout}, "verifier_timeout"), + ("query-encoder-alignment", {"reward-details.json": primary_error}, "verifier_failure"), + ("join-aware-table-retrieval", {"judge-status.json": failed}, "judge_infrastructure"), + ("join-aware-table-retrieval", {"judge-status.json": rejected}, None), + ("large-scale-formula-retrieval", {"audit-status.json": audit_failed}, "judge_infrastructure"), + ("large-scale-formula-retrieval", {"audit-status.json": audit_rejected}, None), + ("icsi-meeting-memory", {"gate-status.json": {"status": "infrastructure_error"}}, "verifier_measurement"), + ("icsi-meeting-memory", {"gate-status.json": {"status": "complete", "reason": "invalid_submission"}}, None), + ("agentic-search", {"jailbreak-status.json": {"status": "error"}}, "verifier_measurement"), + ("agentic-search", {"jailbreak-status.json": {"status": "ok", "passed": False}}, None), + ("query-encoder-alignment", {"reward-details.json": timeout, "judge-status.json": rejected}, "verifier_timeout"), + ("fixture", {"reward-details.json": {"final_gate": {"measurement_valid": False}}}, "verifier_measurement"), ("fixture", {"reward-details.json": {"final_gate": {"evaluation_valid": True, - "judge_succeeded": False}}}, False), + "judge_succeeded": False}}}, "judge_infrastructure"), ): with self.subTest(task=task, files=files), tempfile.TemporaryDirectory() as temp: self.fake_launcher() @@ -401,9 +413,11 @@ def test_batch_and_regrade_agree_on_verifier_diagnostics(self): row = json.loads((output / "run.json").read_text())["tasks"][self.tasks[0].name] regrade = {} result_summary(Path(row["trial_result"]).parent, regrade) + valid = failure_kind is None self.assertIs(regrade["measurement_valid"], valid) self.assertEqual(row["status"], regrade["status"]) self.assertEqual(row["status"], "completed" if valid else "invalid-measurement") + self.assertEqual(row.get("failure_kind"), failure_kind) self.assertEqual(code, 0 if valid else 1) def test_multilingual_collector_rejection_requires_a_healthy_audit_and_a_submission_error(self): From 37b6412946e427a7b0f0ab7b73fe81066c7238cc Mon Sep 17 00:00:00 2001 From: hanhainebula <2512674094@qq.com> Date: Thu, 19 Nov 2026 21:37:57 +0800 Subject: [PATCH 5/5] Fix worker exit attribution, verifier diagnostics, and cgroup mount paths --- environments/cgroups/cgroup_resources.py | 34 ++++++++++++------- src/search_swe/measurement.py | 29 +++++++++++++++- .../tests/cgroup_resources.py | 34 ++++++++++++------- .../environment/runtime/cgroup_resources.py | 34 ++++++++++++------- .../tests/runtime/cgroup_resources.py | 34 ++++++++++++------- .../environment/runtime/cgroup_resources.py | 34 ++++++++++++------- .../tests/runtime/cgroup_resources.py | 34 ++++++++++++------- .../environment/cgroup_resources.py | 34 ++++++++++++------- .../tests/cgroup_resources.py | 34 ++++++++++++------- .../tests/cgroup_resources.py | 34 ++++++++++++------- .../environment/runtime/evaluate.py | 6 ++-- .../environment/runtime/process.py | 15 ++++++-- .../environment/runtime/worker.py | 2 ++ .../tests/runtime/evaluate.py | 6 ++-- .../tests/runtime/process.py | 15 ++++++-- .../tests/runtime/worker.py | 2 ++ .../tests/test_runtime_controls.py | 11 ++++-- .../tests/cgroup_resources.py | 34 ++++++++++++------- tests/unit/test_cgroup_resources.py | 16 ++++++--- tests/unit/test_cli.py | 24 +++++++++++-- 20 files changed, 325 insertions(+), 141 deletions(-) diff --git a/environments/cgroups/cgroup_resources.py b/environments/cgroups/cgroup_resources.py index ef60a48..9758926 100644 --- a/environments/cgroups/cgroup_resources.py +++ b/environments/cgroups/cgroup_resources.py @@ -23,6 +23,14 @@ def _read(path): raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error +def _read_proc(path): + # Proc path fields contain filesystem bytes, including non-UTF-8 names. + try: + return path.read_bytes() + except OSError as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + def _integer(value, path, minimum=0): try: result = int(value) @@ -34,7 +42,8 @@ def _integer(value, path, minimum=0): def _mount_path(value): - return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + unescaped = re.sub(rb"\\([0-3][0-7]{2})", lambda m: bytes([int(m[1], 8)]), value) + return Path(os.fsdecode(unescaped)) @cache @@ -45,29 +54,30 @@ def controller(name): def resolve_controller(name, membership_file): """Resolve a process's membership against mounts visible to the reader.""" + key_name = name.encode("ascii") memberships = {} - for line in _read(membership_file).splitlines(): - fields = line.split(":", 2) + for line in _read_proc(membership_file).rstrip(b"\n").split(b"\n"): + fields = line.split(b":", 2) if len(fields) != 3: raise CgroupError(f"Invalid cgroup membership: {line!r}") _, controllers, location = fields - for key in controllers.split(","): - memberships[key] = Path(location) + for key in controllers.split(b","): + memberships[key] = Path(os.fsdecode(location)) # A hybrid host can mount v2 alongside active v1 controllers. - version = 1 if name in memberships else 2 - member = memberships.get(name if version == 1 else "") + version = 1 if key_name in memberships else 2 + member = memberships.get(key_name if version == 1 else b"") if member is None or not member.is_absolute() or ".." in member.parts: raise CgroupError(f"No visible cgroup membership for the {name} controller") candidates = [] - for line in _read(Path("/proc/self/mountinfo")).splitlines(): - before, _, after = line.partition(" - ") - mount, filesystem = before.split(), after.split() + for line in _read_proc(Path("/proc/self/mountinfo")).split(b"\n"): + before, _, after = line.partition(b" - ") + mount, filesystem = before.split(b" "), after.split(b" ") if len(mount) < 6 or len(filesystem) < 3: continue if version == 1: - if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + if filesystem[0] != b"cgroup" or key_name not in filesystem[2].split(b","): continue - elif filesystem[0] != "cgroup2": + elif filesystem[0] != b"cgroup2": continue root, target = map(_mount_path, mount[3:5]) if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): diff --git a/src/search_swe/measurement.py b/src/search_swe/measurement.py index 225e4f9..7c85fb4 100644 --- a/src/search_swe/measurement.py +++ b/src/search_swe/measurement.py @@ -42,12 +42,36 @@ def _legacy_multilingual_submission_zero(verifier, status, reward): } +def _legacy_primary_timeout(verifier, task): + if task not in {"instruction-following-embedding", "protein-text-bidirectional-retrieval"}: + return False + try: + status = (verifier / "status.txt").read_text().strip() + except (OSError, ValueError): + return False + # These supervisors also record exits caused by a rejected submission. + return status in {"primary evaluator failed with status 124", "primary evaluator failed with status 137"} + + +def _reported_failure(details): + if details.get("infrastructure_error") is True: + return "judge_infrastructure" if details.get("audit_valid") is False else "verifier_measurement" + if details.get("semantic_judge_succeeded") is False: + return "answer_judging" + if any(details.get(key) is False for key in ("judge_output_valid", "judge_details_valid", + "trajectory_present")): + return "judge_infrastructure" + return None + + def verifier_measurement(verifier, task_name, reward): """Return (valid, failure kind). None means the verifier gave no health signal.""" valid = cgroup_measurement_valid(verifier / "cgroup-status.json") if valid is False: return False, "cgroup_environment" task = task_name.rsplit("/", 1)[-1] + if _legacy_primary_timeout(verifier, task): + return False, "verifier_timeout" evaluation = _report(verifier / f"{task}-eval/evaluation.json") errors = evaluation.get("judge_error_count") if evaluation.get("valid") is False and type(errors) is int and errors > 0: @@ -69,13 +93,16 @@ def verifier_measurement(verifier, task_name, reward): exit_status = guard.get("exit_status") return False, "verifier_timeout" if exit_status in ("124", "137", 124, 137) else "verifier_failure" if filename in ("judge-status.json", "audit-status.json"): - if data.get("infrastructure_error") is True or data.get("healthy") is False: + if data.get("healthy") is False: return False, "verifier_measurement" if data.get("audit_valid") is True else "judge_infrastructure" if data.get("healthy") is True: valid = True for details in (data, data.get("final_gate")): if not isinstance(details, dict): continue + failure = _reported_failure(details) + if failure: + return False, failure flag = details.get("measurement_valid") if flag is False: if not (task == "multilingual-vector-postprocessing" and filename == "judge-status.json" diff --git a/tasks/budgeted-bimetric-retrieval/tests/cgroup_resources.py b/tasks/budgeted-bimetric-retrieval/tests/cgroup_resources.py index ef60a48..9758926 100644 --- a/tasks/budgeted-bimetric-retrieval/tests/cgroup_resources.py +++ b/tasks/budgeted-bimetric-retrieval/tests/cgroup_resources.py @@ -23,6 +23,14 @@ def _read(path): raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error +def _read_proc(path): + # Proc path fields contain filesystem bytes, including non-UTF-8 names. + try: + return path.read_bytes() + except OSError as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + def _integer(value, path, minimum=0): try: result = int(value) @@ -34,7 +42,8 @@ def _integer(value, path, minimum=0): def _mount_path(value): - return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + unescaped = re.sub(rb"\\([0-3][0-7]{2})", lambda m: bytes([int(m[1], 8)]), value) + return Path(os.fsdecode(unescaped)) @cache @@ -45,29 +54,30 @@ def controller(name): def resolve_controller(name, membership_file): """Resolve a process's membership against mounts visible to the reader.""" + key_name = name.encode("ascii") memberships = {} - for line in _read(membership_file).splitlines(): - fields = line.split(":", 2) + for line in _read_proc(membership_file).rstrip(b"\n").split(b"\n"): + fields = line.split(b":", 2) if len(fields) != 3: raise CgroupError(f"Invalid cgroup membership: {line!r}") _, controllers, location = fields - for key in controllers.split(","): - memberships[key] = Path(location) + for key in controllers.split(b","): + memberships[key] = Path(os.fsdecode(location)) # A hybrid host can mount v2 alongside active v1 controllers. - version = 1 if name in memberships else 2 - member = memberships.get(name if version == 1 else "") + version = 1 if key_name in memberships else 2 + member = memberships.get(key_name if version == 1 else b"") if member is None or not member.is_absolute() or ".." in member.parts: raise CgroupError(f"No visible cgroup membership for the {name} controller") candidates = [] - for line in _read(Path("/proc/self/mountinfo")).splitlines(): - before, _, after = line.partition(" - ") - mount, filesystem = before.split(), after.split() + for line in _read_proc(Path("/proc/self/mountinfo")).split(b"\n"): + before, _, after = line.partition(b" - ") + mount, filesystem = before.split(b" "), after.split(b" ") if len(mount) < 6 or len(filesystem) < 3: continue if version == 1: - if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + if filesystem[0] != b"cgroup" or key_name not in filesystem[2].split(b","): continue - elif filesystem[0] != "cgroup2": + elif filesystem[0] != b"cgroup2": continue root, target = map(_mount_path, mount[3:5]) if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): diff --git a/tasks/budgeted-candidate-reranking/environment/runtime/cgroup_resources.py b/tasks/budgeted-candidate-reranking/environment/runtime/cgroup_resources.py index ef60a48..9758926 100644 --- a/tasks/budgeted-candidate-reranking/environment/runtime/cgroup_resources.py +++ b/tasks/budgeted-candidate-reranking/environment/runtime/cgroup_resources.py @@ -23,6 +23,14 @@ def _read(path): raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error +def _read_proc(path): + # Proc path fields contain filesystem bytes, including non-UTF-8 names. + try: + return path.read_bytes() + except OSError as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + def _integer(value, path, minimum=0): try: result = int(value) @@ -34,7 +42,8 @@ def _integer(value, path, minimum=0): def _mount_path(value): - return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + unescaped = re.sub(rb"\\([0-3][0-7]{2})", lambda m: bytes([int(m[1], 8)]), value) + return Path(os.fsdecode(unescaped)) @cache @@ -45,29 +54,30 @@ def controller(name): def resolve_controller(name, membership_file): """Resolve a process's membership against mounts visible to the reader.""" + key_name = name.encode("ascii") memberships = {} - for line in _read(membership_file).splitlines(): - fields = line.split(":", 2) + for line in _read_proc(membership_file).rstrip(b"\n").split(b"\n"): + fields = line.split(b":", 2) if len(fields) != 3: raise CgroupError(f"Invalid cgroup membership: {line!r}") _, controllers, location = fields - for key in controllers.split(","): - memberships[key] = Path(location) + for key in controllers.split(b","): + memberships[key] = Path(os.fsdecode(location)) # A hybrid host can mount v2 alongside active v1 controllers. - version = 1 if name in memberships else 2 - member = memberships.get(name if version == 1 else "") + version = 1 if key_name in memberships else 2 + member = memberships.get(key_name if version == 1 else b"") if member is None or not member.is_absolute() or ".." in member.parts: raise CgroupError(f"No visible cgroup membership for the {name} controller") candidates = [] - for line in _read(Path("/proc/self/mountinfo")).splitlines(): - before, _, after = line.partition(" - ") - mount, filesystem = before.split(), after.split() + for line in _read_proc(Path("/proc/self/mountinfo")).split(b"\n"): + before, _, after = line.partition(b" - ") + mount, filesystem = before.split(b" "), after.split(b" ") if len(mount) < 6 or len(filesystem) < 3: continue if version == 1: - if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + if filesystem[0] != b"cgroup" or key_name not in filesystem[2].split(b","): continue - elif filesystem[0] != "cgroup2": + elif filesystem[0] != b"cgroup2": continue root, target = map(_mount_path, mount[3:5]) if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): diff --git a/tasks/budgeted-candidate-reranking/tests/runtime/cgroup_resources.py b/tasks/budgeted-candidate-reranking/tests/runtime/cgroup_resources.py index ef60a48..9758926 100644 --- a/tasks/budgeted-candidate-reranking/tests/runtime/cgroup_resources.py +++ b/tasks/budgeted-candidate-reranking/tests/runtime/cgroup_resources.py @@ -23,6 +23,14 @@ def _read(path): raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error +def _read_proc(path): + # Proc path fields contain filesystem bytes, including non-UTF-8 names. + try: + return path.read_bytes() + except OSError as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + def _integer(value, path, minimum=0): try: result = int(value) @@ -34,7 +42,8 @@ def _integer(value, path, minimum=0): def _mount_path(value): - return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + unescaped = re.sub(rb"\\([0-3][0-7]{2})", lambda m: bytes([int(m[1], 8)]), value) + return Path(os.fsdecode(unescaped)) @cache @@ -45,29 +54,30 @@ def controller(name): def resolve_controller(name, membership_file): """Resolve a process's membership against mounts visible to the reader.""" + key_name = name.encode("ascii") memberships = {} - for line in _read(membership_file).splitlines(): - fields = line.split(":", 2) + for line in _read_proc(membership_file).rstrip(b"\n").split(b"\n"): + fields = line.split(b":", 2) if len(fields) != 3: raise CgroupError(f"Invalid cgroup membership: {line!r}") _, controllers, location = fields - for key in controllers.split(","): - memberships[key] = Path(location) + for key in controllers.split(b","): + memberships[key] = Path(os.fsdecode(location)) # A hybrid host can mount v2 alongside active v1 controllers. - version = 1 if name in memberships else 2 - member = memberships.get(name if version == 1 else "") + version = 1 if key_name in memberships else 2 + member = memberships.get(key_name if version == 1 else b"") if member is None or not member.is_absolute() or ".." in member.parts: raise CgroupError(f"No visible cgroup membership for the {name} controller") candidates = [] - for line in _read(Path("/proc/self/mountinfo")).splitlines(): - before, _, after = line.partition(" - ") - mount, filesystem = before.split(), after.split() + for line in _read_proc(Path("/proc/self/mountinfo")).split(b"\n"): + before, _, after = line.partition(b" - ") + mount, filesystem = before.split(b" "), after.split(b" ") if len(mount) < 6 or len(filesystem) < 3: continue if version == 1: - if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + if filesystem[0] != b"cgroup" or key_name not in filesystem[2].split(b","): continue - elif filesystem[0] != "cgroup2": + elif filesystem[0] != b"cgroup2": continue root, target = map(_mount_path, mount[3:5]) if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): diff --git a/tasks/constrained-molecular-search/environment/runtime/cgroup_resources.py b/tasks/constrained-molecular-search/environment/runtime/cgroup_resources.py index ef60a48..9758926 100644 --- a/tasks/constrained-molecular-search/environment/runtime/cgroup_resources.py +++ b/tasks/constrained-molecular-search/environment/runtime/cgroup_resources.py @@ -23,6 +23,14 @@ def _read(path): raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error +def _read_proc(path): + # Proc path fields contain filesystem bytes, including non-UTF-8 names. + try: + return path.read_bytes() + except OSError as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + def _integer(value, path, minimum=0): try: result = int(value) @@ -34,7 +42,8 @@ def _integer(value, path, minimum=0): def _mount_path(value): - return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + unescaped = re.sub(rb"\\([0-3][0-7]{2})", lambda m: bytes([int(m[1], 8)]), value) + return Path(os.fsdecode(unescaped)) @cache @@ -45,29 +54,30 @@ def controller(name): def resolve_controller(name, membership_file): """Resolve a process's membership against mounts visible to the reader.""" + key_name = name.encode("ascii") memberships = {} - for line in _read(membership_file).splitlines(): - fields = line.split(":", 2) + for line in _read_proc(membership_file).rstrip(b"\n").split(b"\n"): + fields = line.split(b":", 2) if len(fields) != 3: raise CgroupError(f"Invalid cgroup membership: {line!r}") _, controllers, location = fields - for key in controllers.split(","): - memberships[key] = Path(location) + for key in controllers.split(b","): + memberships[key] = Path(os.fsdecode(location)) # A hybrid host can mount v2 alongside active v1 controllers. - version = 1 if name in memberships else 2 - member = memberships.get(name if version == 1 else "") + version = 1 if key_name in memberships else 2 + member = memberships.get(key_name if version == 1 else b"") if member is None or not member.is_absolute() or ".." in member.parts: raise CgroupError(f"No visible cgroup membership for the {name} controller") candidates = [] - for line in _read(Path("/proc/self/mountinfo")).splitlines(): - before, _, after = line.partition(" - ") - mount, filesystem = before.split(), after.split() + for line in _read_proc(Path("/proc/self/mountinfo")).split(b"\n"): + before, _, after = line.partition(b" - ") + mount, filesystem = before.split(b" "), after.split(b" ") if len(mount) < 6 or len(filesystem) < 3: continue if version == 1: - if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + if filesystem[0] != b"cgroup" or key_name not in filesystem[2].split(b","): continue - elif filesystem[0] != "cgroup2": + elif filesystem[0] != b"cgroup2": continue root, target = map(_mount_path, mount[3:5]) if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): diff --git a/tasks/constrained-molecular-search/tests/runtime/cgroup_resources.py b/tasks/constrained-molecular-search/tests/runtime/cgroup_resources.py index ef60a48..9758926 100644 --- a/tasks/constrained-molecular-search/tests/runtime/cgroup_resources.py +++ b/tasks/constrained-molecular-search/tests/runtime/cgroup_resources.py @@ -23,6 +23,14 @@ def _read(path): raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error +def _read_proc(path): + # Proc path fields contain filesystem bytes, including non-UTF-8 names. + try: + return path.read_bytes() + except OSError as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + def _integer(value, path, minimum=0): try: result = int(value) @@ -34,7 +42,8 @@ def _integer(value, path, minimum=0): def _mount_path(value): - return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + unescaped = re.sub(rb"\\([0-3][0-7]{2})", lambda m: bytes([int(m[1], 8)]), value) + return Path(os.fsdecode(unescaped)) @cache @@ -45,29 +54,30 @@ def controller(name): def resolve_controller(name, membership_file): """Resolve a process's membership against mounts visible to the reader.""" + key_name = name.encode("ascii") memberships = {} - for line in _read(membership_file).splitlines(): - fields = line.split(":", 2) + for line in _read_proc(membership_file).rstrip(b"\n").split(b"\n"): + fields = line.split(b":", 2) if len(fields) != 3: raise CgroupError(f"Invalid cgroup membership: {line!r}") _, controllers, location = fields - for key in controllers.split(","): - memberships[key] = Path(location) + for key in controllers.split(b","): + memberships[key] = Path(os.fsdecode(location)) # A hybrid host can mount v2 alongside active v1 controllers. - version = 1 if name in memberships else 2 - member = memberships.get(name if version == 1 else "") + version = 1 if key_name in memberships else 2 + member = memberships.get(key_name if version == 1 else b"") if member is None or not member.is_absolute() or ".." in member.parts: raise CgroupError(f"No visible cgroup membership for the {name} controller") candidates = [] - for line in _read(Path("/proc/self/mountinfo")).splitlines(): - before, _, after = line.partition(" - ") - mount, filesystem = before.split(), after.split() + for line in _read_proc(Path("/proc/self/mountinfo")).split(b"\n"): + before, _, after = line.partition(b" - ") + mount, filesystem = before.split(b" "), after.split(b" ") if len(mount) < 6 or len(filesystem) < 3: continue if version == 1: - if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + if filesystem[0] != b"cgroup" or key_name not in filesystem[2].split(b","): continue - elif filesystem[0] != "cgroup2": + elif filesystem[0] != b"cgroup2": continue root, target = map(_mount_path, mount[3:5]) if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): diff --git a/tasks/large-scale-formula-retrieval/environment/cgroup_resources.py b/tasks/large-scale-formula-retrieval/environment/cgroup_resources.py index ef60a48..9758926 100644 --- a/tasks/large-scale-formula-retrieval/environment/cgroup_resources.py +++ b/tasks/large-scale-formula-retrieval/environment/cgroup_resources.py @@ -23,6 +23,14 @@ def _read(path): raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error +def _read_proc(path): + # Proc path fields contain filesystem bytes, including non-UTF-8 names. + try: + return path.read_bytes() + except OSError as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + def _integer(value, path, minimum=0): try: result = int(value) @@ -34,7 +42,8 @@ def _integer(value, path, minimum=0): def _mount_path(value): - return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + unescaped = re.sub(rb"\\([0-3][0-7]{2})", lambda m: bytes([int(m[1], 8)]), value) + return Path(os.fsdecode(unescaped)) @cache @@ -45,29 +54,30 @@ def controller(name): def resolve_controller(name, membership_file): """Resolve a process's membership against mounts visible to the reader.""" + key_name = name.encode("ascii") memberships = {} - for line in _read(membership_file).splitlines(): - fields = line.split(":", 2) + for line in _read_proc(membership_file).rstrip(b"\n").split(b"\n"): + fields = line.split(b":", 2) if len(fields) != 3: raise CgroupError(f"Invalid cgroup membership: {line!r}") _, controllers, location = fields - for key in controllers.split(","): - memberships[key] = Path(location) + for key in controllers.split(b","): + memberships[key] = Path(os.fsdecode(location)) # A hybrid host can mount v2 alongside active v1 controllers. - version = 1 if name in memberships else 2 - member = memberships.get(name if version == 1 else "") + version = 1 if key_name in memberships else 2 + member = memberships.get(key_name if version == 1 else b"") if member is None or not member.is_absolute() or ".." in member.parts: raise CgroupError(f"No visible cgroup membership for the {name} controller") candidates = [] - for line in _read(Path("/proc/self/mountinfo")).splitlines(): - before, _, after = line.partition(" - ") - mount, filesystem = before.split(), after.split() + for line in _read_proc(Path("/proc/self/mountinfo")).split(b"\n"): + before, _, after = line.partition(b" - ") + mount, filesystem = before.split(b" "), after.split(b" ") if len(mount) < 6 or len(filesystem) < 3: continue if version == 1: - if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + if filesystem[0] != b"cgroup" or key_name not in filesystem[2].split(b","): continue - elif filesystem[0] != "cgroup2": + elif filesystem[0] != b"cgroup2": continue root, target = map(_mount_path, mount[3:5]) if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): diff --git a/tasks/large-scale-formula-retrieval/tests/cgroup_resources.py b/tasks/large-scale-formula-retrieval/tests/cgroup_resources.py index ef60a48..9758926 100644 --- a/tasks/large-scale-formula-retrieval/tests/cgroup_resources.py +++ b/tasks/large-scale-formula-retrieval/tests/cgroup_resources.py @@ -23,6 +23,14 @@ def _read(path): raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error +def _read_proc(path): + # Proc path fields contain filesystem bytes, including non-UTF-8 names. + try: + return path.read_bytes() + except OSError as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + def _integer(value, path, minimum=0): try: result = int(value) @@ -34,7 +42,8 @@ def _integer(value, path, minimum=0): def _mount_path(value): - return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + unescaped = re.sub(rb"\\([0-3][0-7]{2})", lambda m: bytes([int(m[1], 8)]), value) + return Path(os.fsdecode(unescaped)) @cache @@ -45,29 +54,30 @@ def controller(name): def resolve_controller(name, membership_file): """Resolve a process's membership against mounts visible to the reader.""" + key_name = name.encode("ascii") memberships = {} - for line in _read(membership_file).splitlines(): - fields = line.split(":", 2) + for line in _read_proc(membership_file).rstrip(b"\n").split(b"\n"): + fields = line.split(b":", 2) if len(fields) != 3: raise CgroupError(f"Invalid cgroup membership: {line!r}") _, controllers, location = fields - for key in controllers.split(","): - memberships[key] = Path(location) + for key in controllers.split(b","): + memberships[key] = Path(os.fsdecode(location)) # A hybrid host can mount v2 alongside active v1 controllers. - version = 1 if name in memberships else 2 - member = memberships.get(name if version == 1 else "") + version = 1 if key_name in memberships else 2 + member = memberships.get(key_name if version == 1 else b"") if member is None or not member.is_absolute() or ".." in member.parts: raise CgroupError(f"No visible cgroup membership for the {name} controller") candidates = [] - for line in _read(Path("/proc/self/mountinfo")).splitlines(): - before, _, after = line.partition(" - ") - mount, filesystem = before.split(), after.split() + for line in _read_proc(Path("/proc/self/mountinfo")).split(b"\n"): + before, _, after = line.partition(b" - ") + mount, filesystem = before.split(b" "), after.split(b" ") if len(mount) < 6 or len(filesystem) < 3: continue if version == 1: - if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + if filesystem[0] != b"cgroup" or key_name not in filesystem[2].split(b","): continue - elif filesystem[0] != "cgroup2": + elif filesystem[0] != b"cgroup2": continue root, target = map(_mount_path, mount[3:5]) if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): diff --git a/tasks/memory-constrained-dense-retrieval/tests/cgroup_resources.py b/tasks/memory-constrained-dense-retrieval/tests/cgroup_resources.py index ef60a48..9758926 100644 --- a/tasks/memory-constrained-dense-retrieval/tests/cgroup_resources.py +++ b/tasks/memory-constrained-dense-retrieval/tests/cgroup_resources.py @@ -23,6 +23,14 @@ def _read(path): raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error +def _read_proc(path): + # Proc path fields contain filesystem bytes, including non-UTF-8 names. + try: + return path.read_bytes() + except OSError as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + def _integer(value, path, minimum=0): try: result = int(value) @@ -34,7 +42,8 @@ def _integer(value, path, minimum=0): def _mount_path(value): - return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + unescaped = re.sub(rb"\\([0-3][0-7]{2})", lambda m: bytes([int(m[1], 8)]), value) + return Path(os.fsdecode(unescaped)) @cache @@ -45,29 +54,30 @@ def controller(name): def resolve_controller(name, membership_file): """Resolve a process's membership against mounts visible to the reader.""" + key_name = name.encode("ascii") memberships = {} - for line in _read(membership_file).splitlines(): - fields = line.split(":", 2) + for line in _read_proc(membership_file).rstrip(b"\n").split(b"\n"): + fields = line.split(b":", 2) if len(fields) != 3: raise CgroupError(f"Invalid cgroup membership: {line!r}") _, controllers, location = fields - for key in controllers.split(","): - memberships[key] = Path(location) + for key in controllers.split(b","): + memberships[key] = Path(os.fsdecode(location)) # A hybrid host can mount v2 alongside active v1 controllers. - version = 1 if name in memberships else 2 - member = memberships.get(name if version == 1 else "") + version = 1 if key_name in memberships else 2 + member = memberships.get(key_name if version == 1 else b"") if member is None or not member.is_absolute() or ".." in member.parts: raise CgroupError(f"No visible cgroup membership for the {name} controller") candidates = [] - for line in _read(Path("/proc/self/mountinfo")).splitlines(): - before, _, after = line.partition(" - ") - mount, filesystem = before.split(), after.split() + for line in _read_proc(Path("/proc/self/mountinfo")).split(b"\n"): + before, _, after = line.partition(b" - ") + mount, filesystem = before.split(b" "), after.split(b" ") if len(mount) < 6 or len(filesystem) < 3: continue if version == 1: - if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + if filesystem[0] != b"cgroup" or key_name not in filesystem[2].split(b","): continue - elif filesystem[0] != "cgroup2": + elif filesystem[0] != b"cgroup2": continue root, target = map(_mount_path, mount[3:5]) if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): diff --git a/tasks/multilingual-vector-postprocessing/environment/runtime/evaluate.py b/tasks/multilingual-vector-postprocessing/environment/runtime/evaluate.py index 5b171f2..2d3cc1f 100644 --- a/tasks/multilingual-vector-postprocessing/environment/runtime/evaluate.py +++ b/tasks/multilingual-vector-postprocessing/environment/runtime/evaluate.py @@ -12,7 +12,7 @@ from threadpoolctl import threadpool_limits from failures import ContractViolation, LEGACY_COLLECTION_REJECTIONS, SubmissionViolation from metrics import aggregate, normalize, quality -from process import TransformError, TransformProcess +from process import TransformError, TransformProcess, WorkerExited from storage import StorageBudget @@ -167,14 +167,14 @@ def prepare_vectors(vectors, languages, *, normalized=False): try: transformed = worker.transform(vectors, languages, prepare_deadline) return normalize(transformed) if normalized else transformed - except (ValueError, TimeoutError) as error: + except (ValueError, TimeoutError, WorkerExited, BrokenPipeError) as error: raise SubmissionViolation(f"Document preparation failed: {error}") from error try: try: worker = TransformProcess(submission, temporary.name, output / "worker.stderr.log", prepare_deadline, sandbox=sandbox, require_gpu=require_gpu) - except (TransformError, TimeoutError) as error: + except TransformError as error: raise SubmissionViolation(f"Document preparation failed: {error}") from error for dataset in ("belebele", "xquad"): metadata, raw = inputs(Path(corpus) / dataset, "documents") diff --git a/tasks/multilingual-vector-postprocessing/environment/runtime/process.py b/tasks/multilingual-vector-postprocessing/environment/runtime/process.py index afac025..d8a872a 100644 --- a/tasks/multilingual-vector-postprocessing/environment/runtime/process.py +++ b/tasks/multilingual-vector-postprocessing/environment/runtime/process.py @@ -19,6 +19,10 @@ class TransformError(ValueError): pass +class WorkerExited(RuntimeError): + """The worker closed its response pipe before completing a request.""" + + class TransformProcess: def __init__(self, submission, work, log, deadline, sandbox=False, require_gpu=True): self.sandbox = sandbox @@ -53,11 +57,16 @@ def drain(): self.drainer = threading.Thread(target=drain, daemon=True) self.drainer.start() try: - response = self.read(deadline) + if self.read(deadline) != {"loading": True}: + raise RuntimeError("Transform failed to complete trusted startup") + try: + response = self.read(deadline) + except (WorkerExited, TimeoutError, ValueError) as error: + raise TransformError(str(error)) from error if "error" in response: raise TransformError(str(response["error"])) if response != {"ready": True}: - raise ValueError("Transform did not become ready") + raise TransformError("Transform did not become ready") except BaseException: self.close() raise @@ -69,7 +78,7 @@ def read(self, deadline): raise TimeoutError("Transform response timed out") chunk = os.read(self.process.stdout.fileno(), 65536) if not chunk: - raise RuntimeError("Transform exited before responding") + raise WorkerExited("Transform exited before responding") self.buffer += chunk if len(self.buffer) > MAX_FRAME: raise ValueError("Transform response exceeded the frame limit") diff --git a/tasks/multilingual-vector-postprocessing/environment/runtime/worker.py b/tasks/multilingual-vector-postprocessing/environment/runtime/worker.py index 4fbe3c2..7c643af 100644 --- a/tasks/multilingual-vector-postprocessing/environment/runtime/worker.py +++ b/tasks/multilingual-vector-postprocessing/environment/runtime/worker.py @@ -48,6 +48,8 @@ def main(): isolate(work) else: os.chdir(work) + # Acknowledge trusted startup before importing or running submitted code. + print('{"loading":true}', flush=True) # Do not import submitted Python or deserialize its weights before isolation. sys.path.insert(0, str(submission)) spec = importlib.util.spec_from_file_location("submission_transform", submission / "transform.py") diff --git a/tasks/multilingual-vector-postprocessing/tests/runtime/evaluate.py b/tasks/multilingual-vector-postprocessing/tests/runtime/evaluate.py index 5b171f2..2d3cc1f 100644 --- a/tasks/multilingual-vector-postprocessing/tests/runtime/evaluate.py +++ b/tasks/multilingual-vector-postprocessing/tests/runtime/evaluate.py @@ -12,7 +12,7 @@ from threadpoolctl import threadpool_limits from failures import ContractViolation, LEGACY_COLLECTION_REJECTIONS, SubmissionViolation from metrics import aggregate, normalize, quality -from process import TransformError, TransformProcess +from process import TransformError, TransformProcess, WorkerExited from storage import StorageBudget @@ -167,14 +167,14 @@ def prepare_vectors(vectors, languages, *, normalized=False): try: transformed = worker.transform(vectors, languages, prepare_deadline) return normalize(transformed) if normalized else transformed - except (ValueError, TimeoutError) as error: + except (ValueError, TimeoutError, WorkerExited, BrokenPipeError) as error: raise SubmissionViolation(f"Document preparation failed: {error}") from error try: try: worker = TransformProcess(submission, temporary.name, output / "worker.stderr.log", prepare_deadline, sandbox=sandbox, require_gpu=require_gpu) - except (TransformError, TimeoutError) as error: + except TransformError as error: raise SubmissionViolation(f"Document preparation failed: {error}") from error for dataset in ("belebele", "xquad"): metadata, raw = inputs(Path(corpus) / dataset, "documents") diff --git a/tasks/multilingual-vector-postprocessing/tests/runtime/process.py b/tasks/multilingual-vector-postprocessing/tests/runtime/process.py index afac025..d8a872a 100644 --- a/tasks/multilingual-vector-postprocessing/tests/runtime/process.py +++ b/tasks/multilingual-vector-postprocessing/tests/runtime/process.py @@ -19,6 +19,10 @@ class TransformError(ValueError): pass +class WorkerExited(RuntimeError): + """The worker closed its response pipe before completing a request.""" + + class TransformProcess: def __init__(self, submission, work, log, deadline, sandbox=False, require_gpu=True): self.sandbox = sandbox @@ -53,11 +57,16 @@ def drain(): self.drainer = threading.Thread(target=drain, daemon=True) self.drainer.start() try: - response = self.read(deadline) + if self.read(deadline) != {"loading": True}: + raise RuntimeError("Transform failed to complete trusted startup") + try: + response = self.read(deadline) + except (WorkerExited, TimeoutError, ValueError) as error: + raise TransformError(str(error)) from error if "error" in response: raise TransformError(str(response["error"])) if response != {"ready": True}: - raise ValueError("Transform did not become ready") + raise TransformError("Transform did not become ready") except BaseException: self.close() raise @@ -69,7 +78,7 @@ def read(self, deadline): raise TimeoutError("Transform response timed out") chunk = os.read(self.process.stdout.fileno(), 65536) if not chunk: - raise RuntimeError("Transform exited before responding") + raise WorkerExited("Transform exited before responding") self.buffer += chunk if len(self.buffer) > MAX_FRAME: raise ValueError("Transform response exceeded the frame limit") diff --git a/tasks/multilingual-vector-postprocessing/tests/runtime/worker.py b/tasks/multilingual-vector-postprocessing/tests/runtime/worker.py index 4fbe3c2..7c643af 100644 --- a/tasks/multilingual-vector-postprocessing/tests/runtime/worker.py +++ b/tasks/multilingual-vector-postprocessing/tests/runtime/worker.py @@ -48,6 +48,8 @@ def main(): isolate(work) else: os.chdir(work) + # Acknowledge trusted startup before importing or running submitted code. + print('{"loading":true}', flush=True) # Do not import submitted Python or deserialize its weights before isolation. sys.path.insert(0, str(submission)) spec = importlib.util.spec_from_file_location("submission_transform", submission / "transform.py") diff --git a/tasks/multilingual-vector-postprocessing/tests/test_runtime_controls.py b/tasks/multilingual-vector-postprocessing/tests/test_runtime_controls.py index 56cdd1b..9286ba9 100644 --- a/tasks/multilingual-vector-postprocessing/tests/test_runtime_controls.py +++ b/tasks/multilingual-vector-postprocessing/tests/test_runtime_controls.py @@ -305,6 +305,13 @@ def test_worker_load_exception_is_a_submission_failure(self): with self.assertRaisesRegex(SubmissionViolation, "Document preparation failed: ValueError"): self.run_control() + def test_worker_exit_during_load_or_document_preparation_is_a_submission_failure(self): + for script in ({"load": "os._exit(2)"}, {"transform": "os._exit(2)"}): + with self.subTest(script=script): + self.module(**script) + with self.assertRaisesRegex(SubmissionViolation, "Transform exited before responding"): + self.run_control() + def test_labels_rewards_network_and_privilege_protected(self): marker = ROOT / "secret-label.txt" marker.write_text("private fixture label") @@ -347,8 +354,8 @@ def test_preparation_timeout_stops_worker(self): with self.assertRaisesRegex(SubmissionViolation, "timed out"): evaluate("/app/submission", ROOT / "corpus", ROOT / "hidden", ROOT / "development", ROOT / "logs/preparation-timeout", sandbox=True, require_gpu=False, - preparation_seconds=.5, frozen_status=ROOT / "frozen-status.json") - self.assertLess(time.monotonic() - began, 10) + preparation_seconds=5, frozen_status=ROOT / "frozen-status.json") + self.assertLess(time.monotonic() - began, 15) def test_query_timeout_preserves_denominators(self): self.module(transform="if vectors[0,2] > .5:\n import time\n time.sleep(60)\nreturn vectors.copy()") diff --git a/tasks/training-error-tracing/tests/cgroup_resources.py b/tasks/training-error-tracing/tests/cgroup_resources.py index ef60a48..9758926 100644 --- a/tasks/training-error-tracing/tests/cgroup_resources.py +++ b/tasks/training-error-tracing/tests/cgroup_resources.py @@ -23,6 +23,14 @@ def _read(path): raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error +def _read_proc(path): + # Proc path fields contain filesystem bytes, including non-UTF-8 names. + try: + return path.read_bytes() + except OSError as error: + raise CgroupError(f"Cannot read cgroup evidence at {path}: {error}") from error + + def _integer(value, path, minimum=0): try: result = int(value) @@ -34,7 +42,8 @@ def _integer(value, path, minimum=0): def _mount_path(value): - return Path(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)) + unescaped = re.sub(rb"\\([0-3][0-7]{2})", lambda m: bytes([int(m[1], 8)]), value) + return Path(os.fsdecode(unescaped)) @cache @@ -45,29 +54,30 @@ def controller(name): def resolve_controller(name, membership_file): """Resolve a process's membership against mounts visible to the reader.""" + key_name = name.encode("ascii") memberships = {} - for line in _read(membership_file).splitlines(): - fields = line.split(":", 2) + for line in _read_proc(membership_file).rstrip(b"\n").split(b"\n"): + fields = line.split(b":", 2) if len(fields) != 3: raise CgroupError(f"Invalid cgroup membership: {line!r}") _, controllers, location = fields - for key in controllers.split(","): - memberships[key] = Path(location) + for key in controllers.split(b","): + memberships[key] = Path(os.fsdecode(location)) # A hybrid host can mount v2 alongside active v1 controllers. - version = 1 if name in memberships else 2 - member = memberships.get(name if version == 1 else "") + version = 1 if key_name in memberships else 2 + member = memberships.get(key_name if version == 1 else b"") if member is None or not member.is_absolute() or ".." in member.parts: raise CgroupError(f"No visible cgroup membership for the {name} controller") candidates = [] - for line in _read(Path("/proc/self/mountinfo")).splitlines(): - before, _, after = line.partition(" - ") - mount, filesystem = before.split(), after.split() + for line in _read_proc(Path("/proc/self/mountinfo")).split(b"\n"): + before, _, after = line.partition(b" - ") + mount, filesystem = before.split(b" "), after.split(b" ") if len(mount) < 6 or len(filesystem) < 3: continue if version == 1: - if filesystem[0] != "cgroup" or name not in filesystem[2].split(","): + if filesystem[0] != b"cgroup" or key_name not in filesystem[2].split(b","): continue - elif filesystem[0] != "cgroup2": + elif filesystem[0] != b"cgroup2": continue root, target = map(_mount_path, mount[3:5]) if root.is_absolute() and ".." not in root.parts and member.is_relative_to(root): diff --git a/tests/unit/test_cgroup_resources.py b/tests/unit/test_cgroup_resources.py index 32a632a..b55cb8d 100644 --- a/tests/unit/test_cgroup_resources.py +++ b/tests/unit/test_cgroup_resources.py @@ -26,16 +26,16 @@ def put(self, **files): (self.path / name).write_text(value + "\n") def resolve(self, membership, mountinfo, *names): - original = cg._read + original = cg._read_proc def read(path): if path == Path("/proc/self/cgroup"): - return membership + return os.fsencode(membership) if path == Path("/proc/self/mountinfo"): - return mountinfo + return os.fsencode(mountinfo) return original(path) - with patch.object(cg, "_read", side_effect=read) as reads: + with patch.object(cg, "_read_proc", side_effect=read) as reads: result = [cg.controller(name) for name in names] self.assertIs(cg.controller(names[0]), result[0]) self.assertEqual(reads.call_count, 2 * len(names)) @@ -61,6 +61,14 @@ def test_mount_for_a_different_group_is_not_a_fallback(self): with self.assertRaisesRegex(cg.CgroupError, "No cgroup v1 mount covers memory"): self.resolve("5:memory:/docker/isolated", mounts, "memory") + def test_mount_and_membership_preserve_filesystem_bytes_and_mount_escapes(self): + parent = os.fsdecode(b"/slice-\xff") + "-中文" + target = os.fsdecode(b"/mount-\xfe") + r"\040中文\134cgroup" + mounts = (f"10 1 0:2 /普通挂载 /inputs ro - tmpfs tmpfs ro\n" + f"17 1 0:12 {parent} {target} ro - cgroup2 cgroup ro") + memory, = self.resolve(f"0::{parent}/任务", mounts, "memory") + self.assertEqual(os.fsencode(memory.path), b"/mount-\xfe " + "中文\\cgroup/任务".encode()) + def test_v2_budget_current_peak_and_oom_are_separate(self): self.put(**{"cpu.max": "1600000 100000", "memory.max": str(16 << 30), "memory.swap.max": "0", "memory.current": str(120 << 20), diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 7da2722..2c1e0e9 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -380,6 +380,9 @@ def test_batch_and_regrade_agree_on_verifier_diagnostics(self): "healthy": False, "integrity": 0} audit_rejected = {"primary_valid": True, "audit_valid": True, "infrastructure_error": False, "healthy": True, "integrity": 0} + primary_timeout = {"status.txt": "primary evaluator failed with status 124\n"} + submission_exit = {"status.txt": "primary evaluator failed with status 1\n"} + bad_output = {"judge_succeeded": True, "judge_output_valid": False} for task, files, failure_kind in ( ("query-encoder-alignment", {"reward-details.json": timeout}, "verifier_timeout"), ("query-encoder-alignment", {"reward-details.json": primary_error}, "verifier_failure"), @@ -391,6 +394,19 @@ def test_batch_and_regrade_agree_on_verifier_diagnostics(self): ("icsi-meeting-memory", {"gate-status.json": {"status": "complete", "reason": "invalid_submission"}}, None), ("agentic-search", {"jailbreak-status.json": {"status": "error"}}, "verifier_measurement"), ("agentic-search", {"jailbreak-status.json": {"status": "ok", "passed": False}}, None), + ("training-error-tracing", {"reward-details.json": {"infrastructure_error": True}}, "verifier_measurement"), + ("training-error-tracing", {"reward-details.json": {"final_gate": audit_failed}}, "judge_infrastructure"), + ("memory-constrained-dense-retrieval", {"reward-details.json": {"final_gate": bad_output}}, "judge_infrastructure"), + ("long-pdf-evidence", {"reward-details.json": {"final_gate": bad_output}}, "judge_infrastructure"), + ("instruction-following-embedding", primary_timeout, "verifier_timeout"), + ("protein-text-bidirectional-retrieval", primary_timeout, "verifier_timeout"), + ("instruction-following-embedding", submission_exit, None), + ("protein-text-bidirectional-retrieval", submission_exit, None), + ("evolving-personal-memory", {"reward-details.json": {"semantic_judge_succeeded": False, + "trajectory_judge_succeeded": True}}, "answer_judging"), + ("training-error-tracing", {"reward-details.json": {"final_gate": {"primary_valid": False, + "audit_valid": True, + "infrastructure_error": False}}}, None), ("query-encoder-alignment", {"reward-details.json": timeout, "judge-status.json": rejected}, "verifier_timeout"), ("fixture", {"reward-details.json": {"final_gate": {"measurement_valid": False}}}, "verifier_measurement"), ("fixture", {"reward-details.json": {"final_gate": {"evaluation_valid": True, @@ -405,7 +421,8 @@ def test_batch_and_regrade_agree_on_verifier_diagnostics(self): '(trial/"verifier").mkdir()\n' ) for name, content in files.items(): - script.write(f'(trial/"verifier"/{name!r}).write_text({json.dumps(content)!r})\n') + payload = content if isinstance(content, str) else json.dumps(content) + script.write(f'(trial/"verifier"/{name!r}).write_text({payload!r})\n') output = Path(temp) / "batch" with redirect_stdout(io.StringIO()): code = run_batch(self.root, self.tasks[:1], Setting(), dict(os.environ), @@ -414,7 +431,10 @@ def test_batch_and_regrade_agree_on_verifier_diagnostics(self): regrade = {} result_summary(Path(row["trial_result"]).parent, regrade) valid = failure_kind is None - self.assertIs(regrade["measurement_valid"], valid) + if valid: + self.assertIsNot(regrade["measurement_valid"], False) + else: + self.assertIs(regrade["measurement_valid"], False) self.assertEqual(row["status"], regrade["status"]) self.assertEqual(row["status"], "completed" if valid else "invalid-measurement") self.assertEqual(row.get("failure_kind"), failure_kind)