Skip to content

story-041: The verification runner is not assumed to be Python #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jerodw merged 1 commit into from
Aug 15, 2026
Merged

story-041: The verification runner is not assumed to be Python #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .harness/config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ standards_dir: .harness/standards
architecture_docs:
- .harness/docs/ARCHITECTURE.md
test_command: .venv/bin/python -m pytest tests/ -q
# The interpreter the clean-clone check runs the suite under. Deliberately
# the oldest Python CI tests (3.10), not the one the harness runs in (3.14),
# so a version incompatibility is found before CI rather than by it. Absent,
# the check falls back to test_command's own interpreter.
clean_clone_python: .venv310/bin/python
# The executable the clean-clone check runs the suite under. Deliberately an
# older environment than the one the developer works in, and the one CI
# exercises, so an incompatibility is found before CI rather than by it. It
# replaces the first word of test_command; absent, that first word is used.
verification_runner: .venv310/bin/python
# Bash commands stage agents may run without prompting. Everything else
# is denied in headless mode. Read-only search and inspection is granted
# broadly: a denial costs a turn and buys nothing, because the harness's
Expand Down
16 changes: 11 additions & 5 deletions .harness/docs/ARCHITECTURE.md
View file
Open in desktop

Large diffs are not rendered by default.

25 changes: 25 additions & 0 deletions orchestration/harness_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,31 @@
# refused.
CONFIG_SCHEMA_NAME = "harness-config"

#: Keys the harness once read, mapped to the key that replaced each. A
#: retired key is refused rather than ignored: a config still carrying one
#: would fall through to the replacement's fallback and quietly change what
#: the harness does, which is the drift the declaration exists to stop. It
#: lives beside `declared_config_keys` so the config vocabulary — what is
#: read, and what used to be — has one home.
RETIRED_CONFIG_KEYS: dict[str, str] = {
"clean_clone_python": "verification_runner",
}


def retired_config_problems(config: dict) -> list[str]:
"""One problem per retired key a loaded config still carries.

Each names the retired key and the key that replaced it, so the refusal
is actionable without opening the schema. An empty list is the whole of
"this config carries none".
"""
return [
f"'{key}' is no longer read by the harness; it was replaced by "
f"'{replacement}'"
for key, replacement in RETIRED_CONFIG_KEYS.items()
if key in config
]


def find_target_root(start: Path) -> Path:
for candidate in [start, *start.parents]:
Expand Down
104 changes: 54 additions & 50 deletions orchestration/story_coordinator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ class of edit, it does not audit one.
import functools
import hashlib
import json
import re
import shlex
import shutil
import subprocess
Expand Down Expand Up @@ -942,9 +941,6 @@ def append_retry_record(
#: what failed; not the whole log, which the run directory is not a home for.
CLEAN_CLONE_OUTPUT_TAIL = 8000

_VERSION_PROBE = "import platform; print(platform.python_version())"
_VERSION = re.compile(r"\d+\.\d+\.\d+\S*")


@dataclass(frozen=True)
class CleanCloneResult:
Expand All @@ -957,17 +953,15 @@ class CleanCloneResult:

ran: bool
command: str
python: str
python_version: str | None = None
runner: str
clone_path: str | None = None
exit_code: int | None = None
output_tail: str | None = None
reason: str | None = None

def as_record(self) -> dict:
record: dict = {"ran": self.ran, "command": self.command, "python": self.python}
record: dict = {"ran": self.ran, "command": self.command, "runner": self.runner}
optional = {
"python_version": self.python_version,
"clone_path": self.clone_path,
"exit_code": self.exit_code,
"output_tail": self.output_tail,
Expand All @@ -992,27 +986,6 @@ def _resolve_interpreter(target_root: Path, interpreter: str) -> Path | None:
return Path(found) if found else None


def _interpreter_version(interpreter: Path) -> str | None:
"""What the interpreter reports its version to be, or None.

None is not a failure: the configured test command need not be a Python
interpreter at all, and a record with no version is honest about that. A
*configured* clean_clone_python that does not exist is a different case,
handled by the caller, because a check quietly testing the wrong version
is worse than one that refuses.
"""
try:
result = subprocess.run(
[str(interpreter), "-c", _VERSION_PROBE],
capture_output=True,
text=True,
)
except OSError:
return None
version = result.stdout.strip()
return version if result.returncode == 0 and _VERSION.fullmatch(version) else None


def _build_clone(
target_root: Path,
clone: Path,
Expand Down Expand Up @@ -1166,18 +1139,24 @@ def _link_interpreter_roots(target_root: Path, clone: Path, interpreters: list[s
def run_clean_clone(
target_root: Path,
test_command: str,
clean_clone_python: str | None,
verification_runner: str | None,
destination: Path,
revert: list[str] | tuple[str, ...] = (),
baseline: Path | None = None,
) -> CleanCloneResult:
"""Run the configured test command in a fresh clone with the story committed.

The command is the target repository's own `test_command`; nothing about
it is written here. Only its interpreter is substituted, and only when the
configuration names a `clean_clone_python`, so the check can exercise the
oldest supported Python rather than whichever one the developer works in.
The caller owns `destination` and removes it whatever the result.
it is written here. Only its *first word* is substituted, and only when the
configuration names a `verification_runner`, so the check can exercise an
environment other than the one the developer works in and find an
incompatibility before CI rather than by it. The caller owns `destination`
and removes it whatever the result.

That substitution is a first-word swap and nothing more: every remaining
argument is the configured command's own. A target whose environment
difference cannot be expressed by replacing the first word expresses it in
`test_command` instead.

This is the single build-a-clone-and-run-the-suite path. `revert` and the
`baseline` it is restored from are passed through to the clone builder and
Expand All @@ -1186,27 +1165,27 @@ def run_clean_clone(
restored to the state the stage found them in rather than applied.
"""
argv = shlex.split(test_command)
interpreter = clean_clone_python or argv[0]
command = shlex.join([interpreter, *argv[1:]])
runner = verification_runner or argv[0]
command = shlex.join([runner, *argv[1:]])

resolved = _resolve_interpreter(target_root, interpreter)
if clean_clone_python and resolved is None:
resolved = _resolve_interpreter(target_root, runner)
if verification_runner and resolved is None:
return CleanCloneResult(
ran=False,
command=command,
python=interpreter,
runner=runner,
reason=(
f"clean_clone_python names {clean_clone_python}, which is not an "
f"interpreter that exists under {target_root}"
f"verification_runner names {verification_runner}, which is not "
f"an executable that exists under {target_root}"
),
)

clone = destination / "clone"
_build_clone(target_root, clone, revert=revert, baseline=baseline)
_link_interpreter_roots(target_root, clone, [argv[0], interpreter])
_link_interpreter_roots(target_root, clone, [argv[0], runner])

result = subprocess.run(
[interpreter, *argv[1:]],
[runner, *argv[1:]],
cwd=clone,
capture_output=True,
text=True,
Expand All @@ -1215,8 +1194,7 @@ def run_clean_clone(
return CleanCloneResult(
ran=True,
command=command,
python=interpreter,
python_version=_interpreter_version(resolved) if resolved else None,
runner=runner,
clone_path=str(clone),
exit_code=result.returncode,
output_tail=output[-CLEAN_CLONE_OUTPUT_TAIL:],
Expand All @@ -1238,7 +1216,7 @@ def clean_clone_check(
result = run_clean_clone(
target_root,
config["test_command"],
config.get("clean_clone_python"),
config.get("verification_runner"),
scratch,
)
finally:
Expand Down Expand Up @@ -1556,14 +1534,14 @@ def revert_check(
cannot be built at all.
"""
command = config["test_command"]
python = config.get("clean_clone_python") or shlex.split(command)[0]
runner = config.get("verification_runner") or shlex.split(command)[0]
resolved = baseline if baseline is not None and baseline.is_dir() else None

if resolved is None:
result = CleanCloneResult(
ran=False,
command=command,
python=python,
runner=runner,
reason=(
"no baseline was captured for the stage, so there is no state "
f"to revert the edits to: {baseline}"
Expand All @@ -1575,7 +1553,7 @@ def revert_check(
result = run_clean_clone(
target_root,
command,
config.get("clean_clone_python"),
config.get("verification_runner"),
scratch,
revert=list(paths),
baseline=resolved,
Expand All @@ -1584,7 +1562,7 @@ def revert_check(
result = CleanCloneResult(
ran=False,
command=command,
python=python,
runner=runner,
reason=f"the clone with the edits reverted could not be built: {error}",
)
finally:
Expand Down Expand Up @@ -2351,6 +2329,21 @@ def _refuse_bad_self_routes(workflow: dict, problems: list[str]) -> int:
)


def _refuse_retired_config_keys(target_root: Path, problems: list[str]) -> int:
"""Refuse a run whose configuration still carries a retired key.

Thin, like every other caller of `refuse`. The configuration is wrong, not
the story and not the tree, so the guidance names the file to edit and the
edit to make.
"""
return refuse(
f"{target_root / '.harness' / 'config.yaml'} carries configuration keys "
f"the harness no longer reads:",
problems,
"Rename each key to its replacement before running a story.",
)


def _refuse_dirty_tree(target_root: Path, paths: list[str]) -> int:
"""Refuse a run whose target tree already holds work no stage produced.

Expand Down Expand Up @@ -2538,6 +2531,17 @@ def run_story(
case and means the repository's own default branch; see `resolve_base`.
"""
config = harness_config.load_config(target_root)

# Pre-flight: a retired configuration key is refused rather than ignored.
# Ignoring one lets the run fall back to the replacement's default and
# quietly exercise something other than what the config asked for. Above
# every other pre-flight, because it is decidable the moment the config
# loads: a refusal here leaves no run directory, no state.json, no log, no
# new branch, and invokes no agent.
retired = harness_config.retired_config_problems(config)
if retired:
return _refuse_retired_config_keys(target_root, retired)

workflow = harness_config.load_workflow(harness_root, config.get("workflow", "story-workflow"))
rules = harness_config.load_rules(harness_root)
stages = workflow["stages"]
Expand Down
14 changes: 5 additions & 9 deletions schemas/clean-clone-result.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,19 @@
"title": "clean-clone-result",
"description": "The coordinator's record of the clean-clone check: the configured test command run a second time in a fresh clone of the repository with the story committed into it, after the verifier passes and before the documenter runs. Coordinator-written rather than stage-written, so it appears in no stage's schemas map and no agent is asked to satisfy it; the artifact exists so a reader can tell the check ran rather than inferring it from a pass. Optional fields are expressed by absence rather than by null, as execution-history does: a check that refused to run has no exit code to report.",
"type": "object",
"required": ["ran", "command", "python"],
"required": ["ran", "command", "runner"],
"properties": {
"ran": {
"type": "boolean",
"description": "Whether the suite actually ran in the clone. False when the check refused to run, in which case reason says why and exit_code, output_tail and python_version are absent."
"description": "Whether the suite actually ran in the clone. False when the check refused to run, in which case reason says why and exit_code and output_tail are absent."
},
"command": {
"type": "string",
"description": "The command executed with the clone as its working directory, taken from the target repository's configured test_command with its interpreter replaced by the one named below."
"description": "The command executed with the clone as its working directory, taken from the target repository's configured test_command with its first word replaced by the executable named below."
},
"python": {
"runner": {
"type": "string",
"description": "The interpreter the run used: .harness/config.yaml's clean_clone_python when that key is set, and test_command's own interpreter otherwise."
},
"python_version": {
"type": "string",
"description": "The version that interpreter reported, so a reader can tell which Python the check exercised rather than assuming it matched CI. Absent when the interpreter reported no recognizable version, which is what a test command that is not a Python interpreter does."
"description": "The executable the run used: .harness/config.yaml's verification_runner when that key is set, and test_command's own first word otherwise."
},
"clone_path": {
"type": "string",
Expand Down
8 changes: 4 additions & 4 deletions schemas/harness-config.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,6 @@
"type": "string",
"description": "Prepended to a story id to form the story branch name. Defaults to story/."
},
"clean_clone_python": {
"type": "string",
"description": "The interpreter the clean-clone and revert checks run the configured test command under. Unset, the check falls back to test_command's own leading word."
},
"logs_dir": {
"type": "string",
"description": "Directory, relative to the target root, holding raw agent output logs. Defaults to .harness/logs."
Expand Down Expand Up @@ -55,6 +51,10 @@
"type": "string",
"description": "The command the clean-clone and revert checks run inside a scratch clone. Read without a fallback: a target that omits it cannot run either check."
},
"verification_runner": {
"type": "string",
"description": "The executable the clean-clone and revert checks run the configured test command under: it replaces the first word of test_command, and every remaining argument is the configured command's own. Unset, the checks use test_command's own first word, so the command runs exactly as written. An environment difference that cannot be expressed as a first-word swap belongs in test_command instead."
},
"workflow": {
"type": "string",
"description": "The name of the workflow definition under workflows/ that a run executes. Defaults to story-workflow."
Expand Down
14 changes: 5 additions & 9 deletions schemas/revert-check-result.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@
"title": "revert-check-result",
"description": "The coordinator's record of the revert check: after a stage that declares both a changed-files record and a may_not_create list, its own modifications and deletions under those prefixes are permitted iff reverting them makes the suite fail. A forced repair breaks the suite when reverted; new coverage does not. Coordinator-written rather than stage-written, so it appears in no stage's schemas map and no agent is asked to satisfy it, and nothing in orchestration routes on it — it is evidence, like clean-clone-result. GRANULARITY: the check reverts every governed path at once and decides on that single run of the suite. A set containing one forced repair is therefore permitted in full, including any added coverage sitting in the other files of that set, and a single file mixing a forced repair with added coverage is not caught at all. The paths field states exactly what was reverted, so a reader can tell what the decision covered rather than assuming it discriminated per file. Optional fields are expressed by absence rather than by null, as clean-clone-result does: a check that could not run decided nothing.",
"type": "object",
"required": ["ran", "paths", "command", "python"],
"required": ["ran", "paths", "command", "runner"],
"properties": {
"ran": {
"type": "boolean",
"description": "Whether the suite actually ran in the clone with the edits reverted. False when the check could not run, in which case reason says why and permitted, exit_code, output_tail and python_version are absent."
"description": "Whether the suite actually ran in the clone with the edits reverted. False when the check could not run, in which case reason says why and permitted, exit_code and output_tail are absent."
},
"paths": {
"type": "array",
Expand All @@ -16,20 +16,16 @@
},
"command": {
"type": "string",
"description": "The command executed with the clone as its working directory, taken from the target repository's configured test_command with its interpreter replaced by the one named below."
"description": "The command executed with the clone as its working directory, taken from the target repository's configured test_command with its first word replaced by the executable named below."
},
"python": {
"runner": {
"type": "string",
"description": "The interpreter the run used: .harness/config.yaml's clean_clone_python when that key is set, and test_command's own interpreter otherwise."
"description": "The executable the run used: .harness/config.yaml's verification_runner when that key is set, and test_command's own first word otherwise."
},
"permitted": {
"type": "boolean",
"description": "Whether the edits are permitted. True when the suite failed with every path above reverted, which is what makes the set maintenance the change forced rather than validation the stage authored. False escalates the run immediately, without incrementing retry_count. Absent when ran is false, because a check that could not run permitted nothing and refused nothing."
},
"python_version": {
"type": "string",
"description": "The version that interpreter reported, so a reader can tell which Python the check exercised. Absent when the interpreter reported no recognizable version, which is what a test command that is not a Python interpreter does."
},
"clone_path": {
"type": "string",
"description": "Where the clone was built, under a temporary directory outside the target repository. The directory is removed once the run completes, so this identifies the run rather than naming a path to visit. Absent when no clone was built."
Expand Down
Loading
Loading