From 6b432b640ab9e549ca17b57db98ed4cc93d85591 Mon Sep 17 00:00:00 2001 From: "jerod.wilkerson" <30474318+jerodw@users.noreply.github.com> Date: Sat, 15 Aug 2026 11:35:44 -0600 Subject: [PATCH] story-039: Every configurable value is proven configurable Implemented by the l5 harness story workflow. --- .harness/docs/ARCHITECTURE.md | 8 +- orchestration/harness_config.py | 42 + schemas/harness-config.schema.json | 63 ++ schemas/manifest.json | 1 + tests/test_config_keys_are_obeyed.py | 1199 ++++++++++++++++++++++++++ 5 files changed, 1312 insertions(+), 1 deletion(-) create mode 100644 schemas/harness-config.schema.json create mode 100644 tests/test_config_keys_are_obeyed.py diff --git a/.harness/docs/ARCHITECTURE.md b/.harness/docs/ARCHITECTURE.md index e69bdf0..836fb7e 100644 --- a/.harness/docs/ARCHITECTURE.md +++ b/.harness/docs/ARCHITECTURE.md @@ -60,6 +60,12 @@ Since story-032 the `create` value is **a path at or beneath** one of that stage `technical_plan` deliberately carries no `type` keyword. story-001 and story-002 write it as a free-form block scalar and story-003 onward write it as a structured object; the validator subset has no union keyword, and those artifacts are committed and unedited. Omitting `type` keeps every nested constraint biting — `schema_validator` applies `properties`/`required` only when the value is a dict — so a malformed *object* form is still rejected while a block scalar is accepted. The reason is recorded in that property's own `description`. +One file in the inventory declares a *contract* rather than an artifact shape. Since story-039 the set of configuration keys the harness reads is declared in **`schemas/harness-config.schema.json`** — one property per key, thirteen of them (`allowed_tools`, `architecture_docs`, `base_branch`, `branch_prefix`, `clean_clone_python`, `logs_dir`, `model`, `permission_mode`, `runs_dir`, `standards_dir`, `stories_dir`, `test_command`, `workflow`), each typed as `load_config` produces it and described by what it governs and what it falls back to. It is in `schemas/manifest.json` like every other file there, and passes the same parametrized draft-2020-12, unsupported-keyword and no-`additionalProperties` checks, but no stage is asked to satisfy it because no agent produces a config file. `harness_config.declared_config_keys(harness_root=None)` is its **only reader**, resolving it relative to its own module through `schema_validator.load_schema` so `schemas/` keeps one reader, and raising `ValueError` naming the path on a missing, unparseable or wrong-shaped schema rather than degrading to an empty or partial tuple — a degraded return would make the coverage below vacuous instead of red. + +The declaration is **not a run-time check**. Nothing calls `declared_config_keys` while a run executes, no target's `.harness/config.yaml` is validated against it, and no unknown key is refused — a target carrying an extra key runs exactly as it did before the file existed. `project` is the live example of a key a config file carries and nothing reads, and it is deliberately undeclared: the declared set *is* the set of keys the harness reads, so a key nothing reads is out of the set by construction rather than by oversight. + +What the declaration is for is coverage, and the coverage is **set equality in both directions**, twice, in `tests/test_config_keys_are_obeyed.py`. Against `KEY_PROOFS` — a key declared with no proof fails, a proof naming an undeclared key fails — and against an AST scan of `orchestration/` and `scripts/` collecting every `config.get("...")` and `config["..."]`, extensionless scripts included, so a key the harness reads and the schema does not declare fails. Neither comparison is against a second maintained list. Every declared key carries a proof that **varies** it: the value is one the harness would never pick (each carries the token `xyzzy`, and an assertion checks that none coincides with the key's default or with this repository's own configured value), and ten keys are proven behaviourally while `model`, `permission_mode` and `allowed_tools` are proven on the invocation built for a fake runner, because those three are handed to the agent runner and observable nowhere else. `KEY_PROOFS` records which of the two each key gets, so a reader knows what *proven* means for it. A mutation control then replaces each key's read with its fallback literal in a throwaway copy of `orchestration/` and requires that key's proof to go red there, so a proof that sets a key and asserts nothing about its effect cannot survive. This subsumes but does not retire story-028's absence assertions: proving a literal is *absent* and proving the configured value *governs* are different claims, and both are kept. + ### Prompts (`prompts/`) One reusable template per agent role: `planner.md`, `implementer.md`, `tester.md`, `verifier.md`, `documenter.md`, `assist.md`. Each follows the five-layer structure: harness layer (durable rules shared by every agent), role layer (responsibilities and do-not boundaries), workflow layer (workflow priorities), stage layer (current objective), and runtime state layer (`{{placeholder}}` fields the coordinator fills at runtime). Optional placeholders render as `None` when nothing applies. @@ -130,7 +136,7 @@ Outliving the session is what made **plan-time validation** possible, and story- ### Target-repository state (`.harness/`) -- `config.yaml` — repository-specific settings (branch prefix, model, permission mode, workflow name, `test_command`, and `clean_clone_python`), plus the optional `base_branch` key story-030 added, shipped as a commented example. Leaving it unset is the normal case: the base is then the repository's own default branch, read from `refs/remotes/origin/HEAD`. Set it only where the base is not the git default. +- `config.yaml` — repository-specific settings (branch prefix, model, permission mode, workflow name, `test_command`, and `clean_clone_python`), plus the optional `base_branch` key story-030 added, shipped as a commented example. The keys the harness reads are declared in `schemas/harness-config.schema.json` and each is proven to govern (see the `schemas/` section); the file itself is validated against nothing at run time. Leaving `base_branch` unset is the normal case: the base is then the repository's own default branch, read from `refs/remotes/origin/HEAD`. Set it only where the base is not the git default. - `standards/` — repository standards (architecture, coding, testing) that verifiers evaluate against. - `stories/` — approved story artifacts produced by `l5-plan`, and validated, committed and pushed by it (see the `plan_validation.py` and `plan_commit.py` bullets above). The directory is what `snapshot` watches; its location comes from the config's `stories_dir`, defaulting to `.harness/stories`. - `runs//` — per-run state, events, and artifacts (not committed). diff --git a/orchestration/harness_config.py b/orchestration/harness_config.py index 23cc8e6..9ba261f 100644 --- a/orchestration/harness_config.py +++ b/orchestration/harness_config.py @@ -10,6 +10,14 @@ import sys from pathlib import Path +import schema_validator + +# The declaration of which keys the harness reads, beside the artifact +# schemas. It is a declaration and not a run-time check: nothing here +# validates a target's config file against it, and no unknown key is +# refused. +CONFIG_SCHEMA_NAME = "harness-config" + def find_target_root(start: Path) -> Path: for candidate in [start, *start.parents]: @@ -50,6 +58,40 @@ def load_config(target_root: Path) -> dict: return config +def declared_config_keys(harness_root: Path | None = None) -> tuple[str, ...]: + """The keys the harness reads, declared in schemas/harness-config.schema.json. + + The schema ships with the harness code, so it is resolved relative to + this module's package exactly as the artifact schemas are, and read + through schema_validator.load_schema so schemas/ keeps one reader. + + The declaration is not a run-time check. Nothing calls this while a run + is executing, no target's config file is validated against it, and no + unknown key is refused; what reads it is the coverage that asserts set + equality against the keys the harness actually reads. + + A missing, unparseable or wrong-shaped schema raises ValueError naming + the path, rather than degrading to an empty or partial tuple, which + would silently make that coverage vacuous instead of red. + """ + path = schema_validator.schemas_dir(harness_root) / f"{CONFIG_SCHEMA_NAME}.schema.json" + try: + schema = schema_validator.load_schema(CONFIG_SCHEMA_NAME, harness_root) + except OSError as error: + raise ValueError(f"{path} could not be read: {error}") from error + except json.JSONDecodeError as error: + raise ValueError(f"{path} is not parseable as JSON: {error}") from error + if not isinstance(schema, dict): + raise ValueError(f"{path}: expected an object, found {type(schema).__name__}") + properties = schema.get("properties") + if not isinstance(properties, dict) or not properties: + raise ValueError(f"{path}: 'properties' must be a non-empty object of key declarations") + for key in properties: + if not isinstance(key, str) or not key: + raise ValueError(f"{path}: every declared property must be a key name") + return tuple(properties) + + def load_workflow(harness_root: Path, name: str) -> dict: path = harness_root / "workflows" / f"{name}.json" return json.loads(path.read_text(encoding="utf-8")) diff --git a/schemas/harness-config.schema.json b/schemas/harness-config.schema.json new file mode 100644 index 0000000..7c65727 --- /dev/null +++ b/schemas/harness-config.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "harness-config", + "description": "The set of keys the harness reads out of a target repository's .harness/config.yaml. The declared properties are the whole of that set: a key a target's config file carries and no harness code reads is out of the set by construction, which is why 'project' is absent. This file is a declaration rather than a run-time check — nothing validates a target's config file against it during a run and no unknown key is refused, so a target carrying an extra key runs exactly as it would without this schema. It is a contract rather than an artifact shape, so no stage is asked to produce or satisfy it; what holds it is coverage, asserted as set equality against declared_config_keys() in both directions. Every declared key carries a proof that varies it to a value the harness would never pick and observes the harness obey it. Types are the types harness_config.load_config produces: every scalar is a string, because the config parser never coerces, and a list-valued key is an array of strings.", + "type": "object", + "required": [], + "properties": { + "allowed_tools": { + "type": "array", + "description": "Bash command patterns stage agents may run without prompting. Passed to every agent invocation as --allowedTools, and injected into the shared harness-layer prompt partial as {{allowed_tools}}.", + "items": { "type": "string" } + }, + "architecture_docs": { + "type": "array", + "description": "Repository-relative paths of the architecture documents assembled into every stage's context.", + "items": { "type": "string" } + }, + "base_branch": { + "type": "string", + "description": "The branch a story branch is cut from and the branch l5-plan expects to commit a plan artifact onto. Unset, the base is resolved from refs/remotes/origin/HEAD and then from the literal main." + }, + "branch_prefix": { + "type": "string", + "description": "Prepended to a story id to form the story branch name. Defaults to story/." + }, + "clean_clone_python": { + "type": "string", + "description": "The interpreter the clean-clone and revert checks run the configured test command under. Unset, the check falls back to test_command's own leading word." + }, + "logs_dir": { + "type": "string", + "description": "Directory, relative to the target root, holding raw agent output logs. Defaults to .harness/logs." + }, + "model": { + "type": "string", + "description": "The Claude model each stage invocation is run with. Unset, the agent runner passes no model and the developer's default is used." + }, + "permission_mode": { + "type": "string", + "description": "The permission mode each stage invocation is run under. Defaults to acceptEdits." + }, + "runs_dir": { + "type": "string", + "description": "Directory, relative to the target root, holding per-run state, events and artifacts. Defaults to .harness/runs. Also what l5-status lists." + }, + "standards_dir": { + "type": "string", + "description": "Directory, relative to the target root, holding the repository standards injected into every stage's context. Defaults to .harness/standards." + }, + "stories_dir": { + "type": "string", + "description": "Directory, relative to the target root, holding approved story artifacts. Defaults to .harness/stories. Also the directory l5-plan snapshots to decide what a planning session produced." + }, + "test_command": { + "type": "string", + "description": "The command the clean-clone and revert checks run inside a scratch clone. Read without a fallback: a target that omits it cannot run either check." + }, + "workflow": { + "type": "string", + "description": "The name of the workflow definition under workflows/ that a run executes. Defaults to story-workflow." + } + } +} diff --git a/schemas/manifest.json b/schemas/manifest.json index 63b772c..0f0c016 100644 --- a/schemas/manifest.json +++ b/schemas/manifest.json @@ -3,6 +3,7 @@ "changed-files", "clean-clone-result", "execution-history", + "harness-config", "retry-guidance", "retry-history", "revert-check-result", diff --git a/tests/test_config_keys_are_obeyed.py b/tests/test_config_keys_are_obeyed.py new file mode 100644 index 0000000..374f3a2 --- /dev/null +++ b/tests/test_config_keys_are_obeyed.py @@ -0,0 +1,1199 @@ +"""story-039 validation: every configured value is proven to govern. + +Thirteen keys are read out of `.harness/config.yaml`. Until this module, the +suite could not tell a key that is obeyed from a key that was moved into +configuration and then hardcoded to the same literal, because every fixture +in the repository configured the value the harness would have picked anyway. + +So nothing here asserts that a key is *mentioned*. Each key is set to a value +this harness would never choose — every one of them carries the token +``xyzzy`` — and the harness is then observed acting on that value: + +* `KEY_PROOFS` maps each declared key to the node id that proves it and to + what "proven" means for it. Ten keys are proven **behaviourally**: the + fixture configures the varying value and the run is observed following it. + Three — `model`, `permission_mode` and `allowed_tools` — are handed + straight to the agent runner and are observable nowhere else, so their + proof is an **argument-list** assertion on the invocation the coordinator + builds for a fake runner. `allowed_tools` covers both sites that pass it: + the runner call and the rendered prompt. + +* What the key set *is* comes from `schemas/harness-config.schema.json`, + through `harness_config.declared_config_keys()`. Coverage is set equality + against it in both directions — against `KEY_PROOFS`, so a key added with + no proof fails, and against an AST scan of `orchestration/` and `scripts/`, + so a key the harness reads and the schema does not declare fails. Neither + comparison is against a second maintained list. + +Every absence asserted here carries a control that constructs the violation: + +* the AST scan is fed a synthetic module reading a fourteenth key, and + reports it; +* the coverage comparison is fed a schema with an unproven key and a proof + naming an undeclared key, and reports both; +* the "no varying value is a default" assertion is fed a proof value changed + to its default, and reports it; +* and, the control that matters most, for **every** declared key a throwaway + copy of `orchestration/` has that key's read replaced by the literal it + falls back to, and that key's own proof is run there by a real pytest and + required to go red. A proof that set a key and asserted nothing about its + effect would survive the coverage checks and die here. + +Nothing in this module resolves a baseline out of git; the shared resolution +in `tests/conftest.py` is used where history is read at all. Nothing invokes +a model: every run below goes through a fake runner. +""" +from __future__ import annotations + +import ast +import json +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path + +import pytest + +import conftest +import harness_config +import run_status +import schema_validator +import story_coordinator +from agent_runner import AgentResult + +REPO_ROOT = Path(harness_config.__file__).resolve().parents[1] +TESTS_DIR = REPO_ROOT / "tests" +MODULE_NAME = Path(__file__).name +NODE_PREFIX = f"tests/{MODULE_NAME}::" + +#: The declaration. Read once, here, so every comparison below is against the +#: schema rather than against a list this module maintains. +DECLARED = harness_config.declared_config_keys() + +#: This repository's own configuration, which no varying value may coincide +#: with — a proof whose value is what the repository already carries would +#: pass against a hardcoded literal and prove nothing. +THIS_REPO_CONFIG = harness_config.load_config(REPO_ROOT) + + +# -------------------------------------------------------------------------- +# The varying values, the fallbacks they must not equal, and what "proven" +# means for each key +# -------------------------------------------------------------------------- + +#: One distinctive token in every value, so an accidental coincidence with +#: anything the harness would pick is impossible rather than unlikely. +TOKEN = "xyzzy" + +VARYING: dict[str, object] = { + "allowed_tools": ["Bash(xyzzy:*)"], + "architecture_docs": ["docs/xyzzy-architecture.md"], + "base_branch": "xyzzy-base", + "branch_prefix": "xyzzy-branch/", + "clean_clone_python": "/xyzzy/bin/interpreter", + "logs_dir": ".harness/xyzzy-logs", + "model": "xyzzy-model", + "permission_mode": "xyzzyPrompt", + "runs_dir": ".harness/xyzzy-runs", + "standards_dir": ".harness/xyzzy-standards", + "stories_dir": ".harness/xyzzy-stories", + "test_command": "xyzzy-runner --all", + "workflow": "xyzzy-workflow", +} + +#: What the harness uses when the key is absent, as written in the code that +#: reads it. `None` is the answer for the four keys with no fallback at all: +#: `allowed_tools`, `base_branch`, `clean_clone_python` and `model` are read +#: with a bare `config.get`, and `test_command` is read with no default and +#: no fallback, so a target that omits it cannot run the clean-clone check. +FALLBACKS: dict[str, object] = { + "allowed_tools": None, + "architecture_docs": [], + "base_branch": None, + "branch_prefix": "story/", + "clean_clone_python": None, + "logs_dir": ".harness/logs", + "model": None, + "permission_mode": "acceptEdits", + "runs_dir": ".harness/runs", + "standards_dir": ".harness/standards", + "stories_dir": ".harness/stories", + "test_command": None, + "workflow": "story-workflow", +} + +BEHAVIOURAL = "behavioural" +ARGUMENT_LIST = "argument-list" + + +@dataclass(frozen=True) +class Proof: + """Which node proves a key, and what proving it consists of. + + `kind` is recorded rather than left to a reader's inference. Three keys + are handed to the agent runner and never touch the repository, the run + directory or the rendered prompt in a form anything else can observe, so + "proven" for them means the invocation the coordinator built carried the + configured value. Saying so is the honest description of a weaker + observation, not an excuse for it: the mutation control holds those three + to exactly the same standard as the ten behavioural ones. + """ + + node: str + kind: str + + @property + def node_id(self) -> str: + return NODE_PREFIX + self.node + + +KEY_PROOFS: dict[str, Proof] = { + "allowed_tools": Proof( + "test_allowed_tools_reaches_both_the_runner_and_the_rendered_prompt", + ARGUMENT_LIST), + "architecture_docs": Proof( + "test_architecture_docs_names_the_documents_injected_into_a_stage", + BEHAVIOURAL), + "base_branch": Proof( + "test_base_branch_is_the_base_the_pre_flight_resolves_and_decides_on", + BEHAVIOURAL), + "branch_prefix": Proof( + "test_branch_prefix_names_the_branch_the_run_creates_and_works_on", + BEHAVIOURAL), + "clean_clone_python": Proof( + "test_clean_clone_python_is_the_interpreter_the_check_resolves", + BEHAVIOURAL), + "logs_dir": Proof( + "test_logs_dir_is_where_the_stage_log_is_written", + BEHAVIOURAL), + "model": Proof( + "test_model_is_the_model_every_stage_invocation_carries", + ARGUMENT_LIST), + "permission_mode": Proof( + "test_permission_mode_is_the_mode_every_stage_invocation_carries", + ARGUMENT_LIST), + "runs_dir": Proof( + "test_runs_dir_is_where_the_run_state_is_written_and_read_back", + BEHAVIOURAL), + "standards_dir": Proof( + "test_standards_dir_is_where_the_injected_standards_are_read_from", + BEHAVIOURAL), + "stories_dir": Proof( + "test_stories_dir_is_where_the_story_artifact_is_read_from", + BEHAVIOURAL), + "test_command": Proof( + "test_test_command_is_the_command_the_clean_clone_path_builds", + BEHAVIOURAL), + "workflow": Proof( + "test_workflow_names_the_definition_the_run_actually_executes", + BEHAVIOURAL), +} + + +# -------------------------------------------------------------------------- +# The mutations: each key's read replaced by the literal it falls back to +# +# This is what makes every proof above non-vacuous. A proof that configured a +# key and asserted nothing about its effect passes the coverage checks and +# dies here, because the copy it runs in no longer reads the key at all. +# +# Four keys have no fallback, so the substitution is `None` — which is what +# the code would compute if the key were absent. `test_command` has neither a +# default nor a fallback, so its substitution is this repository's own +# configured value: a literal standing where a configured read used to be is +# precisely the defect this story exists to detect. +# -------------------------------------------------------------------------- + +HARDCODED_TEST_COMMAND = '".venv/bin/python -m pytest tests/ -q"' + +MUTATIONS: dict[str, tuple[tuple[str, str, str], ...]] = { + "allowed_tools": ( + ("orchestration/story_coordinator.py", + 'allowed_tools=config.get("allowed_tools"),', + "allowed_tools=None,"), + ), + "architecture_docs": ( + ("orchestration/context_assembler.py", + 'config.get("architecture_docs", [])', + "[]"), + ), + "base_branch": ( + ("orchestration/story_coordinator.py", + 'configured = config.get("base_branch")', + "configured = None"), + ), + "branch_prefix": ( + ("orchestration/story_coordinator.py", + 'config.get("branch_prefix", "story/")', + '"story/"'), + ), + "clean_clone_python": ( + ("orchestration/story_coordinator.py", + 'config.get("clean_clone_python")', + "None"), + ), + "logs_dir": ( + ("orchestration/story_coordinator.py", + 'config.get("logs_dir", ".harness/logs")', + '".harness/logs"'), + ), + "model": ( + ("orchestration/story_coordinator.py", + 'model=config.get("model"),', + "model=None,"), + ), + "permission_mode": ( + ("orchestration/story_coordinator.py", + 'config.get("permission_mode", "acceptEdits")', + '"acceptEdits"'), + ), + "runs_dir": ( + ("orchestration/story_coordinator.py", + 'config.get("runs_dir", ".harness/runs")', + '".harness/runs"'), + ("orchestration/run_status.py", + 'config.get("runs_dir", ".harness/runs")', + '".harness/runs"'), + ), + "standards_dir": ( + ("orchestration/context_assembler.py", + 'config.get("standards_dir", ".harness/standards")', + '".harness/standards"'), + ), + "stories_dir": ( + ("orchestration/story_coordinator.py", + 'config.get("stories_dir", ".harness/stories")', + '".harness/stories"'), + ), + "test_command": ( + ("orchestration/story_coordinator.py", + 'config["test_command"]', + HARDCODED_TEST_COMMAND), + ("orchestration/context_assembler.py", + 'config.get("test_command")', + HARDCODED_TEST_COMMAND), + ), + "workflow": ( + ("orchestration/story_coordinator.py", + 'config.get("workflow", "story-workflow")', + '"story-workflow"'), + ), +} + + +# -------------------------------------------------------------------------- +# The fixture target and the fixture harness root +# -------------------------------------------------------------------------- + +STORY_ID = "story-001" +FIXTURE_WORKFLOW = "xyzzy-workflow" + +#: A stage no shipped workflow defines. Its appearance in the run's stage +#: sequence is what proves the coordinator loaded the *named* definition +#: rather than the one that ships. +AUDIT_STAGE = "xyzzy-auditor" +AUDIT_ARTIFACT = "xyzzy-audit-report.md" + +STANDARDS_MARKER = "xyzzy-standard: the marker that proves standards_dir" +ARCHITECTURE_MARKER = "xyzzy-architecture: the marker that proves architecture_docs" + +PASS_VERDICT = {"status": "passed", "blocking_issues": [], "unverified": [], + "retry_recommended": False} + + +def _yaml_lines(values: dict[str, object]) -> str: + lines = ["# fixture configuration for story-039's proofs", + "project: xyzzy-target"] + for key in sorted(values): + value = values[key] + if isinstance(value, list): + lines.append(f"{key}:") + lines += [f' - "{item}"' for item in value] + else: + lines.append(f"{key}: {value}") + return "\n".join(lines) + "\n" + + +def fixture_config(**overrides: object) -> dict[str, object]: + """The varying value for every key, with per-test departures applied. + + Every key is varied in every fixture. A test that names one key asserts + about that key alone; the rest being varied too is what makes the fixture + a repository the harness has never seen defaults for. + """ + values = dict(VARYING) + values.update(overrides) + return values + + +def build_harness(tmp_path: Path) -> Path: + """A runnable harness root carrying a workflow named for the varying value. + + `prompts/`, `rules/` and `schemas/` are the shipped ones, copied rather + than symlinked so nothing here can reach back into this repository. + `workflows/` carries the shipped definition under the configured name, + with the two suite-executing checks removed — the clean-clone check and + the revert check both run the configured `test_command`, which in this + fixture is deliberately not a command that exists — and with one extra + stage no shipped workflow defines. + """ + root = tmp_path / "xyzzy-harness" + ignore = shutil.ignore_patterns("__pycache__") + for directory in ("prompts", "rules", "schemas"): + shutil.copytree(REPO_ROOT / directory, root / directory, ignore=ignore) + (root / "workflows").mkdir() + + shipped = json.loads( + (REPO_ROOT / "workflows" / "story-workflow.json").read_text(encoding="utf-8")) + shipped["name"] = FIXTURE_WORKFLOW + for stage in shipped["stages"]: + stage.pop("clean_clone", None) + stage.pop("revert_check", None) + shipped["stages"].append({ + "name": AUDIT_STAGE, + "prompt": "documenter.md", + "outputs": [AUDIT_ARTIFACT], + }) + (root / "workflows" / f"{FIXTURE_WORKFLOW}.json").write_text( + json.dumps(shipped, indent=2) + "\n", encoding="utf-8") + return root + + +def build_target(tmp_path: Path, config: dict[str, object], *, + checkout: str | None = None, + extra_branches: tuple[str, ...] = ()) -> Path: + """A target repository configured with `config` and nothing at a default. + + Every directory the configuration names is created at the configured + path and nowhere else, so a read that ignored the configuration would + find nothing rather than finding this repository's own defaults sitting + where it looked. + """ + root = tmp_path / "xyzzy-target" + stories = root / str(config["stories_dir"]) + standards = root / str(config["standards_dir"]) + for directory in (stories, standards, root / "src"): + directory.mkdir(parents=True) + (root / ".harness").mkdir(exist_ok=True) + (root / ".harness" / "config.yaml").write_text(_yaml_lines(config), + encoding="utf-8") + (stories / f"{STORY_ID}.yaml").write_text(conftest.STORY, encoding="utf-8") + (standards / "coding.md").write_text( + f"# Coding Standards\n- {STANDARDS_MARKER}\n", encoding="utf-8") + (standards / "testing.md").write_text( + "# Testing Standards\n- test everything\n", encoding="utf-8") + for relative in config.get("architecture_docs", []): # type: ignore[union-attr] + doc = root / str(relative) + doc.parent.mkdir(parents=True, exist_ok=True) + doc.write_text(f"# Architecture\n{ARCHITECTURE_MARKER}\n", encoding="utf-8") + (root / "src" / "app.py").write_text("print('hello')\n", encoding="utf-8") + + _git(root, "init", "-q") + _git(root, "config", "user.email", "test@example.com") + _git(root, "config", "user.name", "Test") + _git(root, "add", "-A") + _git(root, "commit", "-q", "-m", "initial") + # Named explicitly rather than inherited: `git init`'s default branch + # varies by version and by the developer's own git configuration, and the + # base-branch proof below decides on which branch HEAD is standing. + _git(root, "branch", "-M", "main") + for branch in extra_branches: + _git(root, "branch", branch) + if checkout: + _git(root, "checkout", "-q", checkout) + return root + + +def _git(root: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run(["git", "-C", str(root), *args], + capture_output=True, text=True, check=True) + + +def branches(root: Path) -> set[str]: + listing = _git(root, "branch", "--format=%(refname:short)").stdout + return set(listing.split()) + + +class RecordingRunner: + """Stands in for `agent_runner.run_agent`, recording every invocation. + + It writes each stage's declared artifacts so the run reaches completion, + and it writes to whatever `log_path` it is handed — exactly as the real + runner does — so `logs_dir` is observable as a file on disk rather than + only as an argument. + """ + + def __init__(self, run_dir: Path): + self.run_dir = run_dir + self.calls: list[dict] = [] + + def __call__(self, prompt, *, stage, cwd, log_path, permission_mode, model, + allowed_tools=None): + self.calls.append({ + "stage": stage, "prompt": prompt, "cwd": Path(cwd), + "log_path": Path(log_path), "permission_mode": permission_mode, + "model": model, "allowed_tools": allowed_tools, + }) + Path(log_path).parent.mkdir(parents=True, exist_ok=True) + with open(log_path, "a", encoding="utf-8") as handle: + handle.write(f"===== stage: {stage} =====\n") + if stage == "implementer": + _write_json(self.run_dir / "changed-files.json", + {"modified": ["src/app.py"], "created": [], "deleted": []}) + (self.run_dir / "implementation-summary.md").write_text( + "Did the work.\n", encoding="utf-8") + elif stage == "tester": + _write_json(self.run_dir / "test-results.json", { + "status": "passed", "tests_written": 1, "tests_run": 1, + "tests_passed": 1, "tests_failed": 0, "failures": [], + }) + _write_json(self.run_dir / "tester-changed-files.json", + {"modified": [], "created": ["tests/test_app.py"], + "deleted": []}) + elif stage == "verifier": + _write_json(self.run_dir / "verification-result.json", PASS_VERDICT) + elif stage == "documenter": + (self.run_dir / "documentation-report.md").write_text( + "No changes needed.\n", encoding="utf-8") + elif stage == AUDIT_STAGE: + (self.run_dir / AUDIT_ARTIFACT).write_text( + "Audited.\n", encoding="utf-8") + return AgentResult(ok=True, result_text=f"{stage} done") + + +def _write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +@dataclass +class Run: + """One completed fixture run, and everything a proof reads off it.""" + + target: Path + harness: Path + config: dict + run_dir: Path + runner: RecordingRunner + code: int + values: dict = field(default_factory=dict) + + @property + def stages(self) -> list[str]: + return [call["stage"] for call in self.runner.calls] + + def prompt_for(self, stage: str) -> str: + for call in self.runner.calls: + if call["stage"] == stage: + return call["prompt"] + raise AssertionError(f"{stage} never ran; stages were {self.stages}") + + def argument(self, name: str) -> list: + return [call[name] for call in self.runner.calls] + + @property + def state(self) -> dict: + return json.loads((self.run_dir / "state.json").read_text(encoding="utf-8")) + + +def start_run(tmp_path: Path, **overrides: object) -> Run: + """Build the fixture and execute one story through the fake runner. + + Returns whatever the coordinator returned; callers that need a completed + run use `complete_run`, which asserts on it. + """ + checkout = overrides.pop("_checkout", None) + extra_branches = tuple(overrides.pop("_branches", ()) or ()) + values = fixture_config(**overrides) + harness = build_harness(tmp_path) + target = build_target(tmp_path, values, checkout=checkout, + extra_branches=extra_branches) + config = harness_config.load_config(target) + run_dir = target / str(values["runs_dir"]) / STORY_ID + runner = RecordingRunner(run_dir) + code = story_coordinator.run_story(STORY_ID, harness, target, runner) + return Run(target=target, harness=harness, config=config, run_dir=run_dir, + runner=runner, code=code, values=values) + + +def complete_run(tmp_path: Path, **overrides: object) -> Run: + run = start_run(tmp_path, **overrides) + assert run.code == 0, ( + f"the fixture run did not complete (exit {run.code}); stages were " + f"{run.stages}") + assert run.state["status"] == "completed" + return run + + +def clean_clone_record(run: Run) -> dict: + """What the clean-clone path builds for the fixture's configuration. + + The command is *observed*, not executed: `clean_clone_python` names an + interpreter that does not exist, so `run_clean_clone` reports a check that + did not run, with the command it would have run and the interpreter it + resolved, before any clone is built. That keeps the proof deterministic + and free of any dependency on a second interpreter being installed. + """ + artifact = "xyzzy-clean-clone-result.json" + story_coordinator.clean_clone_check(run.run_dir, run.target, run.config, + artifact) + return json.loads((run.run_dir / artifact).read_text(encoding="utf-8")) + + +# -------------------------------------------------------------------------- +# The declaration and its only reader +# -------------------------------------------------------------------------- + +EXPECTED_KEYS = ( + "allowed_tools", "architecture_docs", "base_branch", "branch_prefix", + "clean_clone_python", "logs_dir", "model", "permission_mode", "runs_dir", + "standards_dir", "stories_dir", "test_command", "workflow", +) + + +def test_declared_config_keys_returns_exactly_the_thirteen_names(): + assert set(DECLARED) == set(EXPECTED_KEYS) + assert len(DECLARED) == len(EXPECTED_KEYS) + + +def test_the_schema_is_in_the_inventory_and_declares_no_key_the_harness_ignores(): + assert "harness-config" in schema_validator.shipped_schemas() + schema = schema_validator.load_schema("harness-config") + assert "additionalProperties" not in json.dumps(schema) + assert schema_validator.unsupported_keywords(schema) == [] + # `project` is the live instance of a key a target's config file carries + # and no harness code reads. Its absence is the schema's top-level claim + # made concrete: the declared set is what the harness *reads*. + assert "project" in THIS_REPO_CONFIG + assert "project" not in DECLARED + + +#: The three shapes `declared_config_keys` must raise on rather than degrade +#: to an empty or partial tuple, each constructed rather than described. +MALFORMED_SCHEMAS = { + "missing": None, + "unparseable": "{ not json", + "not-an-object": "[]", + "no-properties": '{"type": "object"}', + "properties-not-an-object": '{"type": "object", "properties": []}', + "properties-empty": '{"type": "object", "properties": {}}', +} + + +@pytest.mark.parametrize("case", sorted(MALFORMED_SCHEMAS)) +def test_declared_config_keys_raises_rather_than_returning_a_partial_tuple( + case, tmp_path): + root = tmp_path / case + (root / "schemas").mkdir(parents=True) + text = MALFORMED_SCHEMAS[case] + if text is not None: + (root / "schemas" / "harness-config.schema.json").write_text( + text, encoding="utf-8") + with pytest.raises(ValueError) as raised: + harness_config.declared_config_keys(root) + assert "harness-config.schema.json" in str(raised.value) + + +def test_the_same_reader_returns_the_thirteen_names_from_a_copied_schema(tmp_path): + """The positive control for the six refusals above. + + Each of them asserts a raise. That says nothing about whether the reader + can succeed at all against a root it was handed, so one well-formed + throwaway root is read here and must return exactly what this repository's + does. + """ + root = tmp_path / "well-formed" + (root / "schemas").mkdir(parents=True) + shutil.copy2(REPO_ROOT / "schemas" / "harness-config.schema.json", + root / "schemas" / "harness-config.schema.json") + assert harness_config.declared_config_keys(root) == DECLARED + + +# -------------------------------------------------------------------------- +# Coverage: the declared set against KEY_PROOFS, in both directions +# -------------------------------------------------------------------------- + + +def coverage_problems(declared, proofs) -> list[str]: + """What stops `proofs` from covering `declared` exactly. + + A function rather than a pair of inline assertions, so the comparison can + be *shown* to report each direction against a constructed pair rather than + only observed to be silent against the real one. + """ + problems = [] + for key in sorted(set(declared) - set(proofs)): + problems.append(f"{key} is declared in the schema and has no proof") + for key in sorted(set(proofs) - set(declared)): + problems.append(f"{key} has a proof and is not declared in the schema") + return problems + + +def test_every_declared_key_has_a_proof_and_every_proof_names_a_declared_key(): + assert coverage_problems(DECLARED, KEY_PROOFS) == [] + + +def test_the_coverage_comparison_reports_a_declared_key_with_no_proof(): + """The control for the first direction, constructed rather than argued.""" + declared = (*DECLARED, "xyzzy_fourteenth") + assert coverage_problems(declared, KEY_PROOFS) == [ + "xyzzy_fourteenth is declared in the schema and has no proof"] + + +def test_the_coverage_comparison_reports_a_proof_naming_an_undeclared_key(): + """The control for the second direction.""" + proofs = dict(KEY_PROOFS) + proofs["xyzzy_retired"] = Proof("test_nothing", BEHAVIOURAL) + assert coverage_problems(DECLARED, proofs) == [ + "xyzzy_retired has a proof and is not declared in the schema"] + + +def test_every_proof_names_a_function_this_module_actually_defines(): + """Without this, a typo in a node id would make the mutation control lie. + + pytest exits non-zero when it collects nothing, and the mutation control + below reads a non-zero exit as "the proof went red". A node id naming no + function would therefore report a passing control for a proof that never + ran. + """ + defined = {node.name for node in ast.parse( + Path(__file__).read_text(encoding="utf-8")).body + if isinstance(node, ast.FunctionDef)} + assert {proof.node for proof in KEY_PROOFS.values()} <= defined + + +def test_the_three_runner_arguments_are_the_only_proofs_recorded_as_argument_list(): + """AC8's second half, as a fact of the mapping rather than as prose. + + `model`, `permission_mode` and `allowed_tools` are handed to the agent + runner and are observable nowhere else — no file, no branch, no rendered + artifact carries them — so their proof asserts on the invocation. Every + other key changes something a run leaves behind, so nothing else is + entitled to the weaker observation. + """ + recorded = {key for key, proof in KEY_PROOFS.items() + if proof.kind == ARGUMENT_LIST} + assert recorded == {"model", "permission_mode", "allowed_tools"} + assert {proof.kind for proof in KEY_PROOFS.values()} == {BEHAVIOURAL, + ARGUMENT_LIST} + + +# -------------------------------------------------------------------------- +# Coverage: the declared set against what the harness actually reads +# -------------------------------------------------------------------------- + + +def keys_read_in(path: Path) -> set[str]: + """Every literal key read out of a `config` mapping in one source file. + + Both forms the harness uses: `config.get("key")` and `config["key"]`. A + subscript through a *variable* is not a literal read and is not collected + — `harness_config.load_config` builds the mapping with `config[key]`, and + counting that would report the parser's own loop variable as a key. + """ + found: set[str] = set() + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if (isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "get" + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "config" + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str)): + found.add(node.args[0].value) + if (isinstance(node, ast.Subscript) + and isinstance(node.value, ast.Name) + and node.value.id == "config" + and isinstance(node.slice, ast.Constant) + and isinstance(node.slice.value, str)): + found.add(node.slice.value) + return found + + +def sources_to_scan(root: Path = REPO_ROOT) -> list[Path]: + """Every file the scan reads: `orchestration/*.py` and all of `scripts/`. + + The scripts carry no `.py` suffix — they are entry points on PATH — so + they are listed by iterating the directory rather than by globbing an + extension, and parsed as Python text. + """ + sources = sorted((root / "orchestration").glob("*.py")) + sources += sorted(path for path in (root / "scripts").iterdir() + if path.is_file()) + return sources + + +def keys_read_under(paths) -> set[str]: + found: set[str] = set() + for path in paths: + found |= keys_read_in(path) + return found + + +def test_the_scan_reads_the_scripts_that_carry_no_py_suffix(): + scanned = sources_to_scan() + extensionless = [path.name for path in scanned if path.suffix != ".py"] + assert "l5-plan" in extensionless + # l5-plan is the script that reads configuration; if the scan could not + # parse it, the two keys it reads would be invisible. + assert keys_read_in(REPO_ROOT / "scripts" / "l5-plan") == {"workflow", + "stories_dir"} + + +def test_the_keys_the_harness_reads_and_the_keys_the_schema_declares_are_equal(): + read = keys_read_under(sources_to_scan()) + assert read == set(DECLARED) + + +def test_the_scan_reports_a_module_reading_a_key_the_schema_does_not_declare( + tmp_path): + """The control for the scan above, constructed rather than reasoned about. + + A scan that returns nothing new is worth nothing until it has been shown + to return something. So a fourteenth key is planted in a synthetic module + and the same comparison is run over it. + """ + planted = tmp_path / "reads_a_fourteenth_key.py" + planted.write_text( + "def read(config):\n" + ' return config.get("xyzzy_fourteenth"), config["xyzzy_fifteenth"]\n', + encoding="utf-8") + read = keys_read_under([planted]) + assert read == {"xyzzy_fourteenth", "xyzzy_fifteenth"} + assert read - set(DECLARED) == {"xyzzy_fourteenth", "xyzzy_fifteenth"} + + +def test_the_scan_does_not_count_a_subscript_through_a_variable(tmp_path): + """The other half of the scan's claim, also constructed. + + `harness_config.load_config` writes `config[key]` while building the + mapping. Counting that would report `key` and `current_list` as configured + keys, and the equality above would then have to be weakened to accommodate + them — which is how a coverage rule stops meaning anything. + """ + planted = tmp_path / "builds_a_mapping.py" + planted.write_text( + "def build(lines):\n" + " config = {}\n" + " for key, value in lines:\n" + " config[key] = value\n" + " return config\n", + encoding="utf-8") + assert keys_read_under([planted]) == set() + assert keys_read_in(REPO_ROOT / "orchestration" / "harness_config.py") == set() + + +# -------------------------------------------------------------------------- +# No varying value may be one the harness would have picked +# -------------------------------------------------------------------------- + + +def decayed_values(varying: dict, fallbacks: dict, repo_config: dict) -> list[str]: + """Which proof values would pass against a hardcoded literal. + + A value equal to the fallback proves nothing, because the code would + produce it with the key deleted. A value equal to what this repository + already configures proves nothing either, because that is the literal a + hardcoding would most plausibly be. + """ + problems = [] + for key in sorted(varying): + if varying[key] == fallbacks.get(key): + problems.append(f"{key} is set to the value the harness falls back to") + if key in repo_config and varying[key] == repo_config[key]: + problems.append( + f"{key} is set to the value this repository already configures") + return problems + + +def test_no_proof_value_is_a_default_or_this_repositorys_own_configured_value(): + assert decayed_values(VARYING, FALLBACKS, THIS_REPO_CONFIG) == [] + assert set(VARYING) == set(DECLARED) == set(FALLBACKS) + + +def test_every_varying_value_carries_the_distinctive_token(): + for key, value in sorted(VARYING.items()): + rendered = " ".join(value) if isinstance(value, list) else str(value) + assert TOKEN in rendered, key + + +@pytest.mark.parametrize("key", sorted(VARYING)) +def test_the_decay_check_reports_a_proof_value_changed_to_its_fallback(key): + """The first half of the control, for every key rather than for a sample. + + `None` is the fallback for the five keys the code reads with no default, + and setting a proof value to it is the decay the check must report for + those, exactly as a literal default is for the other eight. + """ + decayed = dict(VARYING) + decayed[key] = FALLBACKS[key] + assert f"{key} is set to the value the harness falls back to" in \ + decayed_values(decayed, FALLBACKS, THIS_REPO_CONFIG) + + +@pytest.mark.parametrize( + "key", sorted(k for k in VARYING if k in THIS_REPO_CONFIG)) +def test_the_decay_check_reports_a_proof_value_set_to_this_repositorys_own(key): + """The second half. `base_branch` and `model` are absent from this + repository's configuration, so there is no own-value for them to decay to + and the parametrization does not claim one.""" + decayed = dict(VARYING) + decayed[key] = THIS_REPO_CONFIG[key] + assert f"{key} is set to the value this repository already configures" in \ + decayed_values(decayed, FALLBACKS, THIS_REPO_CONFIG) + + +# -------------------------------------------------------------------------- +# The ten behavioural proofs +# -------------------------------------------------------------------------- + + +def test_branch_prefix_names_the_branch_the_run_creates_and_works_on(tmp_path): + run = complete_run(tmp_path) + assert run.state["branch"] == "xyzzy-branch/story-001" + assert "xyzzy-branch/story-001" in branches(run.target) + assert "story/story-001" not in branches(run.target) + assert story_coordinator.story_branch(run.config, STORY_ID) == \ + "xyzzy-branch/story-001" + + +def test_base_branch_is_the_base_the_pre_flight_resolves_and_decides_on( + tmp_path, capsys): + """Both directions of the base pre-flight, against a configured base. + + The configured base is what `resolve_base` returns and what the pre-flight + decides against. Standing anywhere else is refused by name; standing on it + is accepted. With the key no longer read, the base resolves to `main` and + both halves reverse — the refusal disappears and the acceptance becomes a + refusal — so neither half can pass against a harness that ignores it. + """ + away = start_run(tmp_path / "away", _branches=("xyzzy-base",)) + assert away.code == 1 + refusal = capsys.readouterr().err + assert "xyzzy-base" in refusal + assert "HEAD is on branch main" in refusal + assert not (away.target / str(VARYING["runs_dir"])).exists() + assert story_coordinator.resolve_base(away.target, away.config, None) == \ + "xyzzy-base" + + standing = complete_run(tmp_path / "standing", _branches=("xyzzy-base",), + _checkout="xyzzy-base") + assert standing.state["branch"] == "xyzzy-branch/story-001" + + +def test_stories_dir_is_where_the_story_artifact_is_read_from(tmp_path): + run = complete_run(tmp_path) + assert (run.target / ".harness" / "xyzzy-stories" / + f"{STORY_ID}.yaml").is_file() + assert not (run.target / ".harness" / "stories").exists() + # The story text the run actually read reached the stage prompts, so this + # is the artifact at the configured path governing rather than merely + # sitting there. + assert "Sample story for coordinator tests" in run.prompt_for("implementer") + + +def test_runs_dir_is_where_the_run_state_is_written_and_read_back(tmp_path): + run = complete_run(tmp_path) + assert (run.target / ".harness" / "xyzzy-runs" / STORY_ID / + "state.json").is_file() + assert not (run.target / ".harness" / "runs").exists() + # The status reader resolves the same directory from the same key, so a + # run recorded under the configured path is a run `l5-status` can find. + assert run_status._runs_dir(run.target) == \ + run.target / ".harness" / "xyzzy-runs" + + +def test_logs_dir_is_where_the_stage_log_is_written(tmp_path): + run = complete_run(tmp_path) + expected = run.target / ".harness" / "xyzzy-logs" / f"{STORY_ID}.log" + assert expected.is_file() + assert not (run.target / ".harness" / "logs").exists() + assert run.argument("log_path") == [expected] * len(run.stages) + + +def test_standards_dir_is_where_the_injected_standards_are_read_from(tmp_path): + run = complete_run(tmp_path) + assert STANDARDS_MARKER in run.prompt_for("implementer") + assert not (run.target / ".harness" / "standards").exists() + + +def test_architecture_docs_names_the_documents_injected_into_a_stage(tmp_path): + run = complete_run(tmp_path) + assert ARCHITECTURE_MARKER in run.prompt_for("implementer") + assert "docs/xyzzy-architecture.md" in run.prompt_for("documenter") + assert not (run.target / ".harness" / "docs").exists() + + +def test_workflow_names_the_definition_the_run_actually_executes(tmp_path): + run = complete_run(tmp_path) + # The stage no shipped definition declares. Its presence is what + # distinguishes "the named definition was loaded" from "a definition with + # the same stages as the shipped one was loaded". + assert AUDIT_STAGE in run.stages + assert run.stages == ["implementer", "tester", "verifier", "documenter", + AUDIT_STAGE] + assert (run.run_dir / AUDIT_ARTIFACT).is_file() + assert AUDIT_STAGE not in [ + stage["name"] for stage in harness_config.load_workflow( + REPO_ROOT, "story-workflow")["stages"]] + + +def test_test_command_is_the_command_the_clean_clone_path_builds(tmp_path): + run = complete_run(tmp_path) + assert "xyzzy-runner --all" in run.prompt_for("implementer") + record = clean_clone_record(run) + # The configured command's own arguments, under the configured + # interpreter: `--all` is the half that comes from `test_command`. + assert record["command"] == "/xyzzy/bin/interpreter --all" + + +def test_clean_clone_python_is_the_interpreter_the_check_resolves(tmp_path): + run = complete_run(tmp_path) + record = clean_clone_record(run) + assert record["python"] == "/xyzzy/bin/interpreter" + assert record["ran"] is False + assert "/xyzzy/bin/interpreter" in record["reason"] + + +# -------------------------------------------------------------------------- +# The three argument-list proofs +# -------------------------------------------------------------------------- + + +def test_model_is_the_model_every_stage_invocation_carries(tmp_path): + run = complete_run(tmp_path) + assert run.argument("model") == ["xyzzy-model"] * len(run.stages) + + +def test_permission_mode_is_the_mode_every_stage_invocation_carries(tmp_path): + run = complete_run(tmp_path) + assert run.argument("permission_mode") == ["xyzzyPrompt"] * len(run.stages) + + +def test_allowed_tools_reaches_both_the_runner_and_the_rendered_prompt(tmp_path): + """Both sites that pass the configured grants, in one proof. + + The coordinator reads `allowed_tools` twice: once into the invocation it + builds for the runner, and once into the context the stage's prompt is + rendered from. A site left reading a hardcoded list would fail exactly one + of these two assertions, so both are here. + """ + run = complete_run(tmp_path) + assert run.argument("allowed_tools") == [["Bash(xyzzy:*)"]] * len(run.stages) + assert "- Bash(xyzzy:*)" in run.prompt_for("implementer") + + +# -------------------------------------------------------------------------- +# The mutation control: every proof is run against a harness that stopped +# reading its key, and required to go red +# -------------------------------------------------------------------------- + +#: What the throwaway root needs to run one proof node: the code under +#: mutation, the schema the module reads its key set from, the workflow, +#: prompt and rule files a run loads, this repository's config file (the +#: module reads it to check no proof value coincides with it), and the two +#: test files. Copying only this keeps collection there to one module. +COPIED_TREES = ("orchestration", "schemas", "workflows", "prompts", "rules") +COPIED_TESTS = ("conftest.py", MODULE_NAME) + + +def harness_copy(tmp_path: Path) -> Path: + """A real, runnable copy of the parts of this harness a proof needs. + + Copied rather than symlinked: every module here resolves its own root as + `Path(__file__).resolve().parents[1]`, and `resolve()` follows a symlink + straight back to this repository, which would make every mutation below + invisible. + """ + root = tmp_path / "throwaway-harness" + ignore = shutil.ignore_patterns("__pycache__") + for directory in COPIED_TREES: + shutil.copytree(REPO_ROOT / directory, root / directory, ignore=ignore) + (root / ".harness").mkdir() + shutil.copy2(REPO_ROOT / ".harness" / "config.yaml", + root / ".harness" / "config.yaml") + (root / "tests").mkdir() + for name in COPIED_TESTS: + shutil.copy2(TESTS_DIR / name, root / "tests" / name) + return root + + +def apply_mutation(root: Path, key: str) -> None: + """Replace `key`'s read with the literal the harness falls back to. + + Every anchor must occur, so a mutation whose target has moved fails as + itself rather than as a mutant that silently changed nothing and then + reported the proof green — which would be a control asserting the + opposite of what it means to. + """ + for relative, old, new in MUTATIONS[key]: + path = root / relative + source = path.read_text(encoding="utf-8") + assert old in source, (key, relative, old) + path.write_text(source.replace(old, new), encoding="utf-8") + + +def run_nodes(root: Path, *nodes: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", *nodes], + cwd=root, capture_output=True, text=True) + + +def test_the_pristine_copy_runs_every_proof_green(tmp_path): + """The positive control the whole mutation control rests on. + + Each case below asserts that a mutated copy goes red. That means nothing + unless an unmutated copy goes green for the right reason, so every proof + node runs here in a copy with nothing changed. + """ + root = harness_copy(tmp_path) + result = run_nodes(root, *sorted( + proof.node_id for proof in KEY_PROOFS.values())) + assert result.returncode == 0, result.stdout + result.stderr + assert f"{len(KEY_PROOFS)} passed" in result.stdout + + +@pytest.mark.parametrize("key", sorted(KEY_PROOFS)) +def test_the_proof_for_each_key_goes_red_when_that_key_stops_being_read( + key, tmp_path): + """For every declared key, not for a sample of them. + + This is the assertion that makes the twelve above mean something. A proof + that configured its key and then asserted nothing about its effect would + satisfy the coverage checks, satisfy the no-default check, and pass in a + copy that no longer reads the key at all — and only this reports it. + """ + root = harness_copy(tmp_path) + apply_mutation(root, key) + node = KEY_PROOFS[key].node_id + result = run_nodes(root, node) + assert result.returncode != 0, ( + f"{node} still passed with {key}'s read replaced by the literal the " + f"harness falls back to, so it does not prove {key} is obeyed:\n" + f"{result.stdout}") + # Non-zero is also what pytest returns when it collected nothing, so the + # red is required to be one test that ran and failed rather than a node + # that was never found. + assert "1 failed" in result.stdout, result.stdout + result.stderr + + +# -------------------------------------------------------------------------- +# The inventory holds the new schema, and would not hold it silently +# -------------------------------------------------------------------------- + +INVENTORY_NODES = ( + "tests/test_schema_validator.py::test_shipped_schemas_are_exactly_the_named_ones", + "tests/test_artifact_schemas.py::test_schemas_directory_holds_exactly_the_named_schemas", +) + +INVENTORY_TESTS = ("conftest.py", "test_schema_validator.py", + "test_artifact_schemas.py") + + +def inventory_copy(tmp_path: Path) -> Path: + root = tmp_path / "inventory-harness" + ignore = shutil.ignore_patterns("__pycache__") + for directory in ("orchestration", "schemas"): + shutil.copytree(REPO_ROOT / directory, root / directory, ignore=ignore) + (root / "tests").mkdir() + for name in INVENTORY_TESTS: + shutil.copy2(TESTS_DIR / name, root / "tests" / name) + return root + + +def test_the_inventory_agrees_in_both_directions_with_the_new_schema_present( + tmp_path): + result = run_nodes(inventory_copy(tmp_path), *INVENTORY_NODES) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_removing_the_new_schemas_manifest_line_turns_the_inventory_red(tmp_path): + """The control: the inventory accepts the file because it is *named*. + + Without this, "the manifest names it" and "the inventory happens not to + look" are the same green. + """ + root = inventory_copy(tmp_path) + manifest = root / "schemas" / "manifest.json" + names = json.loads(manifest.read_text(encoding="utf-8"))["schemas"] + assert "harness-config" in names + manifest.write_text( + json.dumps({"schemas": [n for n in names if n != "harness-config"]}, + indent=2) + "\n", encoding="utf-8") + result = run_nodes(root, *INVENTORY_NODES) + assert result.returncode != 0, result.stdout + result.stderr + + +# -------------------------------------------------------------------------- +# The harness itself is unchanged +# -------------------------------------------------------------------------- + +#: The files this story states carry no edit. Compared over the story's own +#: commit range through the shared resolution, never as the working tree +#: against HEAD: the coordinator commits the tree when a run completes, so a +#: HEAD comparison reports clean for every path the moment the story commits. +UNCHANGED = ( + ".harness/config.yaml", + "orchestration/story_coordinator.py", + "orchestration/context_assembler.py", + "orchestration/run_status.py", + "orchestration/schema_validator.py", + "scripts/", + "workflows/", + "prompts/", +) + + +@pytest.mark.parametrize("relative", UNCHANGED) +def test_this_story_left_the_harnesss_own_behaviour_alone(relative): + assert conftest.story_diff([relative], validation_file=Path(__file__)) == "" + + +def test_the_unchanged_comparison_can_tell_a_changed_path_apart(tmp_path): + """The control for the absence above. + + An empty diff is what a comparison bounded at the wrong commits reports + too, so the same resolution is run against a synthetic repository in which + the file really did change, and must report it. + """ + repo = tmp_path / "synthetic" + repo.mkdir() + _git(repo, "init", "-q") + _git(repo, "config", "user.email", "test@example.com") + _git(repo, "config", "user.name", "Test") + (repo / "tests").mkdir() + (repo / "subject.py").write_text("original\n", encoding="utf-8") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "before the story") + (repo / "subject.py").write_text("edited by the story\n", encoding="utf-8") + (repo / "tests" / MODULE_NAME).write_text("# the story's own module\n", + encoding="utf-8") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "the story's run commit") + + changed = conftest.story_diff(["subject.py"], + validation_file=repo / "tests" / MODULE_NAME, + repo=repo) + assert "edited by the story" in changed + unchanged = conftest.story_diff(["tests/"], + validation_file=repo / "tests" / MODULE_NAME, + repo=repo, diff_filter="M") + assert unchanged == "" + + +def test_no_key_gained_a_run_time_check_and_no_unknown_key_is_refused(tmp_path): + """The declaration is a declaration. A target carrying an extra key runs. + + Constructed rather than argued: a fixture target is given a key the schema + does not declare, and the run completes exactly as it does without it. + """ + run = complete_run(tmp_path, xyzzy_undeclared_key="something-nobody-reads") + assert run.config["xyzzy_undeclared_key"] == "something-nobody-reads" + assert "xyzzy_undeclared_key" not in DECLARED + assert run.state["status"] == "completed" + # And nothing in the coordinator's path calls the reader of the schema. + assert "declared_config_keys" not in ( + REPO_ROOT / "orchestration" / "story_coordinator.py" + ).read_text(encoding="utf-8")