diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d197fe58..9239b333 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -95,8 +95,42 @@ part of the release, not documentation about it. ./shipgate doctor --config samples/support_refund_agent/shipgate.yaml ./shipgate scan --config samples/support_refund_agent/shipgate.yaml ./shipgate list-checks +python scripts/regenerate_goldens.py --check ``` +### Sample goldens + +The [golden generator](scripts/regenerate_goldens.py) owns the 24 committed +artifacts in seven `samples/*/expected/` directories. With the normal development +dependencies and Git available: + +```bash +python scripts/regenerate_goldens.py # regenerate all +python scripts/regenerate_goldens.py conductor_agent # one sample +python scripts/regenerate_goldens.py --check # read-only drift check +``` + +It scans disposable copies with the source-tree writers; the caller's working +tree, manifests, declarations and CI summary are not scan output destinations. +Generation validates the entire selected set before writing any expected file. +Exit 0 means generated/matching, 1 means drift in check mode, and 2 means the +recipe could not finish. Every drift names its file. An added expected artifact +requires an explicit recipe; deleting one does not remove it from the check. +`tests/test_regenerate_goldens.py` invokes the actual `--check` command in the +normal CI suite. Existing behavioral assertions remain independent oracles; +do not regenerate away a changed verdict, open question or safety regression. + +The recipe pins packet time, preserves the ordinary and cold-manifest states, +uses relative output paths under the manifest, normalizes only known path +fields, and writes LF bytes. Text inputs in the disposable copy are normalized +to LF too, so a Windows checkout does not change the manifest bytes bound by +the sample pointer. Scans disable installed plugins and isolate inherited Git +configuration; symlinked fixture paths and leaked temporary paths are errors. The +Conductor scan pointer is rebound **after** normalization and has no predecessor +from another generating run; it retains scan-only permissions, never a verifier +receipt or release authority. These development fixtures are not qualification +evidence. #569 still owns the actual report 1.0 freeze and migration fixtures. + ## Contribution Areas - new deterministic checks; @@ -194,6 +228,11 @@ CI runs `python scripts/generate_schemas.py --check` and fails fast with a unified diff if a committed schema drifts from the live model. The same drift is also caught by `tests/test_schema_roundtrip.py`, so your test suite will reject the change locally before CI does. +After a report or contract change, run +`python scripts/regenerate_goldens.py`, inspect the semantic diff, then run +`python scripts/regenerate_goldens.py --check` and the affected behavioral +tests. The [sample recipe](#sample-goldens) owns path and digest normalization; +do not update a version stamp or pointer hash by hand. ## Check Contributions diff --git a/docs/agent-contract-current.md b/docs/agent-contract-current.md index f06c756a..f7e1ee65 100644 --- a/docs/agent-contract-current.md +++ b/docs/agent-contract-current.md @@ -11,6 +11,13 @@ GitHub acquisition and persistence remain an integration obligation. The single, current statement of what AI coding agents and CI integrations should read from Agents Shipgate output. When the contract changes, update [STABILITY.md](../STABILITY.md) first, then this file. Other agent-facing surfaces (`AGENTS.md`, `llms.txt`, `.well-known/agents-shipgate.json`, the slash command, the skill, the FAQ) link here instead of restating field lists. +For contributors changing the contract: regenerate schemas with +`python scripts/generate_schemas.py`, then run +`python scripts/regenerate_goldens.py` and its `--check` mode using the +[committed sample recipe](../CONTRIBUTING.md#sample-goldens). Review actual +artifact changes and preserve behavioral assertions; sample regeneration does +not freeze a release contract or qualify a candidate wheel. + ## Current versions Verify the installed CLI contract locally before relying on hard-coded docs: diff --git a/docs/architecture.md b/docs/architecture.md index be1c089c..65a7e5dd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -207,6 +207,12 @@ Adding a new wire field: edit the relevant `schemas/.py`, run `report_schema_version` / `packet_schema_version` if the addition is public. The CI step `python scripts/generate_schemas.py --check` fails if the committed JSON drifts from the live model. +Regenerate affected sample artifacts with +`python scripts/regenerate_goldens.py` and confirm +`python scripts/regenerate_goldens.py --check`; the +[contributor recipe](../CONTRIBUTING.md#sample-goldens) owns the output paths, +normalization and subsequent scan-pointer rebinding. Keep the independent +semantic assertions when reviewing the resulting golden diff. ## Typed domain types: `Scope`, `SideEffect`, `Action` diff --git a/llms-full.txt b/llms-full.txt index 492689eb..d4759734 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1495,6 +1495,13 @@ GitHub acquisition and persistence remain an integration obligation. The single, current statement of what AI coding agents and CI integrations should read from Agents Shipgate output. When the contract changes, update [STABILITY.md](../STABILITY.md) first, then this file. Other agent-facing surfaces (`AGENTS.md`, `llms.txt`, `.well-known/agents-shipgate.json`, the slash command, the skill, the FAQ) link here instead of restating field lists. +For contributors changing the contract: regenerate schemas with +`python scripts/generate_schemas.py`, then run +`python scripts/regenerate_goldens.py` and its `--check` mode using the +[committed sample recipe](../CONTRIBUTING.md#sample-goldens). Review actual +artifact changes and preserve behavioral assertions; sample regeneration does +not freeze a release contract or qualify a candidate wheel. + ## Current versions Verify the installed CLI contract locally before relying on hard-coded docs: diff --git a/samples/conductor_agent/expected/current-control.json b/samples/conductor_agent/expected/current-control.json index f80a3194..f24c7d32 100644 --- a/samples/conductor_agent/expected/current-control.json +++ b/samples/conductor_agent/expected/current-control.json @@ -2,8 +2,8 @@ "artifacts": { "report": { "path": "report.json", - "sha256": "sha256:5b66dfcf8e408fba258a02babe161d30cde6eee0076e01e52db4cb3873d8e194", - "size_bytes": 51797 + "sha256": "sha256:f9922a1c9bed4f6a2662d4f9cdb1f2c8524c79a3349a9235a974ae2d317197d6", + "size_bytes": 51800 }, "report_markdown": { "path": "report.md", @@ -25,13 +25,13 @@ "reason": "A standalone scan produced the current report set. A scan does not authorize completion or merge; run `agents-shipgate verify` to obtain a merge decision.", "state": "agent_action_required" }, - "current_control_id": "sha256:fb0d14e5557509cb84ccee70672e5c1ea3d6d4745dbeb2aef5c765921d967913", + "current_control_id": "sha256:f96b3a7c66420b891f8f84646f767108dded2022c77b75f5d232285baaae0028", "decision_id": null, "lifecycle_state": "terminal", "operation": "scan", "request_id": null, "schema_version": "shipgate.current_control/v1", - "supersedes": "sha256:96fb38c3a8f0921a586629205730a7fe10af801d9442e73ec2dab7efbebc1d32", + "supersedes": null, "workspace_identity": { "base_commit_sha": null, "base_ref": null, diff --git a/samples/conductor_agent/expected/report.json b/samples/conductor_agent/expected/report.json index 47dad74b..be46d16a 100644 --- a/samples/conductor_agent/expected/report.json +++ b/samples/conductor_agent/expected/report.json @@ -429,7 +429,7 @@ ], "release_consequence": { "decision": "insufficient_evidence", - "summary": "Static evidence is incomplete; capability/intent analysis may miss release-relevant signal — gather deeper sources before shipping.", + "summary": "Static evidence is incomplete; capability/intent analysis may miss release-relevant signal \u2014 gather deeper sources before shipping.", "blocker_misalignment_count": 0, "review_misalignment_count": 2, "fail_policy": { diff --git a/samples/declaration_repair_agent/README.md b/samples/declaration_repair_agent/README.md index 52536c86..37fe6608 100644 --- a/samples/declaration_repair_agent/README.md +++ b/samples/declaration_repair_agent/README.md @@ -120,66 +120,20 @@ action-row route; keeping the two apart also keeps this golden clear of ## Regenerating -Run from the repository root, after any change that moves values: +Run from the repository root: ```bash -python - <<'PY' -import json -from pathlib import Path -from agents_shipgate.cli.scan import run_scan - -sample = Path("samples/declaration_repair_agent") -expected = sample / "expected" -run_scan( - config_path=sample / "shipgate.yaml", - output_dir=Path("expected"), - formats=["json", "markdown"], - ci_mode="advisory", - packet_enabled=False, -) -(expected / "current-control.json").unlink(missing_ok=True) - -golden = expected / "report.json" -payload = json.loads(golden.read_text(encoding="utf-8")) -payload["manifest_dir"] = f"/{sample.as_posix()}" -payload["generated_reports"] = { - fmt: Path(written).as_posix() - for fmt, written in payload["generated_reports"].items() -} -golden.write_text(json.dumps(payload, indent=2), encoding="utf-8", newline="\n") - -# The scan's own writers use the platform newline. Rewrite every golden with -# an explicit LF, whoever produced it. -for name in ("report.md", "suggested-declarations.yaml"): - path = expected / name - path.write_text(path.read_text(encoding="utf-8"), encoding="utf-8", newline="\n") -PY +python scripts/regenerate_goldens.py declaration_repair_agent +python scripts/regenerate_goldens.py --check declaration_repair_agent ``` -This is the sibling's recipe with the sample path swapped, and every -normalization in it is load-bearing for the same reasons — the four notes under -[`google_adk_cold_start_agent` § Regenerating the goldens](../google_adk_cold_start_agent/README.md) -apply here verbatim. In short: - -- the path rewrite is **structural**, because a textual `` replace is a - silent no-op on Windows: `json.dumps` escapes the separators, so the file - holds `C:\\repo\\samples\…` while `os.getcwd()` is `C:\repo\samples\…` and - the two never match — leaving an absolute `manifest_dir` that fails - `test_sample_expected_report_json_uses_repo_placeholder_for_manifest_dir` on - the machine that produced the golden; -- `generated_reports` needs `.as_posix()`, or a Windows run commits - `expected\report.json` and churns against every other platform; -- all three goldens are rewritten with an explicit `newline="\n"`, because - every writer involved opens text mode with `newline=None` and - `.gitattributes` pins `samples/**/expected/** -text`, so Git stores whatever - bytes were produced. No byte comparison can see it — `read_text` normalizes - CRLF on the way in — which is why - `test_sample_expected_goldens_are_committed_with_lf_newlines` reads raw bytes. - -An earlier draft of this section gave the first two as prose and omitted the -third entirely, which made the repo's designated recovery path — named in -`test_repair_scaffold_matches_its_golden`'s own failure message — the -counterexample to that guard's docstring (#465 review). +The [shared recipe](../../scripts/regenerate_goldens.py) runs the existing +scanner against a disposable copy of the committed inputs. It owns relative +output paths, structural path normalization and LF bytes; see +[the contributor recipe](../../CONTRIBUTING.md#sample-goldens). It does not +change the challenged declarations, execute sample tools or apply the suggested +questionnaire. CI calls the same check mode and retains the existing semantic +and questionnaire assertions. Read the diff on `expected/suggested-declarations.yaml` before committing: a change in the `risk_tags:` values a block publishes is a change to the remedy diff --git a/samples/declaration_repair_agent/expected/report.json b/samples/declaration_repair_agent/expected/report.json index 682ab27e..54c9cfd2 100644 --- a/samples/declaration_repair_agent/expected/report.json +++ b/samples/declaration_repair_agent/expected/report.json @@ -757,7 +757,7 @@ "misalignments": [], "release_consequence": { "decision": "insufficient_evidence", - "summary": "Static evidence is incomplete; capability/intent analysis may miss release-relevant signal — gather deeper sources before shipping.", + "summary": "Static evidence is incomplete; capability/intent analysis may miss release-relevant signal \u2014 gather deeper sources before shipping.", "blocker_misalignment_count": 0, "review_misalignment_count": 0, "fail_policy": { diff --git a/samples/google_adk_cold_start_agent/README.md b/samples/google_adk_cold_start_agent/README.md index 3dddacf9..d569b454 100644 --- a/samples/google_adk_cold_start_agent/README.md +++ b/samples/google_adk_cold_start_agent/README.md @@ -120,132 +120,31 @@ questionnaire is here to own. `test_a_heuristic_cannot_propose_that_an_action_is ## Regenerating the goldens -Run from the repository root, after any change that moves values: +Run from the repository root: ```bash -python - <<'PY' -import json -from pathlib import Path -from agents_shipgate.cli.scan import run_scan - -sample = Path("samples/google_adk_cold_start_agent") -expected = sample / "expected" -run_scan( - config_path=sample / "shipgate.yaml", - output_dir=Path("expected"), - formats=["json", "markdown"], - ci_mode="advisory", - packet_enabled=False, -) -(expected / "current-control.json").unlink(missing_ok=True) - -golden = expected / "report.json" -payload = json.loads(golden.read_text(encoding="utf-8")) -payload["manifest_dir"] = f"/{sample.as_posix()}" -payload["generated_reports"] = { - fmt: Path(written).as_posix() - for fmt, written in payload["generated_reports"].items() -} -golden.write_text(json.dumps(payload, indent=2), encoding="utf-8", newline="\n") - -# The scan's own writers use the platform newline. Rewrite every golden with -# an explicit LF, whoever produced it. -for name in ("report.md", "suggested-declarations.yaml"): - path = expected / name - path.write_text(path.read_text(encoding="utf-8"), encoding="utf-8", newline="\n") -PY +python scripts/regenerate_goldens.py google_adk_cold_start_agent +python scripts/regenerate_goldens.py --check google_adk_cold_start_agent ``` -### Regenerating the cold report - -`expected/cold-report.md` comes from a different repository state than the -ordinary goldens: the agent sources are committed, while `shipgate.yaml` -exists only in the worktree. Recreate that state in a temporary repository; -running the ordinary recipe above cannot produce the cold-reader order. - -```bash -python - <<'PY' -import shutil -import subprocess -import tempfile -from pathlib import Path - -from agents_shipgate.cli.scan import run_scan - -source = Path("samples/google_adk_cold_start_agent").resolve() -golden = source / "expected" / "cold-report.md" - -with tempfile.TemporaryDirectory(prefix="shipgate-cold-golden-") as temp: - repo = Path(temp) / "repo" - shutil.copytree(source, repo, ignore=shutil.ignore_patterns("expected")) - - def git(*args: str) -> None: - subprocess.run( - ["git", "-C", str(repo), *args], - check=True, - capture_output=True, - ) - - git("init", "-q") - git("config", "user.name", "Shipgate Golden") - git("config", "user.email", "shipgate@example.invalid") - git("add", "agent.py", "inventories", "specs") - git("commit", "-qm", "base without manifest") - - out = repo / "reports" - run_scan( - config_path=repo / "shipgate.yaml", - output_dir=out, - formats=["markdown", "json"], - ci_mode="advisory", - ) - golden.write_text( - (out / "report.md").read_text(encoding="utf-8"), - encoding="utf-8", - newline="\n", - ) -PY -``` - -Four things that look like details and are not. - -**Newlines are forced, not inherited.** `Path.write_text(..., encoding="utf-8")` -opens in text mode with `newline=None`, so on Windows every `\n` is written as -`\r\n` — by the recipe *and* by `write_json_report`, the Markdown writer and the -questionnaire writer. `.gitattributes` pins `samples/**/expected/** -text` -precisely so Git hands those bytes over unchanged, so a golden regenerated on -Windows would be committed as CRLF against everyone else's LF. The tests cannot -see it: `read_text()` applies universal newlines and normalizes CRLF back to LF -on the way in, so every byte comparison in this repo passes on a file whose -bytes moved. `newline="\n"` on all three is what makes the artifact the same -artifact everywhere, and -`test_sample_expected_goldens_are_committed_with_lf_newlines` reads the raw -bytes so the guard does not share the blindness. - -**The path normalization is structural, not textual.** An earlier version did -`text.replace(os.getcwd(), "")`, which works on POSIX and is a no-op on -Windows: `json.dumps` escapes the separators, so the file holds -`C:\\repo\\samples\\…` while `os.getcwd()` is `C:\repo\samples\…` and the two -never match. The golden then keeps an absolute `manifest_dir` and fails -`test_sample_expected_report_json_uses_repo_placeholder_for_manifest_dir` — on -the machine that produced it. Assigning the field a POSIX value is correct on -both, and `Path(written).as_posix()` does the same for `generated_reports`, -which would otherwise commit `expected\report.json` and churn against every -other platform. - -`json.dumps(payload, indent=2)` is exactly what `write_json_report` uses — no -`sort_keys`, no trailing newline — so the round trip is byte-identical and -nothing but the two normalized fields moves. - -**`output_dir` is the relative `"expected"`**, which `run_scan` resolves under -the manifest directory rather than under the process directory. The report -records where it wrote itself, so scanning into an absolute temporary directory -and copying the files back bakes a contributor's `/var/folders/…/tmp…` path -into `generated_reports` — a value no test compared, which is exactly why it -sat there churning until #425's review. - -**The `unlink` is not tidying.** A scan also publishes `current-control.json`, -and this fixture deliberately does not commit one: the hash-bound pointer path -is covered by [`conductor_agent`](../conductor_agent/), and a second copy would -have to be rebound *after* the normalization every time, since that rewrite -changes both the length and the digest of `report.json`. +The [committed recipe](../../scripts/regenerate_goldens.py) builds both +repository states in disposable copies. The ordinary report has a committed +manifest; `cold-report.md` commits only `agent.py`, `inventories` and `specs`, +leaving the existing manifest untracked. It never fills declaration blanks or +changes the sample's inputs. Those different states are why an ordinary scan +cannot regenerate the cold-reader ordering. + +The script uses relative `output_dir="expected"`, which the scanner resolves +under the copied manifest directory. It normalizes `manifest_dir` structurally +to `/samples/google_adk_cold_start_agent` and report paths to POSIX, +forces LF bytes on every output, and keeps the JSON writer's field order and +terminator. Absolute output paths or surviving generating-machine paths stop +generation before any repository artifact is written. + +This fixture does not publish a control pointer. The Conductor recipe covers +that contract by rebinding its scan-only pointer after every normalized byte +is final; it cannot authorize merge or completion. See +[the contributor recipe](../../CONTRIBUTING.md#sample-goldens) for all samples. + +Review the semantic assertions above alongside any golden diff. Closing the +open questions or changing a declaration is not a regeneration technique. diff --git a/samples/support_refund_agent/expected/packet.json b/samples/support_refund_agent/expected/packet.json index 504143de..996b528b 100644 --- a/samples/support_refund_agent/expected/packet.json +++ b/samples/support_refund_agent/expected/packet.json @@ -2541,8 +2541,8 @@ "total": 2 }, "declaration_review": { - "base_kind": "none", "base_comparison_requested": false, + "base_kind": "none", "changed_count": 0, "enabled": false, "notes": [ @@ -3037,4 +3037,4 @@ "unchanged_findings": 0 } } -} \ No newline at end of file +} diff --git a/samples/support_refund_agent/expected/summary.json b/samples/support_refund_agent/expected/summary.json index 1e369c61..b8c4c63f 100644 --- a/samples/support_refund_agent/expected/summary.json +++ b/samples/support_refund_agent/expected/summary.json @@ -1,6 +1,6 @@ { "status": "release_blockers_detected", - "critical_count": 2, - "high_count": 14, + "critical_count": 3, + "high_count": 10, "medium_count": 2 } diff --git a/scripts/regenerate_goldens.py b/scripts/regenerate_goldens.py new file mode 100644 index 00000000..4c6127a9 --- /dev/null +++ b/scripts/regenerate_goldens.py @@ -0,0 +1,310 @@ +"""Rebuild the shipped sample artifacts with the current source-tree writers. + + python scripts/regenerate_goldens.py + python scripts/regenerate_goldens.py --check + python scripts/regenerate_goldens.py conductor_agent + +Only synthetic samples are copied and committed in disposable repositories. +No source manifest, declaration, release receipt or qualification input is edited. +""" + +from __future__ import annotations + +import argparse +import contextlib +import html +import json +import os +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Iterator +from pathlib import Path, PurePosixPath, PureWindowsPath + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from agents_shipgate.cli.scan import run_scan # noqa: E402 +from agents_shipgate.core.current_control import publish_current_control # noqa: E402 +from agents_shipgate.report.markdown import _safe_markdown_text # noqa: E402 +from agents_shipgate.schemas.current_control import CurrentControlPointer # noqa: E402 + +GENERATED_AT = "2026-01-01T00:00:00+00:00" +# Explicit ownership: dropping a committed artifact must not silently shrink +# the check, and a new fixture needs a reviewed recipe, not guessed options. +REPORTS = ("report.json", "report.md") +RECIPES = { + "conductor_agent": ( + *REPORTS, + "current-control.json", + "suggested-inventory.json", + "summary.json", + ), + "declaration_repair_agent": (*REPORTS, "suggested-declarations.yaml"), + "google_adk_cold_start_agent": (*REPORTS, "suggested-declarations.yaml", "cold-report.md"), + "simple_crewai_agent": REPORTS, + "simple_langchain_agent": REPORTS, + "simple_openai_api_agent": REPORTS, + "support_refund_agent": (*REPORTS, "packet.json", "packet.md", "packet.html", "summary.json"), +} + + +@contextlib.contextmanager +def _scan_environment() -> Iterator[None]: + # This output sink changes privacy_audit and can write to the caller's CI + # summary. Preserve the environment around the entirely local generation. + keys = {key for key in os.environ if key.startswith("GIT_")} | {"GITHUB_STEP_SUMMARY"} + saved = {key: os.environ[key] for key in keys if key in os.environ} + for key in keys: + os.environ.pop(key, None) + os.environ.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM="1") + try: + yield + finally: + for key in keys | {"GIT_CONFIG_GLOBAL", "GIT_CONFIG_NOSYSTEM"}: + os.environ.pop(key, None) + os.environ.update(saved) + + +def _git(repo: Path, *args: str) -> None: + # Ignore inherited repository handles; never execute contributor hooks or + # signing helpers while creating the disposable fixture's committed state. + env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} + env.update( + GIT_AUTHOR_DATE=GENERATED_AT, + GIT_COMMITTER_DATE=GENERATED_AT, + GIT_CONFIG_GLOBAL=os.devnull, + GIT_CONFIG_NOSYSTEM="1", + ) + subprocess.run( + [ + "git", + "-C", + str(repo), + "-c", + "core.hooksPath=.git/no-golden-hooks", + "-c", + "commit.gpgsign=false", + "-c", + "core.autocrlf=false", + "-c", + "user.name=Shipgate Golden", + "-c", + "user.email=shipgate@example.invalid", + *args, + ], + check=True, + capture_output=True, + env=env, + ) + + +def _copy_sample(source: Path, repo: Path, *, cold: bool = False) -> None: + # All recipe inputs are committed repository fixtures, not arbitrary paths. + if source.is_symlink() or any(path.is_symlink() for path in source.rglob("*")): + raise ValueError(f"{source.name}: sample inputs must not be symlinks") + shutil.copytree( + source, + repo, + ignore=shutil.ignore_patterns("expected", "__pycache__", ".git", "agents-shipgate-reports"), + ) + # Git may check out these text inputs as CRLF on Windows. Scan the same LF + # fixture bytes on every host, including the manifest that the pointer hashes. + for path in repo.rglob("*"): + if path.is_file(): + _lf(path) + _git(repo, "init", "-q") + if cold: + _git(repo, "add", "--", "agent.py", "inventories", "specs") + else: + _git(repo, "add", "--", ".") + _git(repo, "commit", "-qm", "synthetic golden inputs") + + +def _fixture_path(root: Path, relative: Path) -> Path: + path = root + for part in relative.parts: + path = path / part + if path.is_symlink(): + raise ValueError(f"{relative.as_posix()}: symlinked fixture path") + if not path.resolve().is_relative_to(root): + raise ValueError(f"{relative.as_posix()}: fixture path escapes repository") + return path + + +def _lf(path: Path) -> None: + path.write_bytes(path.read_text(encoding="utf-8").encode("utf-8")) + + +def _normalize_report(path: Path, sample: str) -> None: + payload = json.loads(path.read_text(encoding="utf-8")) + if Path(payload["manifest_dir"]).resolve() != path.parent.parent.resolve(): + raise ValueError(f"{sample}/expected/report.json: unexpected manifest_dir") + payload["manifest_dir"] = f"/samples/{sample}" + for key, value in payload["generated_reports"].items(): + if PurePosixPath(value).is_absolute() or PureWindowsPath(value).is_absolute(): + raise ValueError(f"{sample}/expected/report.json: absolute generated_reports.{key}") + # These are host-generated filesystem paths, not arbitrary report text. + normalized = Path(value).as_posix() + if PurePosixPath(normalized).parts != ("expected", Path(normalized).name): + raise ValueError(f"{sample}/expected/report.json: unexpected output path {value!r}") + payload["generated_reports"][key] = normalized + # Match the product JSON writer, including its missing final newline. + path.write_bytes(json.dumps(payload, indent=2).encode("utf-8")) + + +def _assert_no_machine_paths(path: Path, *roots: Path) -> None: + text = path.read_text(encoding="utf-8") + for root in roots: + for spelling in ( + str(root), + root.as_posix(), + str(root.resolve()), + root.resolve().as_posix(), + ): + if any( + rendered in text + for rendered in ( + spelling, + json.dumps(spelling)[1:-1], + _safe_markdown_text(spelling), + html.escape(spelling), + ) + ): + raise ValueError(f"{path.name}: generating-machine path leaked into an artifact") + + +def _build_sample(root: Path, sample: str, temp: Path) -> dict[Path, bytes]: + source = root / "samples" / sample + repo = temp / sample + _copy_sample(source, repo) + out = repo / "expected" + report, _ = run_scan( + config_path=repo / "shipgate.yaml", + output_dir=Path("expected"), + formats=["markdown", "json"], + ci_mode="advisory", + packet_enabled=False, + plugins_enabled=False, + ) + _normalize_report(out / "report.json", sample) + _lf(out / "report.md") + if "summary.json" in RECIPES[sample]: + # This existing compact fixture predates the full ReportSummary model. + summary = report.summary.model_dump(mode="json") + summary = { + key: summary[key] for key in ("status", "critical_count", "high_count", "medium_count") + } + (out / "summary.json").write_bytes((json.dumps(summary, indent=2) + "\n").encode("utf-8")) + if "packet.json" in RECIPES[sample]: + # Report goldens deliberately have packet disabled. The packet golden + # is a separate real scan with the same fixed date as its existing test. + packet_out = repo / "packet-output" + run_scan( + config_path=repo / "shipgate.yaml", + output_dir=packet_out, + formats=["markdown", "json"], + ci_mode="advisory", + packet_generated_at=GENERATED_AT, + plugins_enabled=False, + ) + for name in ("packet.json", "packet.md", "packet.html"): + shutil.copyfile(packet_out / name, out / name) + if "cold-report.md" in RECIPES[sample]: + cold = temp / "cold" + _copy_sample(source, cold, cold=True) + cold_out = cold / "reports" + run_scan( + config_path=cold / "shipgate.yaml", + output_dir=cold_out, + formats=["markdown", "json"], + ci_mode="advisory", + packet_generated_at=GENERATED_AT, + plugins_enabled=False, + ) + shutil.copyfile(cold_out / "report.md", out / "cold-report.md") + for name in RECIPES[sample]: + if name != "current-control.json": + _lf(out / name) + if "current-control.json" in RECIPES[sample]: + pointer_path = out / "current-control.json" + pointer = CurrentControlPointer.model_validate_json(pointer_path.read_bytes()) + # Fresh synthetic fixture, not a continuation of the last generating + # machine's in-progress pointer. Rebind only AFTER normalization/LF. + pointer_path.unlink() + publish_current_control( + out, + operation=pointer.operation, + control=pointer.control, + workspace_identity=pointer.workspace_identity, + artifact_keys={"report", "report_markdown"}, + ) + _lf(pointer_path) + result = {} + for name in RECIPES[sample]: + path = out / name + _assert_no_machine_paths(path, temp, root) + result[Path("samples") / sample / "expected" / name] = path.read_bytes() + return result + + +def build_goldens(root: Path = ROOT, samples: list[str] | None = None) -> dict[Path, bytes]: + root = root.resolve() + selected = sorted(RECIPES if samples is None else set(samples)) + unknown = sorted(set(selected) - RECIPES.keys()) + if unknown: + raise ValueError(f"No golden recipe for: {', '.join(unknown)}") + for sample in selected: + _fixture_path(root, Path("samples") / sample) + managed = {Path("samples") / s / "expected" / name for s in selected for name in RECIPES[s]} + actual = { + path.relative_to(root) + for path in (root / "samples").glob("*/expected/*") + if samples is None or path.parent.parent.name in selected + } + extras = sorted(actual - managed) + if extras: + raise ValueError(f"No golden recipe for: {', '.join(p.as_posix() for p in extras)}") + result = {} + with tempfile.TemporaryDirectory(prefix="shipgate-goldens-") as temp, _scan_environment(): + for sample in selected: + result.update(_build_sample(root, sample, Path(temp))) + return result + + +def main(argv: list[str] | None = None, *, root: Path = ROOT) -> int: + root = root.resolve() + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "samples", nargs="*", choices=None, help="sample directory names (default: all)" + ) + parser.add_argument("--check", action="store_true", help="report drift without writing") + args = parser.parse_args(argv) + try: + artifacts = build_goldens(root, args.samples or None) + # Build and validate the entire selected set before any repository write. + for path in artifacts: + _fixture_path(root, path) + changed = [ + path + for path, data in artifacts.items() + if not (root / path).is_file() or (root / path).read_bytes() != data + ] + for path in changed: + if not args.check: + target = _fixture_path(root, path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(artifacts[path]) + print(f"{'DRIFT' if args.check else 'WROTE'} {path.as_posix()}") + print( + f"{'Checked' if args.check else 'Generated'} {len(artifacts)} sample artifacts; {len(changed)} changed." + ) + return int(args.check and bool(changed)) + except (OSError, ValueError, subprocess.CalledProcessError) as exc: + print(f"Golden generation failed: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_regenerate_goldens.py b/tests/test_regenerate_goldens.py new file mode 100644 index 00000000..6caa1c0d --- /dev/null +++ b/tests/test_regenerate_goldens.py @@ -0,0 +1,225 @@ +"""Real regeneration and refusal checks for the committed sample recipes.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from agents_shipgate.core.current_control import read_current_control + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "regenerate_goldens.py" + + +@pytest.fixture(scope="module") +def generator(): + spec = importlib.util.spec_from_file_location("shipgate_golden_generator", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _copy(root: Path, sample: str = "conductor_agent") -> Path: + target = root / "samples" / sample + shutil.copytree(ROOT / "samples" / sample, target) + return target + + +def _files(root: Path) -> dict[str, bytes]: + return {p.relative_to(root).as_posix(): p.read_bytes() for p in root.rglob("*") if p.is_file()} + + +def test_cli_check_all_goldens_from_another_directory(tmp_path): + """Normal CI invokes the real --check entry point; no workflow copy needed.""" + before = _files(ROOT / "samples") + summary = tmp_path / "step-summary.md" + summary.write_text("caller-owned\n") + env = {**os.environ, "GITHUB_STEP_SUMMARY": str(summary)} + result = subprocess.run( + [sys.executable, str(SCRIPT), "--check"], + cwd=tmp_path, + text=True, + capture_output=True, + env=env, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "24 sample artifacts; 0 changed" in result.stdout + assert _files(ROOT / "samples") == before + assert summary.read_text() == "caller-owned\n" + + +def test_two_fresh_repositories_generate_identical_bytes(generator, tmp_path): + root1, root2 = tmp_path / "one", tmp_path / "a different root" + _copy(root1) + _copy(root2) + first = generator.build_goldens(root1, ["conductor_agent"]) + second = generator.build_goldens(root2, ["conductor_agent"]) + assert first == second + expected = root1 / "samples/conductor_agent/expected" + for path, data in first.items(): + (root1 / path).write_bytes(data) + assert b"\r" not in data + pointer = read_current_control(expected).pointer + assert pointer.supersedes is None + assert set(pointer.artifacts) == {"report", "report_markdown"} + assert not pointer.control.permissions.merge + for reference in pointer.artifacts.values(): + data = (expected / reference.path).read_bytes() + assert reference.size_bytes == len(data) + assert reference.sha256 == "sha256:" + hashlib.sha256(data).hexdigest() + assert ( + json.loads((expected / "report.json").read_bytes())["manifest_dir"] + == "/samples/conductor_agent" + ) + + +@pytest.mark.parametrize( + "name", ["report.json", "report.md", "summary.json", "current-control.json"] +) +def test_check_names_one_byte_drift_without_writing(generator, tmp_path, capsys, name): + sample = _copy(tmp_path) + target = sample / "expected" / name + target.write_bytes(target.read_bytes() + b" ") + before = _files(tmp_path) + assert generator.main(["conductor_agent", "--check"], root=tmp_path) == 1 + assert f"DRIFT samples/conductor_agent/expected/{name}" in capsys.readouterr().out + assert _files(tmp_path) == before + + +def test_missing_golden_is_drift_then_regeneration_repairs_it(generator, tmp_path, capsys): + sample = _copy(tmp_path, "declaration_repair_agent") + target = sample / "expected/suggested-declarations.yaml" + expected = target.read_bytes() + target.unlink() + assert generator.main(["declaration_repair_agent", "--check"], root=tmp_path) == 1 + assert "suggested-declarations.yaml" in capsys.readouterr().out + assert not target.exists() + assert generator.main(["declaration_repair_agent"], root=tmp_path) == 0 + assert target.read_bytes() == expected + assert generator.main(["declaration_repair_agent", "--check"], root=tmp_path) == 0 + + +@pytest.mark.parametrize("leak", ["generated_report", "other_field", "markdown"]) +def test_leaked_temp_path_refuses_all_writes(generator, tmp_path, monkeypatch, capsys, leak): + _copy(tmp_path) + before = _files(tmp_path) + run_scan = generator.run_scan + + def leaking_scan(**kwargs): + result = run_scan(**kwargs) + out = kwargs["config_path"].parent / "expected/report.json" + if leak == "markdown": + from agents_shipgate.report.markdown import _safe_markdown_text + + markdown = out.with_suffix(".md") + markdown.write_text(markdown.read_text() + _safe_markdown_text(str(out.parent.parent))) + return result + payload = json.loads(out.read_text()) + if leak == "generated_report": + payload["generated_reports"]["json"] = str(out) + else: + payload["project"]["unexpected_path"] = str(out.parent.parent) + out.write_text(json.dumps(payload)) + return result + + monkeypatch.setattr(generator, "run_scan", leaking_scan) + monkeypatch.setenv("GITHUB_STEP_SUMMARY", "preserve-caller-setting") + assert generator.main(["conductor_agent"], root=tmp_path) == 2 + assert ("report.md" if leak == "markdown" else "report.json") in capsys.readouterr().err + assert _files(tmp_path) == before + assert os.environ["GITHUB_STEP_SUMMARY"] == "preserve-caller-setting" + + +def test_unowned_artifact_or_sample_is_not_silently_ignored(generator, tmp_path, capsys): + sample = _copy(tmp_path) + (sample / "expected/unowned.json").write_text("{}") + before = _files(tmp_path) + assert generator.main(["--check"], root=tmp_path) == 2 + assert "unowned.json" in capsys.readouterr().err + assert generator.main(["../outside"], root=tmp_path) == 2 + assert "No golden recipe" in capsys.readouterr().err + assert _files(tmp_path) == before + + +def test_regeneration_does_not_load_opted_in_installed_plugins(generator, monkeypatch): + from agents_shipgate.checks import registry + + loaded = [] + + class Plugin: + name = "golden-poison" + value = "golden_poison:checks" + + def load(self): + loaded.append(True) + return lambda context: [] + + monkeypatch.setenv("AGENTS_SHIPGATE_ENABLE_PLUGINS", "1") + monkeypatch.setattr(registry, "entry_points", lambda group: [Plugin()]) + generator.build_goldens() + assert loaded == [] + assert os.environ["AGENTS_SHIPGATE_ENABLE_PLUGINS"] == "1" + + +@pytest.mark.parametrize("component", ["samples", "expected"]) +def test_symlinked_ancestor_cannot_rewrite_external_goldens(generator, tmp_path, capsys, component): + root, outside = tmp_path / "root", tmp_path / "outside" + sample = _copy(outside) + root.mkdir() + try: + if component == "samples": + (root / "samples").symlink_to(outside / "samples", target_is_directory=True) + else: + target = _copy(root) + shutil.rmtree(target / "expected") + (target / "expected").symlink_to(sample / "expected", target_is_directory=True) + except OSError as exc: + pytest.skip(f"symlink unavailable: {exc}") + report = sample / "expected/report.md" + report.write_bytes(report.read_bytes() + b"outside sentinel\n") + before = _files(outside) + assert generator.main(["conductor_agent"], root=root) == 2 + assert "symlink" in capsys.readouterr().err + assert _files(outside) == before + + +@pytest.mark.parametrize("conversions", [1, 2]) +def test_crlf_checkout_inputs_generate_the_same_artifacts(generator, tmp_path, conversions): + for sample in generator.RECIPES: + target = _copy(tmp_path, sample) + for path in target.rglob("*"): + if path.is_file() and "expected" not in path.relative_to(target).parts: + # The second pass models inputs already checked out as CRLF. + for _ in range(conversions): + text = path.read_text(encoding="utf-8") + path.write_bytes(text.replace("\n", "\r\n").encode("utf-8")) + result = generator.build_goldens(tmp_path) + assert all(data == (ROOT / path).read_bytes() for path, data in result.items()) + + +def test_git_context_and_global_exclusions_do_not_change_fixture_state( + generator, tmp_path, monkeypatch +): + _copy(tmp_path) + excludes = tmp_path / "excluded" + excludes.write_text("*\n") + config = tmp_path / "global-config" + config.write_text(f'[core]\n excludesFile = "{excludes.as_posix()}"\n') + settings = { + "GIT_CONFIG_GLOBAL": str(config), + "GIT_DIR": str(ROOT / ".git"), + "GIT_WORK_TREE": str(ROOT), + } + for key, value in settings.items(): + monkeypatch.setenv(key, value) + result = generator.build_goldens(tmp_path, ["conductor_agent"]) + assert all(data == (ROOT / path).read_bytes() for path, data in result.items()) + assert all(os.environ[key] == value for key, value in settings.items())