diff --git a/.github/requirements/development-inputs.lock b/.github/requirements/development-inputs.lock new file mode 100644 index 0000000..70635aa --- /dev/null +++ b/.github/requirements/development-inputs.lock @@ -0,0 +1,76 @@ +# Exact reviewed producer tools; no release/client artifact adoption. +pip==26.2.1 --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e +packaging==26.3 --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +PyYAML==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 diff --git a/.github/workflows/development-source-inputs.yml b/.github/workflows/development-source-inputs.yml new file mode 100644 index 0000000..6502e21 --- /dev/null +++ b/.github/workflows/development-source-inputs.yml @@ -0,0 +1,83 @@ +name: Development source input proposal +on: + pull_request: + push: + branches: [trunk] +permissions: + contents: read +concurrency: + group: development-inputs-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +jobs: + proposal: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: '3.12.3' + - name: Install reviewed producer tools + run: | + python -m pip install --require-hashes --only-binary=:all: -r .github/requirements/development-inputs.lock + - name: Run input tamper controls + run: | + python - <<'PYTHON' + from pathlib import Path + import yaml + for path in Path('.github/workflows').glob('*.yml'): + workflow = yaml.safe_load(path.read_text()) + for job in workflow.get('jobs', {}).values(): + for step in job.get('steps', []): + if 'run' in step: + assert isinstance(step['run'], str), (str(path), step.get('name'), type(step['run']).__name__) + PYTHON + python -m unittest discover -s tests -p test_development_source_inputs.py -v + python -m unittest discover -s tests -p test_governed_source_inputs.py -v + - name: Resolve all five profiles and original ordered source-install sequences + run: | + python scripts/development_source_inputs.py --source "$GITHUB_WORKSPACE" --revision "$(git rev-parse HEAD)" --destination "$RUNNER_TEMP/development-inputs" + python scripts/development_source_inputs.py --verify --source "$GITHUB_WORKSPACE" --revision "$(git rev-parse HEAD)" --destination "$RUNNER_TEMP/development-inputs" + python scripts/check_development_input_mutations.py --source "$GITHUB_WORKSPACE" --revision "$(git rev-parse HEAD)" --destination "$RUNNER_TEMP/development-inputs" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: development-source-input-proposal + path: ${{ runner.temp }}/development-inputs + if-no-files-found: warn + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: development-synthetic-pip-control + path: ${{ runner.temp }}/development-pip-control + if-no-files-found: ignore + governed-document-runtime: + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: producer + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + path: producer + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # Read-only original governed subject; no source or release adoption. + ref: 5a79352066b8aa23f1bae45b5c38e6d83efd003f + path: subject + persist-credentials: false + - name: Pull exact original documentation image + run: | + docker pull python:3.12-bookworm@sha256:e91fec3d1ac69f04e4eddcd29c327e630ce34658cf31075bfa7e8b0e052bafea + - name: Observe original runtime and produce all immutable input sequences + run: | + python3 scripts/run_governed_source_inputs.py --implementation "$GITHUB_WORKSPACE/producer" --source "$GITHUB_WORKSPACE/subject" --revision 5a79352066b8aa23f1bae45b5c38e6d83efd003f --destination "$RUNNER_TEMP/governed-development-inputs" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: governed-development-source-inputs + path: ${{ runner.temp }}/governed-development-inputs + if-no-files-found: warn diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 1aaf444..9e95408 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -18,6 +18,7 @@ * [Authenticate a client](auth.md) * [Run the operator workflow from a terminal](cli/operator-workflow.md) +* [Produce immutable development source inputs](contributor/development-source-inputs.md) * [Collect a support bundle](diagnostic-bundles.md) * [Hand an ArcPy inventory to the codemod](honua-gp/scanner-handoff.md) * [Import the geoprocessing package](migration/gp-import.md) diff --git a/docs/contributor/development-source-inputs.md b/docs/contributor/development-source-inputs.md new file mode 100644 index 0000000..e69034a --- /dev/null +++ b/docs/contributor/development-source-inputs.md @@ -0,0 +1,99 @@ +--- +type: guide +title: "Produce immutable development source inputs" +description: "Capture review-only source, dependency and backend inputs without admitting or publishing packages." +resource: "https://github.com/honua-io/honua-sdk-python/issues/311" +tags: [python, development, release] +--- +# Review-only development source inputs + +`python scripts/development_source_inputs.py --source /absolute/clean/checkout --revision FULL_40_CHARACTER_SHA --destination /absolute/new/proposal-directory` + +Run with the exact Python, pip, packaging and platform declared in +`scripts/development-inputs/runtime.json`. The source must be a clean immutable +checkout of this repository. Outputs stay outside the checkout. The producer +captures every tracked source input and derives the core, grpc, geopandas, +raster, and combined SDK-with-all-extras plus Admin profiles from the actual +package metadata. All five profiles are mandatory. + +The producer implementation and subject checkout have separate immutable +identities. A protected producer can inspect an older, still-governed clean +subject using `--source` and `--revision`; this does not adopt newer package +versions or turn source-built wheels into published artifacts. + +The proposal also captures each original README and INSTALL source-install +block, including its exact document and fence bytes, ordered pip commands, +requested extras and editable flags. Each block uses an isolated environment; +each pip step records its actual resolver delta and installed closure before +and after execution. The backend closure is installed with hash enforcement +before source metadata or editable build hooks execute. Non-pip lint and test +commands remain unchanged for the executable-docs replay and are not executed +or counted as passed by this producer. + +Default pip compilation remains enabled. Verification checks a finite mapping +from original wheel members to installed files, including pip metadata, +entrypoint scripts and compiled bytecode, and rejects extra installed or +editable-source files. Editable wheels and their exact copied source tree have +separate provenance from regular source-built wheels. + +The default runtime policy describes the reviewed producer runtime. It does +not assert equivalence with a documentation container image. The later harness +must bind the actual governed image and invoked interpreter; a runtime mismatch +remains nonpassing until that exact input is produced and reviewed. + +For the original governed documentation subject, first make the exact +`python:3.12-bookworm@sha256:e91fec3d1ac69f04e4eddcd29c327e630ce34658cf31075bfa7e8b0e052bafea` +image available to Docker, then use the governed launcher: + +```bash +python3 scripts/run_governed_source_inputs.py --implementation /absolute/clean/producer-checkout --source /absolute/clean/governed-subject --revision 5a79352066b8aa23f1bae45b5c38e6d83efd003f --destination /absolute/new/governed-inputs +``` + +This subject remains SDK 0.1.16 and Admin 0.1.12. The producer implementation +has its own revision. The launcher uses the original image's Python 3.12.15 +and pip 25.0.1, with their exact reviewed executable and tool hashes. Packaging +26.3 is a separately hash-verified, read-only implementation import; it is +never installed into the image and is excluded from every subject pip/backend +subprocess. Actual child observations and seven input-refusal controls check +that separation. Original pip compilation remains enabled. + +The outer `execution.json` and original container observations bind the actual +image digest, container image ID, immutable subject and implementation, command, +read-only mounts, terminal state and cleanup. The inner `proposal/proposal.json` +binds all artifact and ordered input captures. A consumer must rejudge both +sets of original evidence; the policy's image string alone grants no identity +or admission. Failed actions retain their original output and daemon state. +A transport failure waits for live owned work to finish rather than cancelling +it. Unresolved work remains explicitly pending. + +This captures inputs using declared fixed-interpreter commands, including +separate offline installation of the verified artifacts. It does not report +that the unchanged documentation commands, lint commands or full tests ran. +Those original commands and their actual runtime replay remain required by +honua-release issue 721. + +Each profile retains the actual resolver command, interpreter environment, +original report and output, and anonymously downloaded wheels with canonical +PyPI origins, metadata, exact versions and SHA256 hashes. An independent registry +response must agree with every artifact. Dependency markers and extras are +rejudged recursively; missing dependencies fail. The exact Hatchling build +closure is resolved separately, installed with hash enforcement into a disposable +backend environment, and used without build isolation, dependency resolution or +network indexes to build both packages from tracked source bytes only. + +`python scripts/development_source_inputs.py --verify --source /absolute/clean/checkout --revision FULL_40_CHARACTER_SHA --destination /absolute/existing/proposal-directory` + +Verification rejoins source/runtime, original resolver reports, all profiles, +artifact bytes and recursive closure, and the separate source-built wheels. +Source or runtime refusal occurs before resolver execution. Drift detected after +resolution refuses before backend installation. Failure outputs are retained; +a missing final `proposal.json` is a failed proposal, never an admitted input. + +This is an input proposal for honua-release issue 721 and Python issue 311. +Generation and verification do not admit a source install, relax default +PEP610/direct-origin refusal, establish published wheel identity, publish a +package, change a version or release pin, or qualify a release candidate. The +SDK and Admin publication sources may differ; these source-built wheels carry +their actual checkout identity and never inherit a published artifact's identity. +Every original document and command remains required for the later admitted +source and exact-candidate replay. diff --git a/scripts/check_development_input_mutations.py b/scripts/check_development_input_mutations.py new file mode 100644 index 0000000..cf2f304 --- /dev/null +++ b/scripts/check_development_input_mutations.py @@ -0,0 +1,83 @@ +"""Rejudge one actual complete proposal and original-sequence mutation controls.""" +from __future__ import annotations + +import argparse +from copy import deepcopy +import json +from pathlib import Path + +import development_source_inputs as producer + + +def check(source: Path, revision: str, directory: Path, policy: dict) -> dict: + original = json.loads((directory / "proposal.json").read_bytes()) + producer.verify_proposal(original, source, revision, directory, policy) + mutations = ["context-loss", "step-loss", "step-reorder", "ordinal", "command", "block", "extras-superset", "editable", + "source-before", "source-resolved", "source-after", "installed-before", "installed-after", "derived-command", + "install-command", "raw-report", "backend-command", "editable-identity", "published", "admitted", "candidate"] + for mutation in mutations: + value = deepcopy(original) + context = value["documentationSequences"][0] + step = context["steps"][0] + if mutation == "context-loss": + value["documentationSequences"].pop() + elif mutation == "step-loss": + context["steps"].pop() + elif mutation == "step-reorder": + context["steps"][0], context["steps"][1] = context["steps"][1], context["steps"][0] + elif mutation == "ordinal": + step["ordinal"] = 9 + elif mutation == "command": + step["command"] += " --no-compile" + elif mutation == "block": + context["blockSha256"] = "0" * 64 + elif mutation == "extras-superset": + step["requested"] = ["./packages/honua-sdk[grpc,geopandas,raster]"] + elif mutation == "editable": + value["documentationSequences"][1]["steps"][0]["editable"] = [False] + elif mutation.startswith("source-"): + key = {"source-before": "sourceBefore", "source-resolved": "sourceResolved", "source-after": "sourceAfter"}[mutation] + step[key][next(iter(step[key]))] = "0" * 64 + elif mutation in ("installed-before", "installed-after"): + key = "before" if mutation.endswith("before") else "after" + step[key][next(iter(step[key]))]["version"] = "999.0" + elif mutation == "derived-command": + step["commandDerived"].remove("--no-build-isolation") + elif mutation == "install-command": + step["installCommand"].remove("--no-index") + elif mutation == "raw-report": + step["rawHashes"]["resolver"] = "0" * 64 + elif mutation == "backend-command": + context["bootstrapCommand"].remove("--require-hashes") + elif mutation == "editable-identity": + context["editableWheels"][0]["publishedIdentity"] = True + elif mutation == "published": + value["publishedWheelIdentity"] = True + elif mutation == "admitted": + value["sourceAdmitted"] = True + else: + value["candidateQualified"] = True + try: + producer.verify_proposal(value, source, revision, directory, policy) + except ValueError: + continue + raise AssertionError("complete proposal accepted mutation: " + mutation) + # Controls never rewrite original retained reports or source files. + producer.verify_proposal(original, source, revision, directory, policy) + return {"completeOriginalPositive": "PASS", "mutationsRejected": mutations, "sourceAdmitted": False, + "candidateQualified": False, "publishedWheelIdentity": False} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--destination", type=Path, required=True) + parser.add_argument("--runtime-policy", type=Path, default=Path(__file__).parent / "development-inputs/runtime.json") + args = parser.parse_args() + result = check(args.source, args.revision, args.destination, json.loads(args.runtime_policy.read_bytes())) + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/development-inputs/governed-runtime.json b/scripts/development-inputs/governed-runtime.json new file mode 100644 index 0000000..09fbe34 --- /dev/null +++ b/scripts/development-inputs/governed-runtime.json @@ -0,0 +1,17 @@ +{ + "schema": "honua.python-development-runtime/v1", + "python": "3.12.15", + "implementation": "CPython", + "system": "Linux", + "machine": "x86_64", + "pip": "25.0.1", + "packaging": "26.3", + "executionBinding": { + "containerImage": "python:3.12-bookworm@sha256:e91fec3d1ac69f04e4eddcd29c327e630ce34658cf31075bfa7e8b0e052bafea", + "pythonExecutableSha256": "837a48d7afd08447fb43e28b282026fadb59368620072e57e01f66d12977e5ce", + "pipToolFilesSha256": "cf2bdb80674d9cb5e1323a099793f5fd8352665871c2168c20fcd9176ccd608f", + "pipEntrypointSha256": "6d2cd44b1e30b4969035606e18bcbc4304d45aca49c693b8f2ae2fbca4fa5d87", + "implementationPackagingFilename": "packaging-26.3-py3-none-any.whl", + "implementationPackagingSha256": "d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c" + } +} diff --git a/scripts/development-inputs/runtime.json b/scripts/development-inputs/runtime.json new file mode 100644 index 0000000..3acc8c6 --- /dev/null +++ b/scripts/development-inputs/runtime.json @@ -0,0 +1,9 @@ +{ + "schema": "honua.python-development-runtime/v1", + "python": "3.12.3", + "implementation": "CPython", + "system": "Linux", + "machine": "x86_64", + "pip": "26.2.1", + "packaging": "26.3" +} diff --git a/scripts/development_source_inputs.py b/scripts/development_source_inputs.py new file mode 100644 index 0000000..ac4abf0 --- /dev/null +++ b/scripts/development_source_inputs.py @@ -0,0 +1,486 @@ +"""Produce review-only immutable Python development inputs; never admit or publish them.""" +from __future__ import annotations + +import argparse +import email.parser +import hashlib +import importlib.metadata +import io +import json +import os +from pathlib import Path +import platform +import shutil +import re +import subprocess +import sys +import tempfile +import tomllib +from urllib.parse import unquote, urlsplit +from urllib.request import urlopen +import zipfile + +from packaging.markers import default_environment +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name, parse_wheel_filename + +if __package__: + from . import python_document_inputs as documentation +else: + import python_document_inputs as documentation + +REPOSITORY = "https://github.com/honua-io/honua-sdk-python" +PROFILES = {"core": (), "grpc": ("grpc",), "geopandas": ("geopandas",), "raster": ("raster",), + "combined-sdk-admin": ("grpc", "geopandas", "raster")} +SCHEMA = "honua.python-development-inputs/v1" +INDEX = "https://pypi.org/simple" + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def run(args: list[str], *, cwd: Path | None = None) -> bytes: + # Pip's special PIP_CONFIG_FILE handling also applies in isolated mode. + # No proxy/index/PYTHONPATH startup overrides enter a resolver or backend. + env = {key: os.environ[key] for key in ("PATH", "HOME", "LANG", "SYSTEMROOT") if key in os.environ} + env.update(PIP_CONFIG_FILE=os.devnull, PYTHONDONTWRITEBYTECODE="1") + return subprocess.check_output(args, cwd=cwd, env=env, stderr=subprocess.PIPE) # noqa: S603 + + +def runtime(policy: dict) -> dict: + observed = {"schema": "honua.python-development-runtime/v1", "python": platform.python_version(), + "implementation": platform.python_implementation(), "system": platform.system(), + "machine": platform.machine(), "pip": importlib.metadata.version("pip"), + "packaging": importlib.metadata.version("packaging")} + binding = policy.get("executionBinding") + if observed != {key: value for key, value in policy.items() if key != "executionBinding"}: + raise ValueError("runtime differs from reviewed exact policy") + tool_bytes = {} + for name in ("pip", "packaging"): + distribution = importlib.metadata.distribution(name) + files = {str(path): digest(distribution.locate_file(path).read_bytes()) for path in distribution.files or [] + if path.suffix != ".pyc" and distribution.locate_file(path).is_file()} + if not files: + raise ValueError("runtime tool bytes are uninspectable") + tool_bytes[name] = digest(json.dumps(files, sort_keys=True).encode()) + executable = Path(sys.executable).resolve() + result = {**observed, "toolFilesSha256": tool_bytes, "executableSha256": digest(executable.read_bytes()), + "policySha256": digest(json.dumps(policy, sort_keys=True).encode()), "executable": str(executable)} + if binding is not None: + fields = {"containerImage", "pythonExecutableSha256", "pipToolFilesSha256", "pipEntrypointSha256", + "implementationPackagingFilename", "implementationPackagingSha256"} + if not isinstance(binding, dict) or set(binding) != fields: + raise ValueError("governed runtime binding is incomplete") + if (binding["containerImage"] != "python:3.12-bookworm@sha256:e91fec3d1ac69f04e4eddcd29c327e630ce34658cf31075bfa7e8b0e052bafea" + or result["executableSha256"] != binding["pythonExecutableSha256"] + or tool_bytes["pip"] != binding["pipToolFilesSha256"]): + raise ValueError("original documentation interpreter or pip bytes differ") + pip_entrypoint = executable.parent / "pip" + if not pip_entrypoint.is_file() or digest(pip_entrypoint.read_bytes()) != binding["pipEntrypointSha256"]: + raise ValueError("original documentation pip entrypoint differs") + origin = Path(sys.modules["packaging"].__file__) + wheels = [path for path in origin.parents if path.name == binding["implementationPackagingFilename"]] + if (len(wheels) != 1 or wheels[0].is_symlink() or not wheels[0].is_file() + or digest(wheels[0].read_bytes()) != binding["implementationPackagingSha256"]): + raise ValueError("implementation packaging is not the reviewed standalone wheel") + # Recompute original child-runtime separation for generate AND pure + # verification. A claimed receipt or boolean is never the authority. + program = """import importlib.metadata,json,sys +try: + importlib.metadata.distribution('packaging') + packaging_installed=True +except importlib.metadata.PackageNotFoundError: + packaging_installed=False +print(json.dumps({'packagingInstalled':packaging_installed,'pip':importlib.metadata.version('pip'),'sysPath':sys.path},sort_keys=True))""" + raw = run([str(executable), "-c", program]) + child = json.loads(raw) + if (child.get("packagingInstalled") is not False or child.get("pip") != observed["pip"] + or any(str(wheels[0]) in path for path in child.get("sysPath", []))): + raise ValueError("implementation packaging leaked into the subject runtime") + result["executionBinding"] = {**binding, "implementationPackaging": {"filename": wheels[0].name, + "sha256": digest(wheels[0].read_bytes()), "role": "producer-only-readonly-import", "installed": False}, + "originalChildObservation": child, "originalChildObservationSha256": digest(raw)} + return result + +def source(root: Path, revision: str) -> dict: + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("source requires full immutable revision") + if run(["git", "rev-parse", "HEAD"], cwd=root).decode().strip() != revision: + raise ValueError("wrong immutable checkout") + if run(["git", "status", "--porcelain", "--untracked-files=all"], cwd=root).strip(): + raise ValueError("source checkout is dirty or has untracked inputs") + origin = run(["git", "remote", "get-url", "origin"], cwd=root).decode().strip() + if origin.removesuffix(".git") not in (REPOSITORY, "git@github.com:honua-io/honua-sdk-python"): + raise ValueError("source repository differs") + files = {} + entries = run(["git", "ls-tree", "-rz", "--full-tree", revision], cwd=root).split(b"\0") + tracked = {entry.split(b"\t", 1)[1].decode(): entry.split(b"\t", 1)[0].decode().split() for entry in entries if entry} + for entry in entries: + if not entry: + continue + meta, name = entry.split(b"\t", 1) + mode, kind, blob = meta.decode().split() + path = name.decode() + if kind != "blob" or mode not in ("100644", "100755", "120000"): + raise ValueError("unsupported source entry") + link = None + if mode == "120000": + if (path != "CLAUDE.md" or not (root / path).is_symlink() + or os.readlink(root / path) != "AGENTS.md" + or tracked.get("AGENTS.md", [])[:2] != ["100644", "blob"] + or (root / "AGENTS.md").is_symlink()): + raise ValueError("unreviewed or escaping source symlink") + data = os.fsencode(os.readlink(root / path)) + target = (root / "AGENTS.md").read_bytes() + target_blob = hashlib.sha1(b"blob " + str(len(target)).encode() + b"\0" + target).hexdigest() # noqa: S324 + if target_blob != tracked["AGENTS.md"][2]: + raise ValueError("symlink immutable target differs") + link = {"linkTarget": "AGENTS.md", "targetGitBlob": target_blob, "targetSha256": digest(target)} + else: + if (root / path).is_symlink(): + raise ValueError("regular source replaced with symlink") + data = (root / path).read_bytes() + if hashlib.sha1(b"blob " + str(len(data)).encode() + b"\0" + data).hexdigest() != blob: # noqa: S324 + raise ValueError("source input differs from immutable blob") + files[path] = {"gitBlob": blob, "sha256": digest(data), "mode": mode, **(link or {})} + packages = {} + for name in ("honua-sdk", "honua-admin"): + path = f"packages/{name}/pyproject.toml" + spec = tomllib.loads((root / path).read_text()) + if spec["project"]["name"] != name or spec["build-system"] != { + "requires": ["hatchling==1.30.1"], "build-backend": "hatchling.build"}: + raise ValueError("unreviewed package or build backend") + packages[name] = {"sourceSha": revision, "metadata": spec["project"], "buildSystem": spec["build-system"]} + return {"repository": REPOSITORY, "sourceSha": revision, "files": files, "packages": packages} + + +def requirements(inputs: dict, profile: str) -> list[str]: + if profile not in PROFILES: + raise ValueError("unknown development profile") + sdk = inputs["packages"]["honua-sdk"]["metadata"] + result = list(sdk["dependencies"]) + for extra in PROFILES[profile]: + result.extend(sdk["optional-dependencies"][extra]) + if profile == "combined-sdk-admin": + for dependency in inputs["packages"]["honua-admin"]["metadata"]["dependencies"]: + req = Requirement(dependency) + if canonicalize_name(req.name) == "honua-sdk": + if sdk["version"] not in req.specifier: + raise ValueError("source Admin dependency excludes source SDK") + else: + result.append(dependency) + return sorted(set(result)) + + +def canonical_origin(url: str) -> str: + value = urlsplit(url) + if (value.scheme != "https" or value.hostname != "files.pythonhosted.org" or value.port is not None + or value.username or value.password or value.query or value.fragment + or not value.path.startswith("/packages/") or not value.path.endswith(".whl")): + raise ValueError("unsupported artifact origin") + filename = unquote(value.path.rsplit("/", 1)[-1]) + if Path(filename).name != filename or "\\" in filename: + raise ValueError("artifact filename is unsafe") + parse_wheel_filename(filename) + return filename + + +def wheel_metadata(data: bytes, filename: str) -> tuple[str, str, list[str]]: + wheel_name, wheel_version, _, _ = parse_wheel_filename(filename) + with zipfile.ZipFile(io.BytesIO(data)) as archive: + names = [n for n in archive.namelist() if n.endswith(".dist-info/METADATA")] + if len(names) != 1 or archive.testzip() is not None: + raise ValueError("invalid wheel archive") + metadata = email.parser.BytesParser().parsebytes(archive.read(names[0])) + name, version = str(metadata["Name"]), str(metadata["Version"]) + if canonicalize_name(name) != wheel_name or version != str(wheel_version): + raise ValueError("wheel identity mismatch") + return canonicalize_name(name), version, metadata.get_all("Requires-Dist", []) + + +def validate_closure(artifacts: list[dict], requested: list[str], directory: Path) -> None: + installed = {} + for item in artifacts: + if canonical_origin(item["origin"]) != item["filename"]: + raise ValueError("artifact filename differs from origin") + data = (directory / item["filename"]).read_bytes() + if digest(data) != item["sha256"]: + raise ValueError("artifact digest differs") + name, version, dependencies = wheel_metadata(data, item["filename"]) + if name != item["name"] or version != item["version"] or dependencies != item["requiresDist"] or name in installed: + raise ValueError("artifact metadata differs or duplicates") + installed[name] = item + pending = [(Requirement(value), "") for value in requested] + seen = set() + while pending: + req, extra = pending.pop() + if req.url: + raise ValueError("direct dependency origin refused") + if req.marker and not req.marker.evaluate({"extra": extra}): + continue + name = canonicalize_name(req.name) + if name not in installed or installed[name]["version"] not in req.specifier: + raise ValueError("incomplete dependency closure") + extras = req.extras or {""} + for selected in extras: + if (name, selected) in seen: + continue + seen.add((name, selected)) + pending.extend((Requirement(value), selected) for value in installed[name]["requiresDist"]) + if {name for name, _ in seen} != set(installed): + raise ValueError("unrequested artifacts in closure") + + +def scratch_identity(value: str, prefix: str) -> Path: + path = Path(value) + if (not path.is_absolute() or path != path.resolve() or path.parent != Path(tempfile.gettempdir()).resolve() + or not re.fullmatch(re.escape(prefix) + r"[a-z0-9_]{8}", path.name)): + raise ValueError("unbound private scratch identity") + return path + + +def resolver_command(requested: list[str], scratch: Path) -> list[str]: + return [sys.executable, "-m", "pip", "--isolated", "install", "--dry-run", "--ignore-installed", + "--only-binary=:all:", "--index-url", INDEX, "--disable-pip-version-check", "--no-input", + "--report", str(scratch / "pip-report.json"), *requested] + + +def download_artifacts(selections: list[dict], directory: Path) -> list[dict]: + artifacts = [] + for selected in selections: + info = selected["download_info"] + url = info["url"] + filename = canonical_origin(url) + expected = info["archive_info"]["hashes"]["sha256"] + metadata = selected["metadata"] + registry_url = f'https://pypi.org/pypi/{canonicalize_name(metadata["name"])}/{metadata["version"]}/json' + with urlopen(registry_url, timeout=60) as response: # noqa: S310 + registry = json.load(response) + if not any(x["url"] == url and x["filename"] == filename and x["digests"]["sha256"] == expected + for x in registry["urls"]): + raise ValueError("artifact is not in canonical registry response") + with urlopen(url, timeout=120) as response: # noqa: S310 + if response.url != url: + raise ValueError("artifact redirect refused") + data = response.read() + if digest(data) != expected: + raise ValueError("download differs from resolver digest") + name, version, dependencies = wheel_metadata(data, filename) + if name != canonicalize_name(metadata["name"]) or version != metadata["version"]: + raise ValueError("download metadata differs from original resolver") + target = directory / filename + if target.exists() and digest(target.read_bytes()) != expected: + raise ValueError("artifact filename collision") + target.write_bytes(data) + artifacts.append({"name": name, "version": version, "filename": filename, "origin": url, + "sha256": expected, "requiresDist": dependencies}) + return artifacts + + +def resolve(requested: list[str], directory: Path) -> dict: + directory.mkdir() + # Raw output/report remain outside the public destination until every URL, + # downloaded byte and dependency has been validated. Failed private data dies here. + with tempfile.TemporaryDirectory(prefix="honua-development-resolve-", dir=str(Path(tempfile.gettempdir()).resolve())) as temporary: + scratch = scratch_identity(temporary, "honua-development-resolve-") + report = scratch / "pip-report.json" + command = resolver_command(requested, scratch) + stdout = run(command) + observed = json.loads(report.read_bytes()) + if observed.get("version") != "1" or observed.get("pip_version") != importlib.metadata.version("pip"): + raise ValueError("unreviewed resolver report") + # Validate ALL origins before retaining even one report or artifact. + for selected in observed["install"]: + canonical_origin(selected["download_info"]["url"]) + artifacts = download_artifacts(observed["install"], directory) + validate_closure(artifacts, requested, directory) + shutil.copyfile(report, directory / "pip-report.json") + (directory / "resolver.stdout").write_bytes(stdout) + return {"requested": requested, "command": command, "scratchRoot": str(scratch), + "reportSha256": digest(report.read_bytes()), "stdoutSha256": digest(stdout), + "environment": observed["environment"], "artifacts": artifacts} + + +def build_commands(scratch: Path, destination: Path) -> tuple[list[str], list[str]]: + python = scratch / "backend/bin/python" + install = [sys.executable, "-m", "pip", "--isolated", "--python", str(python), "install", "--no-index", + "--find-links", str(destination / "build"), "--require-hashes", "-r", str(scratch / "backend.lock")] + build = [sys.executable, "-m", "pip", "--isolated", "--python", str(python), "wheel", + "--no-build-isolation", "--no-deps", "--no-index", "--wheel-dir", str(destination / "source-wheels"), + str(scratch / "source/packages/honua-sdk"), str(scratch / "source/packages/honua-admin")] + return install, build + + +def build_source_wheels(inputs: dict, root: Path, destination: Path, build: dict) -> dict: + """Build only immutable tracked bytes using the independently captured backend closure.""" + with tempfile.TemporaryDirectory(prefix="honua-development-build-", dir=str(Path(tempfile.gettempdir()).resolve())) as temporary: + scratch = scratch_identity(temporary, "honua-development-build-") + checkout = scratch / "source" + for name in inputs["files"]: + if name.startswith(("packages/honua-sdk/", "packages/honua-admin/")): + target = checkout / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes((root / name).read_bytes()) + environment = scratch / "backend" + run([sys.executable, "-m", "venv", "--without-pip", str(environment)]) + python = environment / "bin/python" + lock = scratch / "backend.lock" + lock.write_text("\n".join(f'{x["name"]}=={x["version"]} --hash=sha256:{x["sha256"]}' + for x in build["artifacts"]) + "\n") + command, build_command = build_commands(scratch, destination) + if inputs != source(root, inputs["sourceSha"]): + raise ValueError("source changed before backend install") + installed = run(command) + inventory = json.loads(run([str(python), "-c", "import importlib.metadata,json; print(json.dumps({x.metadata['Name']:x.version for x in importlib.metadata.distributions()}))"])) + expected = {x["name"]: x["version"] for x in build["artifacts"]} + if {canonicalize_name(k): v for k, v in inventory.items()} != expected: + raise ValueError("installed backend closure differs") + output = run(build_command) + wheels = [] + for path in sorted((destination / "source-wheels").glob("*.whl")): + data = path.read_bytes() + name, version, dependencies = wheel_metadata(data, path.name) + if name not in inputs["packages"] or version != inputs["packages"][name]["metadata"]["version"]: + raise ValueError("source-built package metadata differs") + wheels.append({"name": name, "version": version, "filename": path.name, "sha256": digest(data), + "requiresDist": dependencies, "sourceSha": inputs["sourceSha"], "publishedIdentity": False}) + if {x["name"] for x in wheels} != set(inputs["packages"]): + raise ValueError("incomplete source wheel build") + (destination / "source-build.stdout").write_bytes(installed + output) + return {"scratchRoot": str(scratch), "backendInstallCommand": command, "buildCommand": build_command, "backendLock": lock.read_text(), + "stdoutSha256": digest(installed + output), "installedBackend": inventory, "wheels": wheels} + + +def implementation() -> dict: + root = Path(__file__).resolve().parents[1] + revision = run(["git", "rev-parse", "HEAD"], cwd=root).decode().strip() + inspected = source(root, revision) + paths = ("scripts/development_source_inputs.py", "scripts/python_document_inputs.py", + "scripts/governed_source_inputs_bootstrap.py", "scripts/run_governed_source_inputs.py", + "scripts/development-inputs/runtime.json", "scripts/development-inputs/governed-runtime.json") + return {"repository": REPOSITORY, "sourceSha": revision, + "files": {path: inspected["files"][path] for path in paths}} + + +def verify_resolution(value: dict, requested: list[str], directory: Path, policy: dict) -> None: + if value["requested"] != requested: + raise ValueError("profile requirements differ") + scratch = scratch_identity(value["scratchRoot"], "honua-development-resolve-") + expected_command = resolver_command(requested, scratch) + if value["command"] != expected_command or value["environment"] != default_environment(): + raise ValueError("actual command or interpreter environment differs") + validate_closure(value["artifacts"], requested, directory) + if digest((directory / "pip-report.json").read_bytes()) != value["reportSha256"]: + raise ValueError("original resolver report differs") + raw = json.loads((directory / "pip-report.json").read_bytes()) + if raw.get("pip_version") != policy["pip"] or raw.get("environment") != value["environment"]: + raise ValueError("original report runtime differs") + selected = {} + for item in raw["install"]: + info = item["download_info"] + filename = canonical_origin(info["url"]) + key = canonicalize_name(item["metadata"]["name"]) + if key in selected: + raise ValueError("duplicate report artifact") + selected[key] = (item["metadata"]["version"], filename, info["url"], info["archive_info"]["hashes"]["sha256"]) + expected = {x["name"]: (x["version"], x["filename"], x["origin"], x["sha256"]) for x in value["artifacts"]} + if selected != expected or digest((directory / "resolver.stdout").read_bytes()) != value["stdoutSha256"]: + raise ValueError("original resolver artifact/output join differs") + + +def verify_artifact_inputs(proposal: dict, root: Path, revision: str, directory: Path, policy: dict) -> dict: + if proposal.get("schema") != SCHEMA or proposal.get("candidateQualified") is not False or proposal.get("sourceAdmitted") is not False or proposal.get("publishedWheelIdentity") is not False: + raise ValueError("proposal is not review-only") + if proposal.get("implementation") != implementation(): + raise ValueError("producer implementation binding differs") + if proposal["source"] != source(root, revision) or proposal["runtime"] != runtime(policy): + raise ValueError("source or runtime binding differs") + if set(proposal["profiles"]) != set(PROFILES): + raise ValueError("incomplete profile set") + expected_build = ["hatchling==1.30.1"] + for name, value in [("build", proposal["build"]), *proposal["profiles"].items()]: + requested = expected_build if name == "build" else requirements(proposal["source"], name) + verify_resolution(value, requested, directory / name, policy) + built = proposal["sourceBuild"] + scratch = scratch_identity(built["scratchRoot"], "honua-development-build-") + expected_install, expected_build_command = build_commands(scratch, directory) + if built["backendInstallCommand"] != expected_install or built["buildCommand"] != expected_build_command: + raise ValueError("source build command input binding differs") + expected_backend = {x["name"]: x["version"] for x in proposal["build"]["artifacts"]} + if {canonicalize_name(k): v for k, v in built["installedBackend"].items()} != expected_backend: + raise ValueError("backend installed identity differs") + expected_lock = "\n".join(f'{x["name"]}=={x["version"]} --hash=sha256:{x["sha256"]}' + for x in proposal["build"]["artifacts"]) + "\n" + if built["backendLock"] != expected_lock: + raise ValueError("backend exact artifact lock differs") + if {x["name"] for x in built["wheels"]} != {"honua-sdk", "honua-admin"}: + raise ValueError("incomplete source-built package set") + for item in built["wheels"]: + data = (directory / "source-wheels" / item["filename"]).read_bytes() + name, version, dependencies = wheel_metadata(data, item["filename"]) + if (digest(data) != item["sha256"] or item["sourceSha"] != revision or item["publishedIdentity"] is not False + or name != item["name"] or version != proposal["source"]["packages"][name]["metadata"]["version"] + or dependencies != item["requiresDist"]): + raise ValueError("source-built wheel binding differs") + if digest((directory / "source-build.stdout").read_bytes()) != built["stdoutSha256"]: + raise ValueError("original source build output differs") + return {"sourceSha": revision, "implementationSha": proposal["implementation"]["sourceSha"], "runtime": proposal["runtime"], "profiles": sorted(PROFILES), + "proposalSha256": digest(json.dumps(proposal, sort_keys=True).encode()), "sourceAdmitted": False} + + +def verify_proposal(proposal: dict, root: Path, revision: str, directory: Path, policy: dict) -> dict: + result = verify_artifact_inputs(proposal, root, revision, directory, policy) + documentation.verify(sys.modules[__name__], proposal, root, directory) + return {**result, "documentationContexts": len(proposal["documentationSequences"])} + + +def generate(root: Path, revision: str, destination: Path, policy: dict) -> dict: + initial, actual_runtime = source(root, revision), runtime(policy) + # Exclusive destination; no repo writes, version edits, adoption or publication. + if destination.resolve().is_relative_to(root.resolve()): + raise ValueError("proposal destination must stay outside immutable source") + destination.mkdir(mode=0o700) + try: + proposal = {"schema": SCHEMA, "implementation": implementation(), "source": initial, "runtime": actual_runtime, + "sourceAdmitted": False, "candidateQualified": False, "publishedWheelIdentity": False, + "build": resolve(["hatchling==1.30.1"], destination / "build"), "profiles": {}} + for name in PROFILES: + proposal["profiles"][name] = resolve(requirements(initial, name), destination / name) + if initial != source(root, revision) or actual_runtime != runtime(policy): + raise ValueError("source/runtime changed before backend installation") + proposal["sourceBuild"] = build_source_wheels(initial, root, destination, proposal["build"]) + proposal["documentationSequences"] = documentation.capture(sys.modules[__name__], proposal, root, destination) + if initial != source(root, revision) or actual_runtime != runtime(policy): + raise ValueError("source/runtime changed during generation") + verify_proposal(proposal, root, revision, destination, policy) + (destination / "proposal.json").write_text(json.dumps(proposal, indent=2) + "\n") + return proposal + except Exception as exc: + (destination / "failure.json").write_text(json.dumps({"status": "FAIL", "errorType": type(exc).__name__, + "sourceAdmitted": False}) + "\n") + raise + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--destination", type=Path, required=True) + parser.add_argument("--runtime-policy", type=Path, default=Path(__file__).parent / "development-inputs/runtime.json") + parser.add_argument("--verify", action="store_true") + args = parser.parse_args() + try: + policy = json.loads(args.runtime_policy.read_text()) + if args.verify: + verify_proposal(json.loads((args.destination / "proposal.json").read_text()), args.source, + args.revision, args.destination, policy) + else: + generate(args.source, args.revision, args.destination, policy) + except Exception as exc: # A refused proposal is nonzero, with URL-free public failure details. + # Do not expose credentialized URLs or resolver stderr on refusal. + parser.exit(1, f"development input proposal refused: {type(exc).__name__}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/governed_source_inputs_bootstrap.py b/scripts/governed_source_inputs_bootstrap.py new file mode 100644 index 0000000..accf365 --- /dev/null +++ b/scripts/governed_source_inputs_bootstrap.py @@ -0,0 +1,97 @@ +"""Bootstrap producer-only packaging without installing into the subject runtime.""" +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +import subprocess +import tempfile +from pathlib import Path +import sys +import zipfile + + +def bootstrap(wheel: Path, policy: dict) -> dict: + binding = policy["executionBinding"] + if wheel.name != binding["implementationPackagingFilename"] or wheel.is_symlink() or not wheel.is_file(): + raise ValueError("implementation tool wheel identity differs") + data = wheel.read_bytes() + if hashlib.sha256(data).hexdigest() != binding["implementationPackagingSha256"]: + raise ValueError("implementation tool wheel bytes differ") + # This check occurs before adding any implementation import path. Original + # image packaging is absent and its original pip remains unchanged. + try: + importlib.metadata.distribution("packaging") + except importlib.metadata.PackageNotFoundError: + pass + else: + raise ValueError("implementation packaging was installed into original runtime") + with zipfile.ZipFile(wheel) as archive: + if archive.testzip() is not None or "packaging/__init__.py" not in archive.namelist(): + raise ValueError("implementation tool wheel is malformed") + sys.path.insert(0, str(wheel)) + implementation = Path(__file__).resolve().parent + sys.path.insert(0, str(implementation)) + return {"filename": wheel.name, "sha256": hashlib.sha256(data).hexdigest(), + "role": "producer-only-readonly-import", "installed": False} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--implementation-packaging-wheel", type=Path, required=True) + parser.add_argument("--action", choices=["probe", "probe-refusals", "generate", "verify", "mutations"], required=True) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--destination", type=Path, required=True) + parser.add_argument("--runtime-policy", type=Path, required=True) + args = parser.parse_args() + policy = json.loads(args.runtime_policy.read_bytes()) + bootstrap(args.implementation_packaging_wheel, policy) + import development_source_inputs as producer + if args.action == "probe": + # Deliberately injected ambient import configuration must never reach + # the subject interpreter. runtime() re-observes a real child process. + os.environ["PYTHONPATH"] = str(args.implementation_packaging_wheel) + print(json.dumps(producer.runtime(policy), sort_keys=True)) + elif args.action == "probe-refusals": + failures = [] + with tempfile.TemporaryDirectory(prefix="honua-tool-refusals-") as temporary: + scratch = Path(temporary) + for case in ("wheel-bytes", "wheel-name", "image", "interpreter", "pip", "pip-entrypoint", "wheel-policy"): + candidate = json.loads(json.dumps(policy)) + wheel = args.implementation_packaging_wheel + if case in ("wheel-bytes", "wheel-name"): + wheel = scratch / (wheel.name if case == "wheel-bytes" else "foreign.whl") + wheel.write_bytes(args.implementation_packaging_wheel.read_bytes() + (b"MUTATION" if case == "wheel-bytes" else b"")) + else: + key = {"image": "containerImage", "interpreter": "pythonExecutableSha256", "pip": "pipToolFilesSha256", + "pip-entrypoint": "pipEntrypointSha256", "wheel-policy": "implementationPackagingSha256"}[case] + candidate["executionBinding"][key] = "invalid-reviewed-input" + path = scratch / "runtime.json" + path.write_text(json.dumps(candidate)) + command = [sys.executable, "-I", str(Path(__file__).resolve()), "--implementation-packaging-wheel", str(wheel), + "--action", "probe", "--source", str(args.source), "--revision", args.revision, + "--destination", str(args.destination), "--runtime-policy", str(path)] + result = subprocess.run(command, capture_output=True, check=False, env={key: value for key, value in os.environ.items() if key in ("PATH", "HOME", "LANG")}) # noqa: S603 + expected = ("implementation tool wheel identity differs" if case == "wheel-name" else + "implementation tool wheel bytes differ" if case in ("wheel-bytes", "wheel-policy") else + "original documentation pip entrypoint differs" if case == "pip-entrypoint" else + "original documentation interpreter or pip bytes differ") + if result.returncode == 0 or expected.encode() not in result.stderr: + raise ValueError("runtime refusal control did not reach its exact input boundary: " + case) + failures.append(case) + print(json.dumps({"actualRefusals": failures, "sourceAdmitted": False, "candidateQualified": False}, sort_keys=True)) + elif args.action == "generate": + producer.generate(args.source, args.revision, args.destination, policy) + elif args.action == "verify": + value = json.loads((args.destination / "proposal.json").read_bytes()) + producer.verify_proposal(value, args.source, args.revision, args.destination, policy) + else: + from check_development_input_mutations import check + print(json.dumps(check(args.source, args.revision, args.destination, policy), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/python_document_inputs.py b/scripts/python_document_inputs.py new file mode 100644 index 0000000..549fa64 --- /dev/null +++ b/scripts/python_document_inputs.py @@ -0,0 +1,666 @@ +"""Capture original ordered source-install inputs; never execute documentation's lint/test commands.""" +from __future__ import annotations + +import base64 +import configparser +import csv +import hashlib +import importlib.metadata +import importlib.util +import io +import json +import marshal +import struct +import zipfile +from pathlib import Path +import re +import shlex +import sys +import tempfile +from urllib.parse import unquote, urlsplit + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +SOURCE_TARGET = re.compile(r"(?:\./)?packages/(honua-sdk|honua-admin)(?:\[([a-z,]+)\])?") +TOOLS = {"pytest", "pytest-cov", "ruff", "mypy"} +FENCE = re.compile(r"^```([^\n]*)\n(.*?)^```[ \t]*$", re.MULTILINE | re.DOTALL) +INVENTORY_PROGRAM = r'''import base64,hashlib,importlib.metadata,json,pathlib,sys +rows={} +boundaries=[pathlib.Path(sys.prefix).resolve(), pathlib.Path(sys.prefix).resolve().parent/"source"] +site_files={} +site_roots=set() +for dist in importlib.metadata.distributions(): + files={} + sizes={} + generated={} + site=pathlib.Path(dist.locate_file('')).resolve() + site_roots.add(site) + for item in dist.files or []: + path=dist.locate_file(item) + if not any(path.resolve().is_relative_to(root) for root in boundaries): raise ValueError('installed metadata escapes owned context') + if path.is_symlink() or not path.is_file(): raise ValueError('installed record is not a regular file') + data=path.read_bytes() + key=str(item) + files[key]=hashlib.sha256(data).hexdigest() + sizes[key]=len(data) + if path.resolve().is_relative_to(site): site_files[str(path.resolve())]=files[key] + if key.endswith(('.pyc','/INSTALLER','/REQUESTED','/direct_url.json','/RECORD')) or key.startswith('../'): + generated[key]=base64.b64encode(data).decode('ascii') + name=dist.metadata['Name'].lower().replace('_','-') + if name in rows: raise ValueError('duplicate installed distribution') + direct=dist.read_text('direct_url.json') + rows[name]={'name':name,'version':dist.version,'requiresDist':dist.requires or [], + 'metadataSha256':hashlib.sha256(dist.read_text('METADATA').encode()).hexdigest(), + 'files':files,'sizes':sizes,'generated':generated,'sitePackages':str(site),'python':sys.executable, + 'directUrl':json.loads(direct) if direct else None} +actual={} +for site in site_roots: + for path in site.rglob('*'): + if path.is_symlink(): raise ValueError('installed site contains symlink') + if path.is_file(): actual[str(path.resolve())]=hashlib.sha256(path.read_bytes()).hexdigest() + elif not path.is_dir(): raise ValueError('installed site contains special entry') +if actual!=site_files: raise ValueError('installed site contains unrecorded or missing files') +census=hashlib.sha256(json.dumps(site_files,sort_keys=True).encode()).hexdigest() +for row in rows.values(): row['siteCensusSha256']=census +print(json.dumps(rows,sort_keys=True))''' +HOOK_PROGRAM = "import hatchling.build,json; print(json.dumps({'wheel':hatchling.build.get_requires_for_build_wheel(), 'editable':hatchling.build.get_requires_for_build_editable()}))" + + +def plan(engine, root: Path, source: dict) -> list[dict]: + contexts = [] + for document in ("README.md", "INSTALL.md"): + text = (root / document).read_text() + for index, block in enumerate(FENCE.finditer(text)): + code = block[2] + if "packages/honua-" not in code and "git clone https://github.com/honua-io/honua-sdk-python.git" not in code: + continue + if block[1].strip().split()[0] not in ("bash", "sh", "shell"): + raise ValueError("source documentation is not executable shell") + invocations = [] + for line in code.splitlines(): + words = shlex.split(line, comments=True) + if words[:2] != ["pip", "install"]: + continue + targets, editable, pending = [], [], False + for word in words[2:]: + if word == "-e" and not pending: + pending = True + continue + match = SOURCE_TARGET.fullmatch(word) + if not match and (pending or word not in TOOLS): + raise ValueError("unreviewed documented source target or option") + targets.append(word) + editable.append(pending) + pending = False + if not targets or pending: + raise ValueError("incomplete documented pip invocation") + invocations.append({"ordinal": len(invocations), "command": line, "commandSha256": engine.digest(line.encode()), + "requested": targets, "editable": editable}) + if not invocations: + raise ValueError("source documentation has no install invocation") + contexts.append({"document": document, "documentSha256": engine.digest(text.encode()), + "documentGitBlob": source["files"][document]["gitBlob"], "blockIndex": index, + "blockSha256": engine.digest(code.encode()), "blockBytes": code, + "invocations": invocations}) + if not any(x["document"] == "INSTALL.md" for x in contexts) or sum(x["document"] == "README.md" for x in contexts) < 2: + raise ValueError("missing original source documentation contexts") + return contexts + + +def translated(invocation: dict, checkout: Path) -> list[str]: + result = [] + for word, editable in zip(invocation["requested"], invocation["editable"], strict=True): + if editable: + result.append("-e") + match = SOURCE_TARGET.fullmatch(word) + result.append(str(checkout / "packages" / match[1]) + ("[" + match[2] + "]" if match[2] else "") if match else word) + return result + + +def roots(source: dict, invocations: list[dict], backend: list[str]) -> list[str]: + requested = set(backend) + for invocation in invocations: + for word in invocation["requested"]: + match = SOURCE_TARGET.fullmatch(word) + if match is None: + requested.add(word) + continue + metadata = source["packages"][match[1]]["metadata"] + values = list(metadata.get("dependencies", [])) + for extra in (match[2] or "").split(","): + if extra: + values.extend(metadata["optional-dependencies"][extra]) + for value in values: + req = Requirement(value) + if canonicalize_name(req.name) not in ("honua-sdk", "honua-admin"): + requested.add(value) + elif source["packages"][canonicalize_name(req.name)]["metadata"]["version"] not in req.specifier: + raise ValueError("source package dependency excludes immutable source version") + return sorted(requested) + + +def inventory(engine, python: Path) -> dict: + return json.loads(engine.run([str(python), "-c", INVENTORY_PROGRAM])) + + +def source_hashes(engine, source: dict, checkout: Path) -> dict: + expected = {path: item["sha256"] for path, item in source["files"].items() + if path.startswith(("packages/honua-sdk/", "packages/honua-admin/"))} + expected_directories = {parent.as_posix() for path in expected for parent in Path(path).parents if parent.as_posix() != "."} + if any(path.is_symlink() or not path.is_dir() for path in (checkout, checkout / "packages")): + raise ValueError("copied immutable source ancestor is not a regular directory") + actual = {} + directories = set() + for package in ("packages/honua-sdk", "packages/honua-admin"): + base = checkout / package + if base.is_symlink() or not base.is_dir(): + raise ValueError("copied immutable source root is not a regular directory") + directories.add(package) + for path in base.rglob("*"): + relative = path.relative_to(checkout).as_posix() + if path.is_symlink(): + raise ValueError("copied immutable source contains a symlink") + if path.is_file(): + actual[relative] = engine.digest(path.read_bytes()) + elif path.is_dir(): + directories.add(relative) + else: + raise ValueError("copied immutable source contains a special entry") + if actual != expected or directories != expected_directories - {"packages"}: + raise ValueError("copied immutable source tree differs from finite tracked inputs") + return actual + + +def copy_source(source: dict, root: Path, checkout: Path) -> None: + for path, item in source["files"].items(): + if path.startswith(("packages/honua-sdk/", "packages/honua-admin/")): + if item["mode"] not in ("100644", "100755"): + raise ValueError("source build package contains unsupported entry") + target = checkout / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes((root / path).read_bytes()) + + +def step_command(python: Path, checkout: Path, invocation: dict, report: Path, *, offline: Path | None) -> list[str]: + flags = ["--no-index", "--find-links", str(offline)] if offline else ["--dry-run", "--index-url", "https://pypi.org/simple"] + return [sys.executable, "-m", "pip", "--isolated", "--python", str(python), "install", "--no-build-isolation", + "--only-binary=:all:", "--disable-pip-version-check", "--no-input", *flags, "--report", str(report), + *translated(invocation, checkout)] + + +def expected_sources(source: dict, invocation: dict, checkout: Path) -> dict: + return {match[1]: {"url": (checkout / "packages" / match[1]).as_uri(), "editable": editable, + "version": source["packages"][match[1]]["metadata"]["version"]} + for word, editable in zip(invocation["requested"], invocation["editable"], strict=True) + if (match := SOURCE_TARGET.fullmatch(word))} + + +def direct_matches(observed: dict, expected: dict) -> bool: + return (isinstance(observed, dict) and set(observed) == {"url", "dir_info"} + and observed["url"] == expected["url"] and isinstance(observed["dir_info"], dict) + and set(observed["dir_info"]) <= {"editable"} + and observed["dir_info"].get("editable", False) is expected["editable"]) + + +def report_items(engine, raw: dict, source: dict, invocation: dict, checkout: Path, *, warehouse: Path | None = None) -> tuple[dict, list[dict]]: + if raw.get("pip_version") != importlib.metadata.version("pip") or raw.get("environment") != engine.default_environment(): + raise ValueError("documentation report interpreter or pip identity differs") + expected = expected_sources(source, invocation, checkout) + local, remote = {}, [] + for item in raw["install"]: + info, metadata = item["download_info"], item["metadata"] + name = canonicalize_name(metadata["name"]) + url = info["url"] + if urlsplit(url).scheme == "file": + if name not in source["packages"]: + filename = Path(unquote(urlsplit(url).path)).name + if warehouse is None or url != (warehouse / filename).as_uri() or not filename.endswith(".whl"): + raise ValueError("offline artifact origin escaped captured warehouse") + remote.append(item) + continue + if name not in expected or not direct_matches(info, expected[name]): + raise ValueError("unbound source resolution origin") + if metadata["version"] != expected[name]["version"]: + raise ValueError("source resolution version or editable mode differs") + if name in local: + raise ValueError("duplicate source resolution item") + local[name] = item + else: + engine.canonical_origin(url) + remote.append(item) + if set(local) != set(expected): + raise ValueError("missing actual source resolution item") + return local, remote + + +SCRIPT_TEMPLATES = { + "25.0.1": r"""# -*- coding: utf-8 -*- +import re +import sys +from {module} import {import_name} +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit({function}()) +""", + "26.2.1": """import sys +from {module} import {import_name} +if __name__ == '__main__': + sys.argv[0] = sys.argv[0].removesuffix('.exe') + sys.exit({function}()) +""", +} + + +def installed_wheel_bytes(engine, observed: dict, archive_path: Path) -> None: + files, sizes, generated = observed.get("files"), observed.get("sizes"), observed.get("generated") + if not isinstance(files, dict) or not files or not isinstance(sizes, dict) or set(files) != set(sizes) or not isinstance(generated, dict): + raise ValueError("installed file observation is incomplete") + site, python = Path(observed["sitePackages"]), Path(observed["python"]) + if not site.is_absolute() or not python.is_absolute() or python.parent.name != "bin": + raise ValueError("installed paths are not bound to an owned interpreter") + version = importlib.metadata.version("pip") + if version not in SCRIPT_TEMPLATES: + raise ValueError("unreviewed pip installed-file transformation") + expected = {} + with zipfile.ZipFile(archive_path) as archive: + metadata_dirs = {name.rsplit("/", 1)[0] for name in archive.namelist() if name.endswith(".dist-info/METADATA")} + if len(metadata_dirs) != 1: + raise ValueError("wheel metadata directory is ambiguous") + metadata_dir = metadata_dirs.pop() + record_name = metadata_dir + "/RECORD" + for name in archive.namelist(): + if name.endswith("/") or name == record_name: + continue + data = archive.read(name) + if ".data/scripts/" in name: + if data.startswith((b"#!python\n", b"#!pythonw\n")): + raise ValueError("unmodeled wheel script shebang transformation") + selected = str(Path("../../../bin") / Path(name).name) + elif ".data/" in name: + raise ValueError("unmodeled wheel install transformation") + else: + selected = name + if selected in expected: + raise ValueError("duplicate installed wheel mapping") + expected[selected] = data + entrypoints = metadata_dir + "/entry_points.txt" + if entrypoints in archive.namelist(): + parser = configparser.ConfigParser(interpolation=None) + parser.optionxform = str + parser.read_string(archive.read(entrypoints).decode()) + for section in ("console_scripts", "gui_scripts"): + for name, value in parser.items(section) if parser.has_section(section) else []: + match = re.fullmatch(r"([A-Za-z_][\w.]*)\s*:\s*([A-Za-z_][\w.]*)\s*", value) + if match is None or not re.fullmatch(r"[A-Za-z0-9_.-]+", name): + raise ValueError("unmodeled installed entrypoint") + if len(str(python).encode()) + 3 > 127 or any(c.isspace() for c in str(python)): + raise ValueError("unmodeled installed script interpreter path") + script = SCRIPT_TEMPLATES[version].format(module=match[1], import_name=match[2].split(".")[0], function=match[2]) + expected["../../../bin/" + name] = ("#!" + str(python) + "\n" + script).encode() + expected[metadata_dir + "/INSTALLER"] = b"pip\n" + if metadata_dir + "/REQUESTED" in files: + expected[metadata_dir + "/REQUESTED"] = b"" + if observed["directUrl"] is not None: + expected[metadata_dir + "/direct_url.json"] = json.dumps(observed["directUrl"], sort_keys=True).encode() + # Default pip compilation is retained. Validate cache code against verified + # wheel sources and this exact interpreter, without executing package code. + for name, source_bytes in list(expected.items()): + if not name.endswith(".py") or name.startswith("../"): + continue + absolute = str(site / name) + cache = str(Path(importlib.util.cache_from_source(absolute)).relative_to(site)) + if cache not in files: + raise ValueError("default pip compiled bytecode is missing") + raw = base64.b64decode(generated.get(cache, ""), validate=True) + if len(raw) < 16 or raw[:4] != importlib.util.MAGIC_NUMBER or raw[4:8] != b"\x00" * 4: + raise ValueError("unmodeled compiled bytecode header") + if struct.unpack(" None: + census = {} + scratch = engine.scratch_identity(context["scratchRoot"], "honua-document-context-") + expected_python = scratch / "environment/bin/python" + expected_site = scratch / ("environment/lib/python" + str(sys.version_info.major) + "." + str(sys.version_info.minor) + "/site-packages") + for row in after.values(): + site = Path(row["sitePackages"]) + if site != expected_site or row["python"] != str(expected_python): + raise ValueError("installed observation differs from the owned interpreter context") + for name, digest in row["files"].items(): + path = (site / name).resolve() + if path.is_relative_to(site): + if str(path) in census: + raise ValueError("installed distributions overlap") + census[str(path)] = digest + census_digest = engine.digest(json.dumps(census, sort_keys=True).encode()) + if any(row.get("siteCensusSha256") != census_digest for row in after.values()): + raise ValueError("installed finite site census differs") + for artifact in active: + installed_wheel_bytes(engine, after[artifact["name"]], warehouse / artifact["filename"]) + for name, origin in origins.items(): + if origin["editable"]: + item = next(item for item in context["editableWheels"] if item["name"] == name) + archive = public / "editable-wheels" / item["filename"] + else: + item = next(item for item in source_build["wheels"] if item["name"] == name) + archive = destination / "source-wheels" / item["filename"] + installed_wheel_bytes(engine, after[name], archive) + + +def capture(engine, proposal: dict, root: Path, destination: Path) -> list[dict]: + source = proposal["source"] + planned = plan(engine, root, source) + base_artifacts = proposal["build"]["artifacts"] + contexts = [] + for number, declaration in enumerate(planned): + public = destination / f"document-{number}" + public.mkdir() + warehouse = public / "wheels" + warehouse.mkdir() + with tempfile.TemporaryDirectory(prefix="honua-document-context-", dir=str(Path(tempfile.gettempdir()).resolve())) as temporary: + scratch = engine.scratch_identity(temporary, "honua-document-context-") + checkout, environment = scratch / "source", scratch / "environment" + python = environment / "bin/python" + copy_source(source, root, checkout) + engine.run([sys.executable, "-m", "venv", "--without-pip", str(environment)]) + lock = scratch / "backend.lock" + lock.write_text("\n".join(f'{x["name"]}=={x["version"]} --hash=sha256:{x["sha256"]}' for x in base_artifacts) + "\n") + bootstrap_command = [sys.executable, "-m", "pip", "--isolated", "--python", str(python), "install", "--no-index", + "--find-links", str(destination / "build"), "--require-hashes", "-r", str(lock)] + if engine.source(root, source["sourceSha"]) != source: + raise ValueError("source changed before documentation backend bootstrap") + source_hashes(engine, source, checkout) + engine.run(bootstrap_command) + source_hashes(engine, source, checkout) + hooks = {} + for name in ("honua-sdk", "honua-admin"): + command = [str(python), "-c", HOOK_PROGRAM] + source_hashes(engine, source, checkout) + raw = engine.run(command, cwd=checkout / "packages" / name) + source_hashes(engine, source, checkout) + result = json.loads(raw) + if set(result) != {"wheel", "editable"} or any(not isinstance(v, list) for v in result.values()): + raise ValueError("backend hook inputs are incomplete") + hooks[name] = {"command": command, "cwd": str(checkout / "packages" / name), + "result": result, "rawSha256": engine.digest(raw)} + (public / f"{name}-backend-hooks.json").write_bytes(raw) + dynamic = sorted({value for item in hooks.values() for values in item["result"].values() for value in values}) + for value in dynamic: + if Requirement(value).url: + raise ValueError("backend dynamic direct origin refused") + baseline_roots = [f'{x["name"]}=={x["version"]}' for x in base_artifacts] + backend_dynamic = None + if dynamic: + backend_dynamic = engine.resolve(sorted(set(baseline_roots + dynamic)), public / "backend-dynamic") + lock.write_text("\n".join(f'{x["name"]}=={x["version"]} --hash=sha256:{x["sha256"]}' for x in backend_dynamic["artifacts"]) + "\n") + dynamic_command = [sys.executable, "-m", "pip", "--isolated", "--python", str(python), "install", "--no-index", + "--find-links", str(public / "backend-dynamic"), "--require-hashes", "-r", str(lock)] + source_hashes(engine, source, checkout) + engine.run(dynamic_command) + source_hashes(engine, source, checkout) + backend_artifacts = backend_dynamic["artifacts"] + else: + dynamic_command = None + backend_artifacts = base_artifacts + # Retain actual editable build provenance separately from regular wheels. + editable_directory = public / "editable-wheels" + editable_directory.mkdir() + editable_wheels = [] + for name in ("honua-sdk", "honua-admin"): + program = "import hatchling.build; print(hatchling.build.build_editable(" + repr(str(editable_directory)) + "))" + command = [str(python), "-c", program] + source_hashes(engine, source, checkout) + engine.run(command, cwd=checkout / "packages" / name) + source_hashes(engine, source, checkout) + candidates = list(editable_directory.glob(name.replace("-", "_") + "-*.whl")) + if len(candidates) != 1: + raise ValueError("editable build did not produce one exact source package") + data = candidates[0].read_bytes() + actual_name, version, dependencies = engine.wheel_metadata(data, candidates[0].name) + if actual_name != name or version != source["packages"][name]["metadata"]["version"]: + raise ValueError("editable source-built wheel identity differs") + editable_wheels.append({"name": name, "version": version, "filename": candidates[0].name, + "sha256": engine.digest(data), "requiresDist": dependencies, + "command": command, "cwd": str(checkout / "packages" / name), + "publishedIdentity": False, "sourceSha": source["sourceSha"]}) + available = {(x["name"], x["version"]): x for x in backend_artifacts} + for item in backend_artifacts: + parent = public / "backend-dynamic" if dynamic else destination / "build" + (warehouse / item["filename"]).write_bytes((parent / item["filename"]).read_bytes()) + initial = inventory(engine, python) + if {k: v["version"] for k, v in initial.items()} != {x["name"]: x["version"] for x in backend_artifacts}: + raise ValueError("documentation backend baseline differs") + context = {**declaration, "scratchRoot": str(scratch), "bootstrapCommand": bootstrap_command, + "bootstrapLock": "\n".join(f'{x["name"]}=={x["version"]} --hash=sha256:{x["sha256"]}' for x in base_artifacts) + "\n", + "hooks": hooks, "dynamicBackend": backend_dynamic, "dynamicCommand": dynamic_command, + "dynamicLock": lock.read_text(), "editableWheels": editable_wheels, + "initialInstalled": initial, "steps": []} + installed_source_and_artifact_bytes(engine, after=initial, active=backend_artifacts, warehouse=warehouse, + origins={}, source_build=proposal["sourceBuild"], destination=destination, + context=context, public=public) + origins = {} + for ordinal, invocation in enumerate(declaration["invocations"]): + before = inventory(engine, python) + before_hashes = source_hashes(engine, source, checkout) + resolver_report = scratch / f"resolver-{ordinal}.json" + installed_report = scratch / f"installed-{ordinal}.json" + command = step_command(python, checkout, invocation, resolver_report, offline=None) + stdout = engine.run(command, cwd=checkout) + resolved_hashes = source_hashes(engine, source, checkout) + raw = json.loads(resolver_report.read_bytes()) + local, remote = report_items(engine, raw, source, invocation, checkout) + artifacts = engine.download_artifacts(remote, warehouse) + for item in artifacts: + available[item["name"], item["version"]] = item + install = step_command(python, checkout, invocation, installed_report, offline=warehouse) + source_hashes(engine, source, checkout) + install_output = engine.run(install, cwd=checkout) + actual_install = json.loads(installed_report.read_bytes()) + installed_local, installed_remote = report_items(engine, actual_install, source, invocation, checkout, warehouse=warehouse) + if installed_local.keys() != local.keys(): + raise ValueError("offline source install differs from resolver") + for item in installed_remote: + filename = Path(unquote(urlsplit(item["download_info"]["url"]).path)).name + if item["download_info"]["url"] != (warehouse / filename).as_uri(): + raise ValueError("offline install escaped captured warehouse") + after = inventory(engine, python) + after_hashes = source_hashes(engine, source, checkout) + versions = {k: x["version"] for k, x in before.items()} + versions.update({canonicalize_name(x["metadata"]["name"]): x["metadata"]["version"] for x in raw["install"]}) + if versions != {k: v["version"] for k, v in after.items()}: + raise ValueError("actual installed delta differs from resolver") + if any(after[k]["version"] != value["version"] for k, value in initial.items()): + raise ValueError("documentation install changed reviewed backend closure") + active = [available[k, v["version"]] for k, v in after.items() if k not in source["packages"]] + required = roots(source, declaration["invocations"][:ordinal + 1], [f'{x["name"]}=={x["version"]}' for x in backend_artifacts]) + engine.validate_closure(active, required, warehouse) + for name, expected in expected_sources(source, invocation, checkout).items(): + direct = after[name]["directUrl"] + if not direct_matches(direct, expected): + raise ValueError("actual source install origin or editable mode differs") + origins.update(expected_sources(source, invocation, checkout)) + installed_source_and_artifact_bytes(engine, after=after, active=active, warehouse=warehouse, + origins=origins, source_build=proposal["sourceBuild"], destination=destination, + context=context, public=public) + # Persist only validated canonical/local owned inputs. No raw failing URLs. + for label, value in [("resolver", resolver_report.read_bytes()), ("installed", installed_report.read_bytes()), + ("before", json.dumps(before, sort_keys=True).encode()), ("after", json.dumps(after, sort_keys=True).encode()), + ("resolver.stdout", stdout), ("install.stdout", install_output)]: + (public / f"step-{ordinal}-{label}.json").write_bytes(value) + context["steps"].append({**invocation, "sourceSha": source["sourceSha"], "blockIndex": declaration["blockIndex"], + "blockSha256": declaration["blockSha256"], "commandDerived": command, "installCommand": install, + "sourceBefore": before_hashes, "sourceResolved": resolved_hashes, "sourceAfter": after_hashes, "before": before, "after": after, + "artifacts": artifacts, "activeArtifacts": active, "requiredClosure": required, + "rawHashes": {label: engine.digest((public / f"step-{ordinal}-{label}.json").read_bytes()) + for label in ("resolver", "installed", "before", "after", "resolver.stdout", "install.stdout")}}) + contexts.append(context) + return contexts + + +def verify(engine, proposal: dict, root: Path, destination: Path) -> None: + source = proposal["source"] + expected = plan(engine, root, source) + contexts = proposal.get("documentationSequences") + if not isinstance(contexts, list) or len(contexts) != len(expected): + raise ValueError("missing original documentation sequence") + expected_source = {path: item["sha256"] for path, item in source["files"].items() + if path.startswith(("packages/honua-sdk/", "packages/honua-admin/"))} + base = proposal["build"]["artifacts"] + for number, (context, declaration) in enumerate(zip(contexts, expected, strict=True)): + if any(context.get(key) != value for key, value in declaration.items()): + raise ValueError("documentation command/source/block bytes differ") + public = destination / f"document-{number}" + warehouse = public / "wheels" + scratch = engine.scratch_identity(context["scratchRoot"], "honua-document-context-") + python, checkout = scratch / "environment/bin/python", scratch / "source" + lock = scratch / "backend.lock" + bootstrap = [sys.executable, "-m", "pip", "--isolated", "--python", str(python), "install", "--no-index", + "--find-links", str(destination / "build"), "--require-hashes", "-r", str(lock)] + if context["bootstrapCommand"] != bootstrap or context["bootstrapLock"] != "\n".join( + f'{x["name"]}=={x["version"]} --hash=sha256:{x["sha256"]}' for x in base) + "\n": + raise ValueError("documentation backend bootstrap input differs") + if set(context["hooks"]) != set(source["packages"]): + raise ValueError("incomplete actual backend hook observations") + dynamic = set() + for name, item in context["hooks"].items(): + raw = (public / f"{name}-backend-hooks.json").read_bytes() + if (engine.digest(raw) != item["rawSha256"] or json.loads(raw) != item["result"] + or item["command"] != [str(python), "-c", HOOK_PROGRAM] + or item["cwd"] != str(checkout / "packages" / name) or set(item["result"]) != {"wheel", "editable"}): + raise ValueError("original backend hook inputs differ") + for values in item["result"].values(): + for value in values: + if Requirement(value).url: + raise ValueError("backend direct origin refused") + dynamic.add(value) + if dynamic: + value = context["dynamicBackend"] + requested = sorted(set([f'{x["name"]}=={x["version"]}' for x in base]) | dynamic) + if value["requested"] != requested: + raise ValueError("editable backend closure input differs") + engine.verify_resolution(value, requested, public / "backend-dynamic", {"pip": proposal["runtime"]["pip"]}) + artifacts = value["artifacts"] + dynamic_command = [sys.executable, "-m", "pip", "--isolated", "--python", str(python), "install", "--no-index", + "--find-links", str(public / "backend-dynamic"), "--require-hashes", "-r", str(lock)] + if context["dynamicCommand"] != dynamic_command: + raise ValueError("editable backend command differs") + else: + if context["dynamicBackend"] is not None or context["dynamicCommand"] is not None: + raise ValueError("unexpected dynamic backend") + artifacts = base + if context["dynamicLock"] != "\n".join(f'{x["name"]}=={x["version"]} --hash=sha256:{x["sha256"]}' for x in artifacts) + "\n": + raise ValueError("actual backend lock differs") + if {x["name"] for x in context["editableWheels"]} != set(source["packages"]): + raise ValueError("editable build coverage incomplete") + for item in context["editableWheels"]: + name = item["name"] + data = (public / "editable-wheels" / item["filename"]).read_bytes() + actual_name, version, dependencies = engine.wheel_metadata(data, item["filename"]) + program = "import hatchling.build; print(hatchling.build.build_editable(" + repr(str(public / "editable-wheels")) + "))" + if (engine.digest(data) != item["sha256"] or actual_name != name or version != source["packages"][name]["metadata"]["version"] + or dependencies != item["requiresDist"] or item["publishedIdentity"] is not False or item["sourceSha"] != source["sourceSha"] + or item["command"] != [str(python), "-c", program] or item["cwd"] != str(checkout / "packages" / name)): + raise ValueError("editable build source/runtime provenance differs") + previous = context["initialInstalled"] + if {k: v["version"] for k, v in previous.items()} != {x["name"]: x["version"] for x in artifacts}: + raise ValueError("original backend installed baseline differs") + installed_source_and_artifact_bytes(engine, after=previous, active=artifacts, warehouse=warehouse, + origins={}, source_build=proposal["sourceBuild"], destination=destination, + context=context, public=public) + steps = context["steps"] + if len(steps) != len(declaration["invocations"]): + raise ValueError("missing ordered pip invocation") + available = {(x["name"], x["version"]): x for x in artifacts} + source_origins = {} + for ordinal, (step, invocation) in enumerate(zip(steps, declaration["invocations"], strict=True)): + if any(step.get(k) != v for k, v in invocation.items()) or any(step.get(k) != declaration[k] for k in ("blockIndex", "blockSha256")): + raise ValueError("original invocation order/editable/target bytes differ") + if step["sourceSha"] != source["sourceSha"] or step["sourceBefore"] != expected_source or step["sourceResolved"] != expected_source or step["sourceAfter"] != expected_source: + raise ValueError("per-invocation source input lineage differs") + if step["before"] != previous: + raise ValueError("ordered installed closure lineage differs") + resolver = scratch / f"resolver-{ordinal}.json" + installed = scratch / f"installed-{ordinal}.json" + if (step["commandDerived"] != step_command(python, checkout, invocation, resolver, offline=None) + or step["installCommand"] != step_command(python, checkout, invocation, installed, offline=warehouse)): + raise ValueError("fixed interpreter invocation differs") + for label in ("resolver", "installed", "before", "after", "resolver.stdout", "install.stdout"): + data = (public / f"step-{ordinal}-{label}.json").read_bytes() + if engine.digest(data) != step["rawHashes"][label]: + raise ValueError("original invocation observation bytes differ") + if label in ("before", "after") and json.loads(data) != step[label]: + raise ValueError("original installed metadata differs") + raw = json.loads((public / f"step-{ordinal}-resolver.json").read_bytes()) + actual = json.loads((public / f"step-{ordinal}-installed.json").read_bytes()) + local, remote = report_items(engine, raw, source, invocation, checkout) + installed_local, installed_remote = report_items(engine, actual, source, invocation, checkout, warehouse=warehouse) + expected_artifacts = {canonicalize_name(x["metadata"]["name"]): (x["metadata"]["version"], x["download_info"]["url"], + x["download_info"]["archive_info"]["hashes"]["sha256"]) for x in remote} + if {x["name"]: (x["version"], x["origin"], x["sha256"]) for x in step["artifacts"]} != expected_artifacts: + raise ValueError("original resolver artifact delta differs") + for item in step["artifacts"]: + available[item["name"], item["version"]] = item + versions = {k: v["version"] for k, v in step["before"].items()} + versions.update({canonicalize_name(x["metadata"]["name"]): x["metadata"]["version"] for x in raw["install"]}) + if versions != {k: v["version"] for k, v in step["after"].items()} or local.keys() != installed_local.keys(): + raise ValueError("actual installed version delta differs") + for item in installed_remote: + name = canonicalize_name(item["metadata"]["name"]) + version = item["metadata"]["version"] + expected_item = available[name, version] + if item["download_info"]["url"] != (warehouse / expected_item["filename"]).as_uri(): + raise ValueError("offline artifact delta differs") + active = [available[k, v["version"]] for k, v in step["after"].items() if k not in source["packages"]] + required = roots(source, declaration["invocations"][:ordinal + 1], [f'{x["name"]}=={x["version"]}' for x in artifacts]) + if step["activeArtifacts"] != active or step["requiredClosure"] != required: + raise ValueError("complete per-invocation installed closure differs") + engine.validate_closure(active, required, warehouse) + source_origins.update(expected_sources(source, invocation, checkout)) + for name, origin in source_origins.items(): + observed = step["after"][name] + if not direct_matches(observed["directUrl"], origin): + raise ValueError("installed source origin differs") + if any(step["after"][name]["version"] != value["version"] for name, value in context["initialInstalled"].items()): + raise ValueError("backend runtime changed during sequence") + installed_source_and_artifact_bytes(engine, after=step["after"], active=active, warehouse=warehouse, + origins=source_origins, source_build=proposal["sourceBuild"], destination=destination, + context=context, public=public) + previous = step["after"] diff --git a/scripts/run_governed_source_inputs.py b/scripts/run_governed_source_inputs.py new file mode 100644 index 0000000..92f9c70 --- /dev/null +++ b/scripts/run_governed_source_inputs.py @@ -0,0 +1,407 @@ +"""Run review-only inputs under the immutable original documentation runtime.""" +from __future__ import annotations + +import argparse +from functools import partial +import hashlib +import json +import os +import secrets +import shutil +import time +from pathlib import Path +import subprocess +import tempfile +from urllib.parse import urlsplit +from urllib.request import urlopen + +IMAGE = 'python:3.12-bookworm@sha256:e91fec3d1ac69f04e4eddcd29c327e630ce34658cf31075bfa7e8b0e052bafea' +REPOSITORY = 'https://github.com/honua-io/honua-sdk-python' +ACTIONS = ('probe', 'probe-refusals', 'generate', 'verify', 'mutations') + + +def action_inputs(action: str, revision: str, context: dict, policy: dict) -> tuple: + """Derive the fixed invocation from separately trusted execution context.""" + if action not in ACTIONS or set(context) != {'mountSources', 'user', 'tmpfs', 'image'}: + raise ValueError('incomplete governed execution context') + binding = policy['executionBinding'] + sources = context['mountSources'] + if set(sources) != {'implementation', 'subject', 'tool', 'output'}: + raise ValueError('incomplete governed mount context') + if any(not isinstance(value, str) or not Path(value).is_absolute() or str(Path(value)) != value + for value in sources.values()) or len(set(sources.values())) != 4: + raise ValueError('governed mount context is not finite and absolute') + user = context['user'] + parts = user.split(':') if isinstance(user, str) else [] + if len(parts) != 2 or any(not item.isdecimal() for item in parts): + raise ValueError('governed execution user differs') + tmpfs = f'/tmp:rw,exec,mode=0700,uid={parts[0]},gid={parts[1]}' # noqa: S108 -- fixed private container tmpfs + if context['tmpfs'] != tmpfs or binding['containerImage'] != IMAGE: + raise ValueError('governed private runtime context differs') + arguments = ['/usr/local/bin/python3.12', '-I', '/implementation/scripts/governed_source_inputs_bootstrap.py', + '--implementation-packaging-wheel', '/implementation-tools/'+binding['implementationPackagingFilename'], + '--action', action, '--source', '/subject', '--revision', revision, '--destination', '/output/proposal', + '--runtime-policy', '/implementation/scripts/development-inputs/governed-runtime.json'] + mounts = [(sources['implementation'], '/implementation', True), (sources['subject'], '/subject', True), + (sources['tool'], '/implementation-tools/'+binding['implementationPackagingFilename'], True), + (sources['output'], '/output', False)] + return arguments, mounts, user, tmpfs + + +def create_command(owner: str, arguments: list, mounts: list, user: str, tmpfs: str) -> list: + command = ['docker', 'create', '--label', 'io.honua.development-input-owner='+owner, '--read-only', '--user', user, + '--workdir', '/implementation', '--env', 'HOME=/tmp', '--env', 'PYTHONDONTWRITEBYTECODE=1', '--tmpfs', tmpfs] + for source_path, target, readonly in mounts: + command.extend(['--mount', 'type=bind,src='+str(source_path)+',dst='+target+(',readonly' if readonly else '')]) + return [*command, IMAGE, *arguments] + + +def canonical_container_state(observed: dict) -> dict: + # Docker's Mounts is a census, not an ordered API. Preserve every row and + # field while comparing repeated observations; retain original bytes. + return {**observed, 'mounts': sorted(observed['mounts'], key=lambda row: json.dumps(row, sort_keys=True))} + + +def verify_execution_receipt(receipt: dict, *, implementation_sha: str, subject_sha: str, policy: dict, + proposal_bytes: bytes, expected_documentation: list, execution_context: dict, + original_files: dict[str, bytes]) -> dict: + """Rejudge public original bytes only; never run commands or grant admission. + + The caller authenticates implementation, subject, expected documentation and + execution_context independently. None is inferred from this receipt. + Inner artifact verification remains mandatory in the actual original runtime. + """ + binding = policy['executionBinding'] + expected_fields = {'schema', 'implementationSha', 'subjectSourceSha', 'image', 'implementationPackaging', + 'commands', 'containers', 'innerProposalSha256', 'executionContextSha256', + 'sourceAdmitted', 'candidateQualified', 'publishedWheelIdentity'} + if (set(receipt) != expected_fields or receipt['schema'] != 'honua.python-governed-input-execution/v1' + or receipt['implementationSha'] != implementation_sha or receipt['subjectSourceSha'] != subject_sha + or any(receipt[key] is not False for key in ('sourceAdmitted', 'candidateQualified', 'publishedWheelIdentity'))): + raise ValueError('governed receipt source or review-only identity differs') + image = execution_context['image'] + if (set(image) != {'id', 'repoDigests', 'declaredImage', 'os', 'architecture'} or image['declaredImage'] != IMAGE + or image['os'] != 'linux' or image['architecture'] != 'amd64' + or policy['system'] != 'Linux' or policy['machine'] != 'x86_64' + or not isinstance(image['id'], str) or len(image['id']) != 71 or not image['id'].startswith('sha256:') + or any(c not in '0123456789abcdef' for c in image['id'][7:]) + or not any(item in {'python@'+IMAGE.split('@')[1], 'docker.io/library/python@'+IMAGE.split('@')[1]} + for item in image['repoDigests']) or receipt['image'] != image): + raise ValueError('governed original image identity differs') + expected_tool = {'filename': binding['implementationPackagingFilename'], + 'sha256': binding['implementationPackagingSha256'], 'installed': False} + if receipt['implementationPackaging'] != expected_tool or receipt['implementationPackaging']['installed'] is not False: + raise ValueError('governed implementation tool identity differs') + if receipt['innerProposalSha256'] != hashlib.sha256(proposal_bytes).hexdigest(): + raise ValueError('governed original inner proposal bytes differ') + proposal = json.loads(proposal_bytes) + runtime = proposal['runtime'] + if (proposal['source']['sourceSha'] != subject_sha or proposal['implementation']['sourceSha'] != implementation_sha + or any(proposal[key] is not False for key in ('sourceAdmitted', 'candidateQualified', 'publishedWheelIdentity')) + or runtime['policySha256'] != hashlib.sha256(json.dumps(policy, sort_keys=True).encode()).hexdigest() + or runtime['executableSha256'] != binding['pythonExecutableSha256'] + or runtime['toolFilesSha256']['pip'] != binding['pipToolFilesSha256'] + or any(runtime.get(key) != value for key, value in policy.items() if key != 'executionBinding') + or any(runtime['executionBinding'].get(key) != value for key, value in binding.items())): + raise ValueError('governed inner source or original runtime differs') + sequences = proposal['documentationSequences'] + if (len(sequences) != len(expected_documentation) or len(expected_documentation) != 3 + or sum(len(item['invocations']) for item in expected_documentation) != 12): + raise ValueError('governed documentation context census differs') + for sequence, expected in zip(sequences, expected_documentation, strict=True): + if any(sequence.get(key) != value for key, value in expected.items()) or len(sequence['steps']) != len(expected['invocations']): + raise ValueError('governed original documentation declarations differ') + for step, invocation in zip(sequence['steps'], expected['invocations'], strict=True): + if any(step.get(key) != value for key, value in invocation.items()): + raise ValueError('governed original ordered invocation differs') + suffixes = ('stdout', 'stderr', 'attach.json', 'container-before.json', 'container-after.json', 'cleanup-observed.json', 'cleanup.json') + if (set(original_files) != {'execution-context.json', *(action+'.'+suffix for action in ACTIONS for suffix in suffixes)} + or any(type(value) is not bytes for value in original_files.values())): + raise ValueError('governed original public file census differs') + if (json.loads(original_files['execution-context.json']) != execution_context + or hashlib.sha256(original_files['execution-context.json']).hexdigest() != receipt['executionContextSha256']): + raise ValueError('governed original pre-action request context differs') + if (len(receipt['commands']) != len(ACTIONS) or len(receipt['containers']) != len(ACTIONS) + or [item.get('action') for item in receipt['commands']] != list(ACTIONS) + or [item.get('action') for item in receipt['containers']] != list(ACTIONS)): + raise ValueError('governed original action order differs') + ids, owners = set(), set() + for action, command, states in zip(ACTIONS, receipt['commands'], receipt['containers'], strict=True): + if set(command) != {'action', 'command', 'stdoutSha256', 'stderrSha256'} or set(states) != {'action', 'before', 'after', 'cleanup'}: + raise ValueError('governed action field census differs') + before, after, cleanup = states['before'], states['after'], states['cleanup'] + owner, container = before['ownerNonce'], before['id'] + if (not isinstance(owner, str) or len(owner) != 32 or any(c not in '0123456789abcdef' for c in owner) + or not isinstance(container, str) or len(container) != 64 or any(c not in '0123456789abcdef' for c in container) + or owner in owners or container in ids): + raise ValueError('governed original action ownership differs') + ids.add(container) + owners.add(owner) + arguments, mounts, user, tmpfs = action_inputs(action, subject_sha, execution_context, policy) + validate = partial(validate_container, owner=owner, image_id=image['id'], arguments=arguments, mounts=mounts, user=user, tmpfs=tmpfs) + for observed in (before, after, cleanup['originalState']): + if set(observed) != {'id', 'imageId', 'declaredImage', 'path', 'args', 'user', 'readonlyRootfs', 'mounts', + 'tmpfs', 'exitCode', 'status', 'running', 'startedAt', 'finishedAt', 'ownerNonce'}: + raise ValueError('governed safe daemon projection census differs') + validate(observed, container) + if (before['status'] != 'created' or before['running'] is not False or not before['startedAt'].startswith('0001-') + or not before['finishedAt'].startswith('0001-') or not exited(after) or after['exitCode'] != 0 + or set(cleanup) != {'ownedContainerId', 'ownerNonce', 'originalState', 'removed', 'sourceAdmitted'} + or cleanup['ownedContainerId'] != container or cleanup['ownerNonce'] != owner + or canonical_container_state(cleanup['originalState']) != canonical_container_state(after) + or cleanup['removed'] is not True or cleanup['sourceAdmitted'] is not False + or command['command'] != create_command(owner, arguments, mounts, user, tmpfs)): + raise ValueError('governed action terminal state or normal owned cleanup differs') + originals = {suffix: original_files[action+'.'+suffix] for suffix in suffixes} + if (json.loads(originals['container-before.json']) != before or json.loads(originals['container-after.json']) != after + or json.loads(originals['cleanup-observed.json']) != cleanup['originalState'] + or json.loads(originals['cleanup.json']) != cleanup + or json.loads(originals['attach.json']) != {'returnCode': 0, 'sourceAdmitted': False} + or type(json.loads(originals['attach.json'])['returnCode']) is not int + or json.loads(originals['attach.json'])['sourceAdmitted'] is not False + or any(command[label+'Sha256'] != hashlib.sha256(originals[label]).hexdigest() for label in ('stdout', 'stderr'))): + raise ValueError('governed original action streams or state bytes differ') + if action == 'probe' and json.loads(originals['stdout']) != runtime: + raise ValueError('governed original probe runtime differs') + if action in ('generate', 'verify') and originals['stdout'] != b'': + raise ValueError('governed original producer output differs') + if action == 'probe-refusals' and json.loads(originals['stdout']) != { + 'actualRefusals': ['wheel-bytes', 'wheel-name', 'image', 'interpreter', 'pip', 'pip-entrypoint', 'wheel-policy'], + 'sourceAdmitted': False, 'candidateQualified': False}: + raise ValueError('governed original runtime refusals differ') + if action == 'mutations' and json.loads(originals['stdout']) != { + 'completeOriginalPositive': 'PASS', 'mutationsRejected': ['context-loss', 'step-loss', 'step-reorder', + 'ordinal', 'command', 'block', 'extras-superset', 'editable', 'source-before', 'source-resolved', + 'source-after', 'installed-before', 'installed-after', 'derived-command', 'install-command', + 'raw-report', 'backend-command', 'editable-identity', 'published', 'admitted', 'candidate'], + 'sourceAdmitted': False, 'candidateQualified': False, 'publishedWheelIdentity': False}: + raise ValueError('governed original complete-proposal controls differ') + return {'implementationSha': implementation_sha, 'subjectSourceSha': subject_sha, + 'innerProposalSha256': receipt['innerProposalSha256'], 'actions': list(ACTIONS), + 'executionContextSha256': receipt['executionContextSha256'], + 'innerArtifactVerificationRequired': True, + 'sourceAdmitted': False, 'candidateQualified': False, 'publishedWheelIdentity': False} + + +def run(command: list[str]) -> bytes: + environment = {key: os.environ[key] for key in ('PATH','HOME','LANG','SYSTEMROOT') if key in os.environ} + return subprocess.check_output(command, env=environment, stderr=subprocess.PIPE) + + +def clone(source: Path, destination: Path, revision: str) -> None: + actual = run(['git','-C',str(source),'rev-parse','HEAD']).decode().strip() + if actual != revision or run(['git','-C',str(source),'status','--porcelain','--untracked-files=all']).strip(): + raise ValueError('input checkout is not the requested immutable clean revision') + remote = run(['git','-C',str(source),'remote','get-url','origin']).decode().strip() + if remote.removesuffix('.git') not in (REPOSITORY,'git@github.com:honua-io/honua-sdk-python'): + raise ValueError('input repository identity differs') + run(['git','clone','--quiet','--no-hardlinks','--no-checkout',str(source),str(destination)]) + run(['git','-C',str(destination),'checkout','--quiet','--detach',revision]) + run(['git','-C',str(destination),'remote','set-url','origin',REPOSITORY]) + if run(['git','-C',str(destination),'status','--porcelain','--untracked-files=all']).strip(): + raise ValueError('isolated clone is not clean') + + +def inspect_container(container: str) -> dict: + # Only fixed, safe daemon projections; never capture Config.Env. + template = '{"id":{{json .Id}},"imageId":{{json .Image}},"declaredImage":{{json .Config.Image}},"path":{{json .Path}},"args":{{json .Args}},"user":{{json .Config.User}},"readonlyRootfs":{{json .HostConfig.ReadonlyRootfs}},"mounts":{{json .Mounts}},"tmpfs":{{json .HostConfig.Tmpfs}},"exitCode":{{json .State.ExitCode}},"status":{{json .State.Status}},"running":{{json .State.Running}},"startedAt":{{json .State.StartedAt}},"finishedAt":{{json .State.FinishedAt}},"ownerNonce":{{json (index .Config.Labels "io.honua.development-input-owner")}}}' + return json.loads(run(['docker','inspect','--format',template,container])) + + +def validate_container(observed: dict, container: str, *, owner: str, image_id: str, arguments: list[str], mounts: list, user: str, tmpfs: str) -> None: + tmpfs_root,tmpfs_options = tmpfs.split(':',1) + if (observed['id'] != container or observed['ownerNonce'] != owner or observed['user'] != user + or observed['imageId'] != image_id or observed['declaredImage'] != IMAGE or observed['readonlyRootfs'] is not True + or observed['path'] != arguments[0] or observed['args'] != arguments[1:] + or observed.get('tmpfs') != {tmpfs_root: tmpfs_options}): + raise ValueError('actual container identity, command or private tmpfs differs') + actual_binds, temporary = [], [] + for row in observed['mounts']: + if type(row['RW']) is not bool: + raise ValueError('daemon mount read/write flag is not boolean') + if row['Type'] == 'bind': + actual_binds.append((row['Source'],row['Destination'],row['RW'])) + elif row['Type'] == 'tmpfs' and row['Source'] == '' and row['Destination'] == tmpfs_root and row['RW'] is True: + temporary.append(row) + else: + raise ValueError('unexpected daemon mount') + expected = [(str(source),target,not readonly) for source,target,readonly in mounts] + if len(actual_binds) != len(expected) or set(actual_binds) != set(expected) or len(temporary) > 1: + raise ValueError('actual readonly bind mount census differs') + + +def exited(observed: dict) -> bool: + return (observed['running'] is False and observed['status'] == 'exited' and type(observed['exitCode']) is int + and not observed['startedAt'].startswith('0001-') and not observed['finishedAt'].startswith('0001-')) + + +def wait_for_terminal(container: str, observed: dict, output: Path, validate) -> dict: + # A detached transport failure is never authority to cancel admitted work. + while observed['running'] is True: + output.write_text(json.dumps({'status':'pending-live-action','container':observed,'sourceAdmitted':False},indent=2)+'\n') + time.sleep(2) + observed = inspect_container(container) + output.with_suffix('.observed.json').write_text(json.dumps(observed,indent=2)+'\n') + validate(observed, container) + return observed + + +def capture_action(container: str, action: str, destination: Path, validate) -> tuple[bytes,bytes,dict]: + environment = {key: os.environ[key] for key in ('PATH','HOME','LANG','SYSTEMROOT') if key in os.environ} + result = subprocess.run(['docker','start','--attach',container],env=environment,capture_output=True,check=False) # noqa: S603 + # Both streams and actual daemon state are retained BEFORE enforcing success. + (destination/(action+'.stdout')).write_bytes(result.stdout) + (destination/(action+'.stderr')).write_bytes(result.stderr) + after = inspect_container(container) + (destination/(action+'.container-after.json')).write_text(json.dumps(after,indent=2)+'\n') + validate(after,container) + after = wait_for_terminal(container,after,destination/(action+'.pending.json'),validate) + (destination/(action+'.container-after.json')).write_text(json.dumps(after,indent=2)+'\n') + (destination/(action+'.attach.json')).write_text(json.dumps({'returnCode':result.returncode,'sourceAdmitted':False})+'\n') + if result.returncode != 0 or not exited(after) or after['exitCode'] != 0: + raise ValueError('governed producer action failed; original streams and terminal state retained') + return result.stdout,result.stderr,after + + +def cleanup_owned(owner: str, action: str, destination: Path, validate) -> dict: + owned = run(['docker','ps','--all','--quiet','--no-trunc','--filter','label=io.honua.development-input-owner='+owner]).decode().splitlines() + if len(owned) != 1: + raise ValueError('owned container cleanup census is missing or ambiguous') + candidate = owned[0] + observed = inspect_container(candidate) + (destination/(action+'.cleanup-observed.json')).write_text(json.dumps(observed,indent=2)+'\n') + validate(observed,candidate) + observed = wait_for_terminal(candidate,observed,destination/(action+'.cleanup-pending.json'),validate) + never_started = (observed['running'] is False and observed['status'] == 'created' + and observed['startedAt'].startswith('0001-') and observed['finishedAt'].startswith('0001-')) + if not exited(observed) and not never_started: + raise ValueError('refusing cleanup without a terminal or never-started owned state') + cleanup = run(['docker','rm',candidate]) # Normal removal only; no force/cancel. + if cleanup.decode().strip() != candidate: + raise ValueError('normal owned container removal did not acknowledge exact identity') + result = {'ownedContainerId':candidate,'ownerNonce':owner,'originalState':observed,'removed':True,'sourceAdmitted':False} + (destination/(action+'.cleanup.json')).write_text(json.dumps(result,indent=2)+'\n') + return result + + +def execute(implementation: Path, subject: Path, revision: str, destination: Path, wheel: Path) -> dict: + implementation,subject,wheel = implementation.resolve(),subject.resolve(),wheel.resolve() + if destination.exists() or destination.resolve().is_relative_to(implementation) or destination.resolve().is_relative_to(subject): + raise ValueError('runtime output must be new and outside both immutable inputs') + destination.mkdir(mode=0o700) + destination = destination.resolve() + implementation_sha = run(['git','-C',str(implementation),'rev-parse','HEAD']).decode().strip() + policy = json.loads((implementation/'scripts/development-inputs/governed-runtime.json').read_bytes()) + binding = policy['executionBinding'] + if binding['containerImage'] != IMAGE or wheel.name != binding['implementationPackagingFilename'] or hashlib.sha256(wheel.read_bytes()).hexdigest() != binding['implementationPackagingSha256']: + raise ValueError('runtime image or implementation tool bytes differ') + image = json.loads(run(['docker','image','inspect','--format','{"Id":{{json .Id}},"RepoDigests":{{json .RepoDigests}},"Os":{{json .Os}},"Architecture":{{json .Architecture}}}',IMAGE])) + if (not isinstance(image.get('Id'),str) or not image['Id'].startswith('sha256:') or len(image['Id']) != 71 + or any(character not in '0123456789abcdef' for character in image['Id'][7:]) + or image['Os'] != 'linux' or image['Architecture'] != 'amd64' + or not any(item in {'python@'+IMAGE.split('@')[1],'docker.io/library/python@'+IMAGE.split('@')[1]} for item in image['RepoDigests'])): + raise ValueError('actual daemon image differs from governed digest') + image_receipt = {'id':image['Id'],'repoDigests':image['RepoDigests'],'declaredImage':IMAGE, + 'os':image['Os'],'architecture':image['Architecture']} + (destination/'image.json').write_text(json.dumps(image_receipt,indent=2)+'\n') + commands,containers = [],[] + scratch = Path(tempfile.mkdtemp(prefix='honua-governed-inputs-',dir=str(Path(tempfile.gettempdir()).resolve()))) + pending = False + try: + producer_copy,subject_copy = scratch/'implementation',scratch/'subject' + clone(implementation,producer_copy,implementation_sha) + clone(subject,subject_copy,revision) + context = {'mountSources': {'implementation': str(producer_copy), 'subject': str(subject_copy), + 'tool': str(wheel), 'output': str(destination)}, 'user': str(os.getuid())+':'+str(os.getgid()), + 'tmpfs': f'/tmp:rw,exec,mode=0700,uid={os.getuid()},gid={os.getgid()}', 'image': image_receipt} # noqa: S108 -- private tmpfs + action_inputs(ACTIONS[0], revision, context, policy) + context_bytes = (json.dumps(context, indent=2)+'\n').encode() + # Original safe request, captured before any producer action. Consumer + # authority also requires protected implementation/run/request provenance. + (destination/'execution-context.json').write_bytes(context_bytes) + for action in ACTIONS: + arguments, mounts, user, tmpfs = action_inputs(action, revision, context, policy) + owner = secrets.token_hex(16) + command = create_command(owner, arguments, mounts, user, tmpfs) + validate = partial(validate_container,owner=owner,image_id=image['Id'],arguments=arguments,mounts=mounts,user=user,tmpfs=tmpfs) + try: + container = run(command).decode().strip() + if len(container) != 64 or any(character not in '0123456789abcdef' for character in container): + raise ValueError('daemon returned malformed owned container ID') + before = inspect_container(container) + (destination/(action+'.container-before.json')).write_text(json.dumps(before,indent=2)+'\n') + validate(before,container) + output,errors,after = capture_action(container,action,destination,validate) + commands.append({'action':action,'command':command,'stdoutSha256':hashlib.sha256(output).hexdigest(),'stderrSha256':hashlib.sha256(errors).hexdigest()}) + containers.append({'action':action,'before':before,'after':after}) + finally: + try: + cleanup = cleanup_owned(owner,action,destination,validate) + if containers and containers[-1]['action'] == action: + containers[-1]['cleanup'] = cleanup + except Exception: + pending = True + raise + finally: + if pending: + (destination/'scratch-pending.json').write_text(json.dumps({'scratchRoot':str(scratch),'status':'retained-for-live-or-unverified-owned-action','sourceAdmitted':False},indent=2)+'\n') + else: + shutil.rmtree(scratch) + receipt = {'schema':'honua.python-governed-input-execution/v1','implementationSha':implementation_sha, + 'subjectSourceSha':revision,'image':image_receipt,'implementationPackaging':{'filename':wheel.name,'sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'installed':False}, + 'commands':commands,'containers':containers, + 'innerProposalSha256':hashlib.sha256((destination/'proposal/proposal.json').read_bytes()).hexdigest(), + 'executionContextSha256':hashlib.sha256(context_bytes).hexdigest(), + 'sourceAdmitted':False,'candidateQualified':False,'publishedWheelIdentity':False} + (destination/'execution.json').write_text(json.dumps(receipt,indent=2)+'\n') + return receipt + + +def download_implementation_tool(policy: dict, directory: Path) -> tuple[Path, dict]: + binding = policy["executionBinding"] + registry = "https://pypi.org/pypi/packaging/" + policy["packaging"] + "/json" + with urlopen(registry) as response: # fixed public registry; no operator URL + if response.url != registry: + raise ValueError("implementation tool registry redirected") + raw = response.read() + values = [item for item in json.loads(raw)["urls"] if item["filename"] == binding["implementationPackagingFilename"]] + if len(values) != 1 or values[0]["digests"]["sha256"] != binding["implementationPackagingSha256"]: + raise ValueError("implementation tool registry identity differs") + origin = values[0]["url"] + parts = urlsplit(origin) + if (parts.scheme != "https" or parts.hostname != "files.pythonhosted.org" or parts.netloc != "files.pythonhosted.org" + or not parts.path.startswith("/packages/") or not parts.path.endswith("/" + binding["implementationPackagingFilename"]) + or parts.query or parts.fragment): + raise ValueError("implementation tool origin is not canonical") + with urlopen(origin) as response: # noqa: S310 -- exact canonical HTTPS origin validated above + if response.url != origin: + raise ValueError("implementation tool origin redirected") + data = response.read() + if hashlib.sha256(data).hexdigest() != binding["implementationPackagingSha256"]: + raise ValueError("implementation tool download bytes differ") + wheel = directory / binding["implementationPackagingFilename"] + wheel.write_bytes(data) + return wheel, {"filename": wheel.name, "origin": origin, "sha256": hashlib.sha256(data).hexdigest(), + "registry": registry, "registrySha256": hashlib.sha256(raw).hexdigest(), "installed": False} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--implementation',type=Path,required=True) + parser.add_argument('--source',type=Path,required=True) + parser.add_argument('--revision',required=True) + parser.add_argument('--destination',type=Path,required=True) + parser.add_argument('--implementation-packaging-wheel',type=Path) + args=parser.parse_args() + if args.implementation_packaging_wheel: + execute(args.implementation,args.source,args.revision,args.destination,args.implementation_packaging_wheel) + else: + policy = json.loads((args.implementation/'scripts/development-inputs/governed-runtime.json').read_bytes()) + with tempfile.TemporaryDirectory(prefix='honua-implementation-tool-',dir=str(Path(tempfile.gettempdir()).resolve())) as temporary: + wheel, origin = download_implementation_tool(policy, Path(temporary)) + execute(args.implementation,args.source,args.revision,args.destination,wheel) + (args.destination/'implementation-tool-origin.json').write_text(json.dumps(origin,indent=2)+'\n') + + +if __name__=='__main__': + main() diff --git a/tests/fixtures/governed_mount_order.json b/tests/fixtures/governed_mount_order.json new file mode 100644 index 0000000..18f4146 --- /dev/null +++ b/tests/fixtures/governed_mount_order.json @@ -0,0 +1,188 @@ +{ + "meaning": "Retained actual Docker repeated-inspect mount-order regression only; no candidate or changed-source proof", + "sourceHead": "9ada61bf981b949ddcef5e5b47e9ac98d78bf2a6", + "runId": 39391902735, + "jobId": 118525181411, + "originalExecutionSha256": "51cab1dbb5f7c327c321617649a7a2d92398ce2672c626060f3d927506d9e8f8", + "originalAfterSha256": "360458d0c70783a5b46a30c5295f96aa1a3ef77cc8f446c6c38673570b6f1111", + "originalCleanupObservedSha256": "f23a0085b3288c7e347dad9442566a579897a22936bac655bb260179c6066f18", + "after": { + "id": "2ce5c6d82dbda78c4c672486f599d583851c8c6f6af886cb1f095de3e8c07aa4", + "imageId": "sha256:4e32743734599c5dbfcd60a97e60804ccbb674e967ea418509f9eeedc038d8bb", + "declaredImage": "python:3.12-bookworm@sha256:e91fec3d1ac69f04e4eddcd29c327e630ce34658cf31075bfa7e8b0e052bafea", + "path": "/usr/local/bin/python3.12", + "args": [ + "-I", + "/implementation/scripts/governed_source_inputs_bootstrap.py", + "--implementation-packaging-wheel", + "/implementation-tools/packaging-26.3-py3-none-any.whl", + "--action", + "probe", + "--source", + "/subject", + "--revision", + "5a79352066b8aa23f1bae45b5c38e6d83efd003f", + "--destination", + "/output/proposal", + "--runtime-policy", + "/implementation/scripts/development-inputs/governed-runtime.json" + ], + "user": "1001:1001", + "readonlyRootfs": true, + "mounts": [ + { + "Destination": "/implementation-tools/packaging-26.3-py3-none-any.whl", + "Mode": "", + "Propagation": "rprivate", + "RW": false, + "Source": "/tmp/honua-implementation-tool-wjfe_wn7/packaging-26.3-py3-none-any.whl", + "Type": "bind" + }, + { + "Destination": "/output", + "Mode": "", + "Propagation": "rprivate", + "RW": true, + "Source": "/home/runner/work/_temp/governed-development-inputs", + "Type": "bind" + }, + { + "Destination": "/implementation", + "Mode": "", + "Propagation": "rprivate", + "RW": false, + "Source": "/tmp/honua-governed-inputs-qalyf0gw/implementation", + "Type": "bind" + }, + { + "Destination": "/subject", + "Mode": "", + "Propagation": "rprivate", + "RW": false, + "Source": "/tmp/honua-governed-inputs-qalyf0gw/subject", + "Type": "bind" + } + ], + "tmpfs": { + "/tmp": "rw,exec,mode=0700,uid=1001,gid=1001" + }, + "exitCode": 0, + "status": "exited", + "running": false, + "startedAt": "2026-10-20T08:14:23.026378911Z", + "finishedAt": "2026-10-20T08:14:24.135930887Z", + "ownerNonce": "a376fe877dded63a9e8f2353459c457e" + }, + "cleanupObserved": { + "id": "2ce5c6d82dbda78c4c672486f599d583851c8c6f6af886cb1f095de3e8c07aa4", + "imageId": "sha256:4e32743734599c5dbfcd60a97e60804ccbb674e967ea418509f9eeedc038d8bb", + "declaredImage": "python:3.12-bookworm@sha256:e91fec3d1ac69f04e4eddcd29c327e630ce34658cf31075bfa7e8b0e052bafea", + "path": "/usr/local/bin/python3.12", + "args": [ + "-I", + "/implementation/scripts/governed_source_inputs_bootstrap.py", + "--implementation-packaging-wheel", + "/implementation-tools/packaging-26.3-py3-none-any.whl", + "--action", + "probe", + "--source", + "/subject", + "--revision", + "5a79352066b8aa23f1bae45b5c38e6d83efd003f", + "--destination", + "/output/proposal", + "--runtime-policy", + "/implementation/scripts/development-inputs/governed-runtime.json" + ], + "user": "1001:1001", + "readonlyRootfs": true, + "mounts": [ + { + "Destination": "/implementation", + "Mode": "", + "Propagation": "rprivate", + "RW": false, + "Source": "/tmp/honua-governed-inputs-qalyf0gw/implementation", + "Type": "bind" + }, + { + "Destination": "/subject", + "Mode": "", + "Propagation": "rprivate", + "RW": false, + "Source": "/tmp/honua-governed-inputs-qalyf0gw/subject", + "Type": "bind" + }, + { + "Destination": "/implementation-tools/packaging-26.3-py3-none-any.whl", + "Mode": "", + "Propagation": "rprivate", + "RW": false, + "Source": "/tmp/honua-implementation-tool-wjfe_wn7/packaging-26.3-py3-none-any.whl", + "Type": "bind" + }, + { + "Destination": "/output", + "Mode": "", + "Propagation": "rprivate", + "RW": true, + "Source": "/home/runner/work/_temp/governed-development-inputs", + "Type": "bind" + } + ], + "tmpfs": { + "/tmp": "rw,exec,mode=0700,uid=1001,gid=1001" + }, + "exitCode": 0, + "status": "exited", + "running": false, + "startedAt": "2026-10-20T08:14:23.026378911Z", + "finishedAt": "2026-10-20T08:14:24.135930887Z", + "ownerNonce": "a376fe877dded63a9e8f2353459c457e" + }, + "expected": { + "owner": "a376fe877dded63a9e8f2353459c457e", + "image_id": "sha256:4e32743734599c5dbfcd60a97e60804ccbb674e967ea418509f9eeedc038d8bb", + "arguments": [ + "/usr/local/bin/python3.12", + "-I", + "/implementation/scripts/governed_source_inputs_bootstrap.py", + "--implementation-packaging-wheel", + "/implementation-tools/packaging-26.3-py3-none-any.whl", + "--action", + "probe", + "--source", + "/subject", + "--revision", + "5a79352066b8aa23f1bae45b5c38e6d83efd003f", + "--destination", + "/output/proposal", + "--runtime-policy", + "/implementation/scripts/development-inputs/governed-runtime.json" + ], + "mounts": [ + [ + "/tmp/honua-governed-inputs-qalyf0gw/implementation", + "/implementation", + true + ], + [ + "/tmp/honua-governed-inputs-qalyf0gw/subject", + "/subject", + true + ], + [ + "/tmp/honua-implementation-tool-wjfe_wn7/packaging-26.3-py3-none-any.whl", + "/implementation-tools/packaging-26.3-py3-none-any.whl", + true + ], + [ + "/home/runner/work/_temp/governed-development-inputs", + "/output", + false + ] + ], + "user": "1001:1001", + "tmpfs": "/tmp:rw,exec,mode=0700,uid=1001,gid=1001" + } +} diff --git a/tests/fixtures/python_source_install_blocks.json b/tests/fixtures/python_source_install_blocks.json new file mode 100644 index 0000000..8bfbe42 --- /dev/null +++ b/tests/fixtures/python_source_install_blocks.json @@ -0,0 +1,49 @@ +{ + "meaning": "Immutable original document regression inputs only; no runtime or admission proof", + "sourceSha": "5a79352066b8aa23f1bae45b5c38e6d83efd003f", + "documents": { + "README.md": { + "text": "# Honua Python SDK\n\n[![CI](https://github.com/honua-io/honua-sdk-python/actions/workflows/ci.yml/badge.svg?branch=trunk)](https://github.com/honua-io/honua-sdk-python/actions/workflows/ci.yml)\n[![Conformance](https://github.com/honua-io/honua-sdk-python/actions/workflows/conformance.yml/badge.svg?branch=trunk)](https://github.com/honua-io/honua-sdk-python/actions/workflows/conformance.yml)\n[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/honua-io/honua-sdk-python/badge)](https://scorecard.dev/viewer/?uri=github.com/honua-io/honua-sdk-python)\n[![Docs](https://img.shields.io/badge/docs-latest-blue)](https://honua-io.github.io/honua-sdk-python/latest/)\n[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)\n\nPython client libraries for [Honua](https://honua.io), the cloud-native\ngeospatial platform built around\n[honua-server](https://github.com/honua-io/honua-server). One typed client\ncovers feature queries, geocoding, map/image export, editing, and server\nadministration across the server's protocol adapters \u2014 GeoServices REST\n(FeatureServer/MapServer/ImageServer/GeocodeServer/GeometryServer), OGC API\nFeatures, STAC, OData, WFS/WMS/WMTS, and gRPC streaming \u2014 with one-call\nGeoDataFrame interop for the pandas/GeoPandas stack.\n\n## Packages\n\nTwo independently installable, Apache-2.0 packages live under `packages/`:\n\n| Package | Import | Description |\n|---------|--------|-------------|\n| [`packages/honua-sdk`](packages/honua-sdk/) | `honua_sdk` | Data-plane client \u2014 feature queries, geocoding, protocol clients, gRPC streaming, GeoPandas/raster interop, `honua` CLI |\n| [`packages/honua-admin`](packages/honua-admin/) | `honua_admin` | Control-plane client \u2014 services, connections, layers, styles, metadata, manifests, compatibility checks (depends on `honua-sdk`) |\n\nAlso in this repo: [`packages/honua-gp`](packages/honua-gp/), a proprietary\ngeoprocessing compatibility layer for teams migrating scripts from ArcGIS\n`arcpy` (separate license, not published). Import it as `honua_gp`.\n\n## Status\n\nAlpha (`0.x`). APIs may change before 1.0; breaking changes to the public API\nare gated by a [compatibility snapshot](docs/compatibility.md) and a\nper-capability [SDK coverage snapshot](docs/sdk-coverage.md).\n\nReleases are automated with release-please and PyPI Trusted Publishing and\nship to PyPI as `honua-sdk` and `honua-admin`. After release-please creates a\npackage tag and GitHub Release, the publication workflow reconciles that exact\nrelease commit, publishes the SDK before the dependent admin package, and\nattaches the wheel and source distribution to the GitHub Release.\n\n## Install\n\nRequires Python 3.11+ (CI tests 3.11, 3.12, 3.13). From PyPI:\n\n```bash\npip install honua-sdk # data-plane client\npip install \"honua-sdk[grpc,geopandas,raster]\" # + optional extras\npip install honua-admin # control-plane (admin) client\n```\n\nFrom source (always works, and the path for development):\n\n```bash\ngit clone https://github.com/honua-io/honua-sdk-python.git\ncd honua-sdk-python\n\n# Data-plane client\npip install ./packages/honua-sdk\n\n# With optional extras: gRPC streaming, GeoPandas vector interop,\n# raster interop (rasterio / rioxarray / xarray)\npip install \"./packages/honua-sdk[grpc,geopandas,raster]\"\n\n# Control-plane (admin) client \u2014 installs honua-sdk alongside it\npip install ./packages/honua-sdk ./packages/honua-admin\n```\n\nFor the published SDK with GeoPandas support, use the PyPI extra:\n\n```bash\npip install \"honua-sdk[geopandas]\"\n```\n\nThis uses the configured package index and version constraints, just like the\ncore install above. For a reproducible release, pin the same SDK version for\nboth the core package and its extras; installing an older Git tag would replace\nthe installed SDK with that older version.\n\nThe repo-root `pyproject.toml` is intentionally **not** installable (it holds\nshared tool config only) \u2014 install the per-package directories, not `.`.\n[INSTALL.md](INSTALL.md) has extras details and install-time troubleshooting.\n\n## I want to...\n\n| Goal | Start here |\n|---------------------------------|-----------------------------------------------------------------|\n| Query features in 5 minutes | [docs/quickstart.md](docs/quickstart.md) |\n| Stream features over gRPC | [INSTALL.md#with-grpc](INSTALL.md#with-grpc) |\n| Build an ETL pipeline | [examples/geospatial_etl/](examples/geospatial_etl/) |\n| Wire a FastAPI service | [examples/fastapi_spatial_service.py](examples/fastapi_spatial_service.py) |\n| Run spatial queries from Jupyter / pandas | [docs/quickstart.md](docs/quickstart.md) + [examples/data_quality_report.py](examples/data_quality_report.py) |\n| Manage services & connections | [packages/honua-admin/](packages/honua-admin/) |\n| Understand the protocol matrix | [docs/protocol-parity.md](docs/protocol-parity.md) |\n| Diagnose an error | [docs/quickstart.md#common-errors](docs/quickstart.md#common-errors) |\n\n## Quick start\n\nQuery features through the canonical `Source` / `Query` / `Result` API and\nconvert to a GeoDataFrame in one call. The examples below read the public,\nanonymous demo server at `https://demo.honua.io` and its `maui-zoning`\nFeatureServer (layer `2`: zoning polygons with `zone_code`, `zone_dist`,\n`cp_area` and `island`). To use your own server, change the base URL, service\nand layer.\n\n```python\nfrom honua_sdk import HonuaClient, Query, SourceDescriptor, SourceLocator\n\nwith HonuaClient(\"https://demo.honua.io\") as client:\n source = client.source(\n SourceDescriptor(\n id=\"maui-zoning\",\n protocol=\"geoservices-feature-service\",\n locator=SourceLocator(service_id=\"maui-zoning\", layer_id=2),\n )\n )\n result = source.query(Query(where=\"island = 'Maui'\", out_fields=[\"*\"]))\n\n print(f\"Found {len(result.features)} features\")\n for feature in result.features[:3]:\n print(feature.id, feature.properties)\n\n # Requires the [geopandas] extra\n gdf = result.to_geodataframe() # GeoDataFrame with geometry column + CRS set\n print(gdf.head(), gdf.crs)\n```\n\n`client.source(...)` returns a source-bound facade with `query()`,\n`query_all()`, `stream()`/`iter_features()`, `apply_edits()`, and\n`protocol(...)`. `source.query()` returns a canonical `Result` of normalized\n`QueryFeature` entries (`id`, `properties`, `geometry`, `protocol`, `source`,\n`raw`) \u2014 the same shape across FeatureServer, OGC API Features, STAC, and\nOData. The reverse helper `honua_sdk.geopandas.geodataframe_to_features` turns\nan edited GeoDataFrame back into `apply_edits` payloads.\n\n### OGC API Features\n\nThe server advertises each layer's OGC API Features collection id in\n`collections()`; the id comes from the layer's publication, so it is not\nalways the layer id. On the demo publication the `maui-zoning` layer above is\ncollection `\"2\"`; on your own server, pick the id from `collections()`:\n\n```python\nfrom honua_sdk import HonuaClient\n\nwith HonuaClient(\"https://demo.honua.io\") as client:\n ogc = client.ogc_features()\n collections = ogc.collections()\n\n zoning = ogc.collection(\"2\")\n items = zoning.items(limit=100, filter=\"island = 'Maui'\")\n feature = zoning.item(items[\"features\"][0][\"id\"])\n```\n\n### Geocoding\n\n```python\nfrom honua_sdk import HonuaGeocodingClient\n\nwith HonuaGeocodingClient(\"https://demo.honua.io\") as geocoder:\n results = geocoder.forward_geocode(\"1600 Pennsylvania Ave NW, Washington, DC\")\n for r in results:\n print(f\"{r.address} ({r.latitude}, {r.longitude}) score={r.score}\")\n```\n\n### Async\n\nEvery HTTP workflow has an async counterpart with the same factory and method\nnames \u2014 works with FastAPI, asyncio pipelines, and Jupyter:\n\n```python\nimport asyncio\n\nfrom honua_sdk import AsyncHonuaClient, Query, SourceDescriptor, SourceLocator\n\nasync def main() -> None:\n async with AsyncHonuaClient(\"https://demo.honua.io\") as client:\n source = client.source(\n SourceDescriptor(\n id=\"maui-zoning\",\n protocol=\"geoservices-feature-service\",\n locator=SourceLocator(service_id=\"maui-zoning\", layer_id=2),\n )\n )\n result = await source.query(Query(where=\"1=1\"))\n print(f\"Found {len(result.features)} features\")\n\n\nasyncio.run(main())\n```\n\n`asyncio.run(main())` is for scripts. Jupyter and IPython already run an event\nloop, where `asyncio.run` raises `RuntimeError`; call `await main()` in a\nnotebook cell instead.\n\n### Admin client\n\nAdmin needs a server of your own: the public demo returns `401` for\n`/api/v1/admin/*`. Install `honua-admin`, then set `HONUA_API_KEY` to an admin\nAPI key for your server (the\n[honua-server quickstart](https://github.com/honua-io/honua-server/blob/trunk/docs/get-started/quickstart.md)\nshows how to mint one with `HONUA_ADMIN_PASSWORD`).\n\n```python\nimport os\n\nfrom honua_admin import HonuaAdminClient\n\n# Admin needs a server of your own: the public demo returns 401 for /api/v1/admin/*.\n# Mint a key with HONUA_ADMIN_PASSWORD on your server - see the honua-server quickstart.\nwith HonuaAdminClient(\"https://your-honua-server.com\", api_key=os.environ[\"HONUA_API_KEY\"]) as admin:\n compatibility = admin.check_compatibility()\n if not compatibility.supported:\n raise RuntimeError(\"; \".join(compatibility.reasons))\n\n features = admin.get_capability_flags()\n if features.manifest_apply:\n manifest = admin.get_manifest()\n print(f\"Manifest resources: {len(manifest.resources)}\")\n```\n\n> **Protocol IDs.** Docs and examples use the canonical cross-SDK protocol ids\n> (`geoservices-feature-service`, `ogc-features`, `stac`, `odata`, ...).\n> Common aliases (`feature-server`, `ogc-api-features`, ...) are accepted at\n> runtime and normalized by `honua_sdk.normalize_protocol(...)`; the full\n> table lives in `honua_sdk.PROTOCOL_ALIASES`. Compact helpers\n> (`client.query(...)`, `client.query_features(...)`) remain for one-liners \u2014\n> see [Protocol Examples](docs/protocol-examples.md) for every wrapper and\n> [Protocol Parity](docs/protocol-parity.md) for the Python/JS coverage map.\n\n## Key features\n\n| | |\n|---|---|\n| Typed, canonical query surface | `Source` / `Query` / `Result` with normalized `QueryFeature` across FeatureServer, OGC Features, STAC, OData |\n| Protocol clients | GeoServices (Feature/Map/Image/Geocode/Geometry/Scene/Elevation servers), OGC API Features/Maps/Tiles/Coverages/Processes/Records, STAC, OData, WFS, WMS, WMTS, geoprocessing + workflow wrappers \u2014 see [protocol parity](docs/protocol-parity.md) |\n| GIS interop | `Result.to_geodataframe()`, `features_to_geodataframe` (Esri JSON aware), raster results via `rasterio`/`rioxarray` (`[raster]` extra) |\n| gRPC streaming | `honua_sdk.grpc.HonuaGrpcClient` / `HonuaGrpcAsyncClient` for unary + streaming feature queries (`[grpc]` extra) |\n| Sync + async | `HonuaClient` / `AsyncHonuaClient` in lockstep (sync clients generated from the async source of truth) |\n| Automatic retry | 429/502/503/504 with exponential backoff and `Retry-After` support for idempotent methods (`POST` only when opted in via `retry_methods` on the retry transport); configurable via `max_retries` |\n| Typed errors | `HonuaAuthError`, `HonuaRateLimitError`, `HonuaHttpError`, `HonuaTimeoutError`, `HonuaTransportError` \u2014 see [common errors](docs/quickstart.md#common-errors) |\n| CLI | `honua` (services / layers / style apply / sanitized `doctor` diagnostics) and `honua-migrate` (offline ArcPy script scan / translate / run, plus `.pyt` / `.atbx` toolbox and GP-service classification, with optional server-attested toolbox translation verdicts via `--server`) |\n| Quality gates | mypy `strict` workspace-wide, 94% coverage gate, public-API [compatibility snapshot](docs/compatibility.md), per-capability [SDK coverage snapshot](docs/sdk-coverage.md), live-server [conformance lane](.github/workflows/conformance.yml) against shared [geospatial-grpc](https://github.com/honua-io/geospatial-grpc) fixtures |\n\n## Documentation\n\nThe rendered docs site lives at\n[honua-io.github.io/honua-sdk-python](https://honua-io.github.io/honua-sdk-python/latest/)\n(versioned MkDocs build of [docs/](docs/README.md)). Platform-level docs are\nat [honua.io](https://honua.io) and\n[honua.gitbook.io/honuaio](https://honua.gitbook.io/honuaio/).\n\n- [5-Minute Quickstart](docs/quickstart.md) \u2014 query, GeoDataFrame, plot, common errors\n- [Core Client](docs/core-client.md) \u2014 typed service, FeatureServer, applyEdits, pagination, error handling\n- [Protocol Examples](docs/protocol-examples.md) \u2014 OGC, STAC, WFS, WMS, WMTS, OData, geocoding, gRPC with response shapes\n- [Authentication](docs/auth.md) \u2014 refreshable bearer tokens, storage, rotation, failure modes\n- [Pagination](docs/pagination.md) and [retries & timeouts](docs/retries-and-timeouts.md)\n- [Sanitized diagnostic bundles](docs/diagnostic-bundles.md) \u2014 the `honua doctor` bundle format\n- [Geospatial ETL demo](examples/geospatial_etl/README.md) \u2014 script-first ETL flow with notebook companion\n- [Compatibility](docs/compatibility.md) \u2014 supported server matrix and public-API snapshot gate\n- [SDK coverage](docs/sdk-coverage.md) \u2014 per-capability coverage snapshot gate\n- [Troubleshooting](docs/troubleshooting.md) \u2014 base URL, auth, staging smoke env vars, cleanup\n\n## Related Honua repos\n\n| Repo | What it is |\n|------|------------|\n| [honua-server](https://github.com/honua-io/honua-server) | Flagship multi-protocol geospatial server this SDK talks to |\n| [honua-sdk-js](https://github.com/honua-io/honua-sdk-js) | JavaScript/TypeScript SDKs + MCP server |\n| [honua-sdk-dotnet](https://github.com/honua-io/honua-sdk-dotnet) | .NET SDKs |\n| [honua-console](https://github.com/honua-io/honua-console) | Display-and-approve web console; Studio is Preview and is not part of the qualified 2026.1 scope |\n| honua-qgis-plugin | QGIS plugin (private preview; repo not yet public) |\n| [geospatial-grpc](https://github.com/honua-io/geospatial-grpc) | Vendor-neutral gRPC protocol standard; source of this repo's conformance fixtures |\n\n## Development\n\nRun these from a clone of this repository (see [Install](#install)). Besides\nPython 3.11+, the test suite needs Node.js with `node` on `PATH`: the `honua`\nCLI forwards `honua admin ...` to the JavaScript CLI, and `tests/test_cli.py`\nchecks that forwarding.\n\n```bash\n# Editable install of both packages with extras\npip install -e \"packages/honua-sdk[grpc,geopandas]\"\npip install -e \"packages/honua-admin\"\npip install pytest pytest-cov ruff mypy\n\n# Lint + type-check (mypy runs in strict mode)\nruff check .\npython -m mypy packages/honua-sdk/honua_sdk packages/honua-admin/honua_admin\n\n# Deterministic local test suite\npython3 -m pytest tests/ -q\n\n# Public-API compatibility gate\npython3 scripts/compatibility_gate.py\n```\n\nSync clients (`client.py`, `_client.py`) are generated from their async\ncounterparts \u2014 edit the async module and run `python scripts/gen_sync.py`;\nnever hand-edit the generated files.\n\nOpt-in integration lanes (staging smoke, live-server conformance, release\nsmoke) need `HONUA_BASE_URL` and the `--run-integration` flag; see\n[docs/troubleshooting.md](docs/troubleshooting.md) and\n[docs/compatibility.md](docs/compatibility.md) for env vars, markers, and\nresult artifacts.\n\nBugs and feature requests: [GitHub issues](https://github.com/honua-io/honua-sdk-python/issues).\nPull requests are welcome \u2014 CI enforces lint, strict typing, the coverage\ngate, and the compatibility snapshot.\n\n## Security\n\nReport vulnerabilities to \u2014 see the\n[org security policy](https://github.com/honua-io/.github/blob/main/SECURITY.md).\nDo not open public issues for security reports.\n\n## License\n\n`honua-sdk` and `honua-admin` are licensed under\n[Apache-2.0](LICENSE). `packages/honua-gp` is proprietary (see its `LICENSE`)\nand is not published.\n", + "gitBlob": "73537ad2bc7c630d53e7825f880413e9aaa2850d" + }, + "INSTALL.md": { + "text": "# Installing the Honua Python SDK\n\n## Packages\n\n| Package | Description |\n|---------|-------------|\n| `honua-sdk` | Data-plane client for Honua Server -- REST queries, geocoding, gRPC features |\n| `honua-admin` | Control-plane / admin client for Honua Server (depends on `honua-sdk`) |\n\n## Prerequisites\n\n- Python 3.11 or later\n- A running Honua Server instance\n\n## Install\n\nThe canonical install path is PyPI:\n\n```bash\n# Core data client (REST/HTTP, sync + async)\npip install honua-sdk\n\n# With extras: gRPC, GeoPandas vector interop, raster interop\npip install \"honua-sdk[grpc,geopandas,raster]\"\n\n# Admin / control-plane client (installs honua-sdk alongside it)\npip install honua-admin\n```\n\nFor unreleased development work, install from a clone instead:\n\n```bash\ngit clone https://github.com/honua-io/honua-sdk-python.git\ncd honua-sdk-python\n\n# Core data client (REST/HTTP, sync + async)\npip install ./packages/honua-sdk\n\n# With gRPC support\npip install \"./packages/honua-sdk[grpc]\"\n\n# With GeoPandas integration (vector result interop)\npip install \"./packages/honua-sdk[geopandas]\"\n\n# With raster result interop (rasterio / rioxarray / xarray)\npip install \"./packages/honua-sdk[raster]\"\n\n# Admin / control-plane client (installs honua-sdk alongside it)\npip install ./packages/honua-sdk ./packages/honua-admin\n\n# Everything\npip install \"./packages/honua-sdk[grpc,geopandas,raster]\" ./packages/honua-admin\n```\n\nFor the published SDK with GeoPandas support, use the PyPI extra:\n\n```bash\npip install \"honua-sdk[geopandas]\"\n```\n\nThis uses the configured package index and version constraints, just like the\ncore install above. For a reproducible release, pin the same SDK version for\nboth the core package and its extras; installing an older Git tag would replace\nthe installed SDK with that older version.\n\nThe repo-root `pyproject.toml` is intentionally **not** installable (it\nholds shared tool config only) -- install the per-package directories,\nnot `.`.\n\n## Quick Start\n\nCheck the install against the public, anonymous demo server at\n`https://demo.honua.io`. Its `maui-zoning` FeatureServer publishes zoning\npolygons as layer `2`. To query your own server, change the base URL, service\nand layer.\n\n```python\nfrom honua_sdk import HonuaClient, Query, SourceDescriptor, SourceLocator\n\nwith HonuaClient(base_url=\"https://demo.honua.io\") as client:\n # Query features through the shared Source/Query/Result API\n source = client.source(\n SourceDescriptor(\n id=\"maui-zoning\",\n protocol=\"geoservices-feature-service\",\n locator=SourceLocator(service_id=\"maui-zoning\", layer_id=2),\n )\n )\n result = source.query(\n Query(where=\"island = 'Maui'\", out_fields=[\"*\"])\n )\n\n print(f\"Found {len(result.features)} features\")\n```\n\nThe context-manager form (`with HonuaClient(...) as client:`) is the\nrecommended default -- it guarantees underlying `httpx` connections are\nreturned to the pool when the block exits, even if a request raises.\n\n## With gRPC\n\ngRPC needs a server of your own, for example one started with the Honua\nServer quickstart. Install the `grpc` extra (`pip install \"honua-sdk[grpc]\"`). The\n[Honua Server quickstart](https://github.com/honua-io/honua-server/blob/trunk/docs/get-started/quickstart.md)\nserves plaintext gRPC (h2c) on port **8081**. Replace\n`your-honua-server.com:8081` with that address, and `maui-zoning` / `2` with\nthe service and layer id of one of your published layers:\n\n\n```python\nfrom honua_sdk.grpc import HonuaGrpcClient, QueryFeaturesRequest\n\nrequest = QueryFeaturesRequest(service_id=\"maui-zoning\", layer_id=2)\n\n# Local dev: plaintext channel (must opt in explicitly)\nwith HonuaGrpcClient(\"your-honua-server.com:8081\", insecure=True) as client:\n # Stream features\n for page in client.query_features_stream(request):\n print(f\"Page with {len(page.features)} features\")\n```\n\nIn production, terminate TLS in front of the gRPC listener and pass channel\ncredentials instead of `insecure=True`. Replace `grpc.your-honua-server.com:443`\nwith the `host:port` of that TLS endpoint:\n\n\n```python\nimport grpc\n\nfrom honua_sdk.grpc import HonuaGrpcClient, QueryFeaturesRequest\n\nrequest = QueryFeaturesRequest(service_id=\"maui-zoning\", layer_id=2)\n\nwith HonuaGrpcClient(\n \"grpc.your-honua-server.com:443\",\n credentials=grpc.ssl_channel_credentials(),\n) as client:\n for page in client.query_features_stream(request):\n print(f\"Page with {len(page.features)} features\")\n```\n\nThe constructor takes `target` positionally; pass exactly one of\n`credentials=`, `channel=`, or `insecure=True`. The same shape is used\nin [docs/quickstart.md](docs/quickstart.md#step-6-query-via-grpc-optional-60-seconds).\n\n## Admin\n\nAdmin needs a server of your own: the public demo returns `401` for\n`/api/v1/admin/*`. Set `HONUA_API_KEY` to an admin API key for your server (the\n[honua-server quickstart](https://github.com/honua-io/honua-server/blob/trunk/docs/get-started/quickstart.md)\nshows how to mint one with `HONUA_ADMIN_PASSWORD`).\n\n```python\nimport os\n\nfrom honua_admin import HonuaAdminClient\n\nwith HonuaAdminClient(\"https://your-honua-server.com\", api_key=os.environ[\"HONUA_API_KEY\"]) as admin:\n compatibility = admin.check_compatibility()\n if not compatibility.supported:\n raise RuntimeError(\"; \".join(compatibility.reasons))\n\n features = admin.get_capability_flags()\n if features.metadata_resources:\n print(\"Metadata resources are supported on this server.\")\n```\n\n## Version Policy\n\n- **Pre-release** (`0.x.xaN`, `0.x.xbN`): Published to PyPI with alpha/beta classifiers\n- **Stable** (`1.0.0+`): Published to PyPI as a stable release\n\nAll packages follow [Semantic Versioning](https://semver.org/). Major versions are coordinated across all Honua SDKs.\n\n## Admin Compatibility Checks\n\nThe admin SDK uses `GET /api/v1/admin/capabilities` as the runtime compatibility\nsource of truth. It currently expects:\n\n- server version `>= 1.0.0`, including the year-led release line (`2026.1.1`)\n- control-plane API major `v1`\n- release channel `preview` or newer\n\nThe coarse feature flags exposed today are:\n\n- `metadata_resources`\n- `manifest_export`\n- `manifest_apply`\n- `manifest_dry_run`\n- `manifest_prune`\n\n## Canonical vs legacy API\n\nNew code should prefer the canonical `Source` / `Query` / `Result`\nsurface:\n\n```python\nfrom honua_sdk import HonuaClient, Query, SourceDescriptor, SourceLocator\n\nwith HonuaClient(base_url=\"https://demo.honua.io\") as client:\n source = client.source(\n SourceDescriptor(\n id=\"maui-zoning\",\n protocol=\"geoservices-feature-service\",\n locator=SourceLocator(service_id=\"maui-zoning\", layer_id=2),\n )\n )\n result = source.query(Query(where=\"island = 'Maui'\", out_fields=[\"*\"]))\n for feature in result.features:\n # Typed ``QueryFeature`` -- attributes live under ``.properties``.\n print(feature.id, feature.properties)\n```\n\n`client.query_features(service_id, layer_id, where=...)` and the rest\nof the raw-dict FeatureServer helpers remain available as the **legacy\n/ compact form**. They return raw JSON dicts (FeatureServer\n`attributes`-shaped payloads) and are useful for one-liners, scripting,\nand protocol-debugging. New library code should reach for the canonical\nform so it gets:\n\n- typed `Result[QueryFeature]` with `.properties` / `.geometry` /\n `.protocol`\n- protocol-aware filter routing (CQL2-text vs SQL `WHERE`) -- including\n the `where_as_cql=True` opt-in for callers who *want* a SQL-style\n string forwarded as CQL on OGC Features or STAC\n- consistent pagination signals across FeatureServer, OGC Features,\n STAC, and OData\n\n## Troubleshooting\n\nSee [docs/troubleshooting.md](docs/troubleshooting.md) for the full\nguide. The most common install-time failures:\n\n- **gRPC wheel build fails on macOS Apple Silicon** -- upgrade pip\n (`python -m pip install --upgrade pip`) so it picks the prebuilt\n `grpcio` arm64 wheel instead of falling back to source.\n- **GeoPandas / Shapely fails on Windows** -- install the\n `honua-sdk[geopandas]` extra inside a `conda` env (or under WSL); the\n pip path on Windows requires a working GEOS / GDAL toolchain.\n- **Python 3.10 install fails with a version-pin error** -- the SDK\n requires Python 3.11+. Upgrade your interpreter or pin a 3.11+ venv.\n- **\"Microsoft Visual C++ 14.0 is required\" / \"command 'gcc' failed\"**\n -- a transitive dep is building from source because no wheel matched\n your platform. Install your platform's C compiler (Build Tools for\n Visual Studio on Windows, `xcode-select --install` on macOS,\n `build-essential` on Debian/Ubuntu) and re-run pip.\n", + "gitBlob": "cdb8cab9d5d11ec8f9d3db57b71c230ae70cca06" + } + }, + "expectedBlocks": [ + { + "document": "README.md", + "index": 1, + "sha256": "52ae3befaa5abb60f863381805ee1230de6081a89f466722946b5edf1e859755", + "commands": [ + "pip install ./packages/honua-sdk", + "pip install \"./packages/honua-sdk[grpc,geopandas,raster]\"", + "pip install ./packages/honua-sdk ./packages/honua-admin" + ] + }, + { + "document": "README.md", + "index": 8, + "sha256": "8e5ca8fcca606626e5908577b8d2293977b9f1e81f71b2a7b7a923ff689b2ea3", + "commands": [ + "pip install -e \"packages/honua-sdk[grpc,geopandas]\"", + "pip install -e \"packages/honua-admin\"", + "pip install pytest pytest-cov ruff mypy" + ] + }, + { + "document": "INSTALL.md", + "index": 1, + "sha256": "bb1abba46229bf51f71ff0c281708c4b2f37f3dce88b26eeb4b8473cbedeeb58", + "commands": [ + "pip install ./packages/honua-sdk", + "pip install \"./packages/honua-sdk[grpc]\"", + "pip install \"./packages/honua-sdk[geopandas]\"", + "pip install \"./packages/honua-sdk[raster]\"", + "pip install ./packages/honua-sdk ./packages/honua-admin", + "pip install \"./packages/honua-sdk[grpc,geopandas,raster]\" ./packages/honua-admin" + ] + } + ] +} diff --git a/tests/test_development_source_inputs.py b/tests/test_development_source_inputs.py new file mode 100644 index 0000000..237a3b4 --- /dev/null +++ b/tests/test_development_source_inputs.py @@ -0,0 +1,504 @@ +"""Synthetic tamper controls; actual PyPI/build positive is a separate workflow artifact.""" +from __future__ import annotations + +import base64 +from copy import deepcopy +import csv +import hashlib +import importlib.util +import marshal +import types +import io +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +from scripts import development_source_inputs as inputs +from scripts import python_document_inputs as documentation + + +class DevelopmentInputsTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + + def wheel(self, name="dep", version="1.0", requires=()): + filename = f"{name}-{version}-py3-none-any.whl" + value = io.BytesIO() + with zipfile.ZipFile(value, "w") as archive: + metadata = f"Metadata-Version: 2.3\nName: {name}\nVersion: {version}\n" + metadata += "".join(f"Requires-Dist: {item}\n" for item in requires) + archive.writestr(f"{name}-{version}.dist-info/METADATA", metadata) + data = value.getvalue() + (self.root / filename).write_bytes(data) + return {"name": name, "version": version, "filename": filename, + "origin": f"https://files.pythonhosted.org/packages/test/{filename}", + "sha256": inputs.digest(data), "requiresDist": list(requires)} + + def test_complete_recursive_closure_positive(self): + a = self.wheel("parent", requires=["child>=1"]) + b = self.wheel("child") + inputs.validate_closure([a, b], ["parent>=1"], self.root) + + def test_missing_transitive_refuses(self): + a = self.wheel("parent", requires=["child>=1"]) + with self.assertRaisesRegex(ValueError, "incomplete"): + inputs.validate_closure([a], ["parent>=1"], self.root) + + def test_artifact_changed_bytes_refuses(self): + a = self.wheel() + (self.root / a["filename"]).write_bytes(b"corrupt original wheel") + with self.assertRaisesRegex(ValueError, "digest"): + inputs.validate_closure([a], ["dep"], self.root) + + def test_coherent_metadata_substitution_refuses(self): + a = self.wheel() + a["version"] = "2.0" + with self.assertRaisesRegex(ValueError, "metadata"): + inputs.validate_closure([a], ["dep"], self.root) + + def test_unsupported_and_credentialized_origins_refuse(self): + for origin in ["https://attacker.invalid/a.whl", "file:///tmp/a.whl", "https://user:secret@files.pythonhosted.org/packages/a.whl", + "https://files.pythonhosted.org/packages/a.whl?token=secret", "https://files.pythonhosted.org:443/packages/a.whl", + "https://files.pythonhosted.org/packages/a.tar.gz"]: + with self.subTest(origin=origin), self.assertRaises(ValueError): + inputs.canonical_origin(origin) + + def test_direct_dependency_refused(self): + a = self.wheel() + with self.assertRaisesRegex(ValueError, "direct dependency"): + inputs.validate_closure([a], ["dep @ https://attacker.invalid/dep.whl"], self.root) + + def test_extra_and_marker_dependencies_are_required(self): + a = self.wheel("parent", requires=['child>=1; extra == "optional"']) + with self.assertRaisesRegex(ValueError, "incomplete"): + inputs.validate_closure([a], ["parent[optional]"], self.root) + inputs.validate_closure([a], ["parent"], self.root) + + def test_unrequested_and_duplicate_artifacts_refused(self): + a, b = self.wheel("first"), self.wheel("second") + with self.assertRaisesRegex(ValueError, "unrequested"): + inputs.validate_closure([a, b], ["first"], self.root) + with self.assertRaisesRegex(ValueError, "duplicates"): + inputs.validate_closure([a, a], ["first"], self.root) + + def test_all_original_profile_requirements_derive_from_metadata(self): + sdk = {"version": "0.1.17", "dependencies": ["httpx>=0.27"], + "optional-dependencies": {"grpc": ["grpcio>=1"], "geopandas": ["geopandas>=1"], "raster": ["rasterio>=1"]}} + source = {"packages": {"honua-sdk": {"metadata": sdk}, "honua-admin": {"metadata": {"dependencies": ["honua-sdk>=0.1,<0.2"]}}}} + self.assertEqual(set(inputs.PROFILES), {"core", "grpc", "geopandas", "raster", "combined-sdk-admin"}) + self.assertEqual(inputs.requirements(source, "combined-sdk-admin"), ["geopandas>=1", "grpcio>=1", "httpx>=0.27", "rasterio>=1"]) + for name in ["grpc", "geopandas", "raster"]: + self.assertIn(sdk["optional-dependencies"][name][0], inputs.requirements(source, name)) + broken = deepcopy(source) + broken["packages"]["honua-admin"]["metadata"]["dependencies"] = ["honua-sdk>=9"] + with self.assertRaisesRegex(ValueError, "excludes"): + inputs.requirements(broken, "combined-sdk-admin") + + def test_runtime_refusal_precedes_resolver_or_install(self): + with patch.object(inputs, "source", return_value={}), patch.object(inputs, "resolve") as resolver: + with self.assertRaisesRegex(ValueError, "runtime"): + inputs.generate(self.root, "a" * 40, self.root / "output", {}) + resolver.assert_not_called() + self.assertFalse((self.root / "output").exists()) + + def test_wrong_source_refusal_precedes_resolver_or_install(self): + with patch.object(inputs, "resolve") as resolver: + with self.assertRaisesRegex(ValueError, "immutable revision"): + inputs.generate(self.root, "trunk", self.root / "output", {}) + resolver.assert_not_called() + + def test_changed_source_refuses_before_backend_install(self): + with patch.object(inputs, "source", side_effect=[{"sourceSha": "a"}, {"sourceSha": "b"}]), \ + patch.object(inputs, "runtime", return_value={}), patch.object(inputs, "resolve", return_value={}), \ + patch.object(inputs, "requirements", return_value=[]), patch.object(inputs, "build_source_wheels") as build, \ + patch.object(inputs, "implementation", return_value={}): + with self.assertRaisesRegex(ValueError, "before backend"): + inputs.generate(self.root / "source", "a", self.root / "output", {}) + build.assert_not_called() + + def complete_proposal(self): + # Complete synthetic original-byte oracle vectors, never a live resolver result. + revision = "a" * 40 + selected = {"sourceSha": revision, "packages": { + "honua-sdk": {"metadata": {"version": "0.1.0", "dependencies": ["dep>=1"], + "optional-dependencies": {"grpc": ["dep>=1"], "geopandas": ["dep>=1"], "raster": ["dep>=1"]}}}, + "honua-admin": {"metadata": {"version": "0.1.0", "dependencies": ["honua-sdk>=0.1,<0.2"]}}}} + proposal = {"schema": inputs.SCHEMA, "sourceAdmitted": False, "candidateQualified": False, "publishedWheelIdentity": False, + "implementation": {"sourceSha": "b" * 40, "synthetic": True}, "source": selected, "runtime": {"synthetic": True}, "profiles": {}} + artifact = self.wheel() + backend = self.wheel("hatchling", "1.30.1") + for name in ["build", *inputs.PROFILES]: + directory = self.root / name + directory.mkdir() + item = backend if name == "build" else artifact + (directory / item["filename"]).write_bytes((self.root / item["filename"]).read_bytes()) + requested = ["hatchling==1.30.1"] if name == "build" else inputs.requirements(selected, name) + scratch = Path(tempfile.gettempdir()).resolve() / "honua-development-resolve-abcdefgh" + raw = {"version": "1", "pip_version": "26.2.1", "environment": inputs.default_environment(), + "install": [{"metadata": {"name": item["name"], "version": item["version"]}, + "download_info": {"url": item["origin"], "archive_info": {"hashes": {"sha256": item["sha256"]}}}}]} + report = json.dumps(raw).encode() + (directory / "pip-report.json").write_bytes(report) + (directory / "resolver.stdout").write_bytes(b"synthetic resolver output") + value = {"requested": requested, "command": inputs.resolver_command(requested, scratch), + "scratchRoot": str(scratch), "environment": inputs.default_environment(), "artifacts": [deepcopy(item)], + "reportSha256": inputs.digest(report), "stdoutSha256": inputs.digest(b"synthetic resolver output")} + if name == "build": + proposal["build"] = value + else: + proposal["profiles"][name] = value + (self.root / "source-wheels").mkdir() + wheels = [] + for name in ["honua_sdk", "honua_admin"]: + item = self.wheel(name, "0.1.0") + (self.root / "source-wheels" / item["filename"]).write_bytes((self.root / item["filename"]).read_bytes()) + item.update(name=name.replace("_", "-"), sourceSha=revision, publishedIdentity=False) + wheels.append(item) + scratch = Path(tempfile.gettempdir()).resolve() / "honua-development-build-abcdefgh" + install, build = inputs.build_commands(scratch, self.root) + (self.root / "source-build.stdout").write_bytes(b"synthetic build output") + proposal["sourceBuild"] = {"scratchRoot": str(scratch), "backendInstallCommand": install, "buildCommand": build, + "installedBackend": {"hatchling": "1.30.1"}, + "backendLock": f'hatchling==1.30.1 --hash=sha256:{backend["sha256"]}\n', + "stdoutSha256": inputs.digest(b"synthetic build output"), "wheels": wheels} + return proposal + + def test_complete_artifact_stage_positive_and_every_original_binding_mutation(self): + original = self.complete_proposal() + def verify(value): + with patch.object(inputs, "source", return_value=original["source"]), \ + patch.object(inputs, "runtime", return_value=original["runtime"]), \ + patch.object(inputs, "implementation", return_value=original["implementation"]): + return inputs.verify_artifact_inputs(value, self.root, "a" * 40, self.root, {"pip": "26.2.1"}) + self.assertEqual(len(verify(original)["profiles"]), 5) + for mutation in ["install-command", "build-command", "scratch", "source", "runtime", "profile", "origin", "digest", + "report", "source-wheel", "backend-lock", "backend-inventory", "published-identity", "top-level-published-identity", "admission"]: + value = deepcopy(original) + if mutation == "install-command": + value["sourceBuild"]["backendInstallCommand"].remove("--require-hashes") + elif mutation == "build-command": + value["sourceBuild"]["buildCommand"].remove("--no-deps") + elif mutation == "scratch": + value["sourceBuild"]["scratchRoot"] = str(Path(tempfile.gettempdir()) / "other") + elif mutation == "source": + value["source"]["sourceSha"] = "f" * 40 + elif mutation == "runtime": + value["runtime"] = {"synthetic": False} + elif mutation == "profile": + del value["profiles"]["raster"] + elif mutation == "origin": + value["profiles"]["core"]["artifacts"][0]["origin"] = "https://attacker.invalid/dep.whl" + elif mutation == "digest": + value["profiles"]["core"]["artifacts"][0]["sha256"] = "0" * 64 + elif mutation == "report": + value["profiles"]["core"]["reportSha256"] = "0" * 64 + elif mutation == "source-wheel": + value["sourceBuild"]["wheels"][0]["sha256"] = "0" * 64 + elif mutation == "backend-lock": + value["sourceBuild"]["backendLock"] = "hatchling>=1\n" + elif mutation == "backend-inventory": + value["sourceBuild"]["installedBackend"]["hatchling"] = "9" + elif mutation == "published-identity": + value["sourceBuild"]["wheels"][0]["publishedIdentity"] = True + elif mutation == "top-level-published-identity": + value["publishedWheelIdentity"] = True + elif mutation == "admission": + value["sourceAdmitted"] = True + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + verify(value) + + + def test_rehashed_original_report_substitution_refuses(self): + proposal = self.complete_proposal() + report = self.root / "core/pip-report.json" + raw = json.loads(report.read_bytes()) + raw["install"][0]["metadata"]["version"] = "9.0" + report.write_text(json.dumps(raw)) + proposal["profiles"]["core"]["reportSha256"] = inputs.digest(report.read_bytes()) + with patch.object(inputs, "source", return_value=proposal["source"]), \ + patch.object(inputs, "runtime", return_value=proposal["runtime"]), \ + patch.object(inputs, "implementation", return_value=proposal["implementation"]), \ + self.assertRaisesRegex(ValueError, "artifact/output join"): + inputs.verify_artifact_inputs(proposal, self.root, "a" * 40, self.root, {"pip": "26.2.1"}) + + def test_failed_private_origin_never_persists_raw_output(self): + private_reports = [] + def resolver(command): + path = Path(command[command.index("--report") + 1]) + private_reports.append(path) + path.write_text(json.dumps({"version": "1", "pip_version": "26.2.1", "install": [ + {"download_info": {"url": "https://user:secret@files.pythonhosted.org/packages/a.whl"}}]})) + return b"secret private resolver output" + with patch.object(inputs, "run", side_effect=resolver), patch.object(inputs, "urlopen") as network: + with self.assertRaisesRegex(ValueError, "origin"): + inputs.resolve(["dep"], self.root / "refused") + network.assert_not_called() + self.assertFalse(list((self.root / "refused").iterdir())) + self.assertFalse(private_reports[0].exists()) + + def test_real_pip_ignores_ambient_config_and_proxy_inputs(self): + malicious = self.root / "pip.conf" + malicious.write_text("[global]\nextra-index-url=https://user:secret@attacker.invalid/simple\n") + with patch.dict(os.environ, {"PIP_CONFIG_FILE": str(malicious), "PIP_INDEX_URL": "https://user:secret@attacker.invalid", + "HTTPS_PROXY": "https://secret@attacker.invalid", "PYTHONPATH": "untrusted"}): + observed = inputs.run([inputs.sys.executable, "-m", "pip", "--isolated", "config", "list"]).decode() + self.assertNotIn("secret", observed) + self.assertNotIn("attacker", observed) + self.assertNotIn("extra-index", observed) + + + def test_producer_workflow_requires_string_run_values_and_block_scalars(self): + path = Path(__file__).resolve().parents[1] / ".github/workflows/development-source-inputs.yml" + text = path.read_text() + # Step run values use eight spaces; defaults.run is a mapping and + # must not be counted as an executable step or a fixed three-step tree. + values = [line.split("run:", 1)[1].strip() for line in text.splitlines() if line.startswith(" run:")] + self.assertTrue(values) + self.assertTrue(all(value == "|" for value in values), values) + self.assertIn("yaml.safe_load(path.read_text())", text) + self.assertIn("assert isinstance(step['run'], str)", text) + self.assertIn("--require-hashes --only-binary=:all:", text) + + + def test_actual_clean_git_source_positive_and_dirty_wrong_source_negative(self): + subprocess.run(["git", "init", "-q", str(self.root)], check=True) # noqa: S603, S607 + def git(*args): + return subprocess.check_output(["git", "-C", str(self.root), *args]).decode().strip() # noqa: S603, S607 + git("config", "core.autocrlf", "false") + git("config", "user.name", "Fixture") + git("config", "user.email", "fixture@example.invalid") + git("remote", "add", "origin", inputs.REPOSITORY) + for name in ["honua-sdk", "honua-admin"]: + path = self.root / "packages" / name / "pyproject.toml" + path.parent.mkdir(parents=True) + path.write_text(f'[build-system]\nrequires=["hatchling==1.30.1"]\nbuild-backend="hatchling.build"\n[project]\nname="{name}"\nversion="0.1.0"\ndependencies=[]\n') + git("add", ".") + git("commit", "-qm", "fixture") + revision = git("rev-parse", "HEAD") + observed = inputs.source(self.root, revision) + self.assertEqual(len(observed["files"]), 2) + with self.assertRaisesRegex(ValueError, "wrong immutable"): + inputs.source(self.root, "f" * 40) + path.write_text(path.read_text() + "# changed input\n") + with self.assertRaisesRegex(ValueError, "dirty"): + inputs.source(self.root, revision) + + def test_actual_reviewed_git_symlink_and_independent_refusals(self): + self.test_actual_clean_git_source_positive_and_dirty_wrong_source_negative() + def git(*args): + return subprocess.check_output(["git", "-C", str(self.root), *args]).decode().strip() # noqa: S603, S607 + git("restore", "packages/honua-admin/pyproject.toml") + (self.root / "AGENTS.md").write_text("Immutable source instructions\n") + alias = self.root / "CLAUDE.md" + if os.name == "nt": + # Windows checkouts without link privileges must refuse inspectability; + # the reviewed production runtime is Linux and runs the real link positive. + alias.write_text("AGENTS.md") + blob = git("hash-object", "-w", str(alias)) + git("add", "AGENTS.md") + git("update-index", "--add", "--cacheinfo", "120000," + blob + ",CLAUDE.md") + git("commit", "-qm", "uninspectable link fixture") + with self.assertRaises(ValueError): + inputs.source(self.root, git("rev-parse", "HEAD")) + return + alias.symlink_to("AGENTS.md") + git("add", ".") + git("commit", "-qm", "reviewed alias fixture") + observed = inputs.source(self.root, git("rev-parse", "HEAD")) + self.assertEqual(observed["files"]["CLAUDE.md"]["linkTarget"], "AGENTS.md") + self.assertEqual(observed["files"]["CLAUDE.md"]["targetSha256"], observed["files"]["AGENTS.md"]["sha256"]) + for case in ["changed", "foreign", "escaping", "package", "changed-target"]: + # Every negative starts from a separately revalidated clean positive. + inputs.source(self.root, git("rev-parse", "HEAD")) + extra = None + if case in ["changed", "escaping"]: + alias.unlink() + alias.symlink_to("./AGENTS.md" if case == "changed" else "../AGENTS.md") + elif case == "foreign": + extra = self.root / "foreign.md" + extra.symlink_to("AGENTS.md") + elif case == "package": + extra = self.root / "packages/honua-sdk/foreign.py" + extra.symlink_to("../../AGENTS.md") + else: + (self.root / "AGENTS.md").write_text("changed immutable target\n") + if case != "changed-target": + git("add", ".") + git("commit", "-qm", "negative link fixture") + with self.subTest(case=case), self.assertRaises(ValueError): + inputs.source(self.root, git("rev-parse", "HEAD")) + if extra: + extra.unlink() + if alias.is_symlink(): + alias.unlink() + alias.symlink_to("AGENTS.md") + (self.root / "AGENTS.md").write_text("Immutable source instructions\n") + git("add", ".") + if git("status", "--porcelain"): + git("commit", "-qm", "restored clean link fixture") + + + def test_finite_editable_source_census_refuses_extra_module_and_symlink(self): + checkout = self.root / "copy" + expected = {} + for name in ("honua-sdk", "honua-admin"): + target = checkout / "packages" / name / "module.py" + target.parent.mkdir(parents=True) + target.write_bytes(b"VALUE = 1\n") + expected[target.relative_to(checkout).as_posix()] = {"sha256": inputs.digest(target.read_bytes())} + source = {"files": expected} + self.assertEqual(documentation.source_hashes(inputs, source, checkout), {k: v["sha256"] for k, v in expected.items()}) + extra = checkout / "packages/honua-sdk/extra.py" + extra.write_bytes(b"MUTATION = True\n") + with self.assertRaisesRegex(ValueError, "finite tracked"): + documentation.source_hashes(inputs, source, checkout) + extra.unlink() + extra.symlink_to("module.py") + with self.assertRaisesRegex(ValueError, "symlink"): + documentation.source_hashes(inputs, source, checkout) + extra.unlink() + (extra.parent / "empty-extra").mkdir() + with self.assertRaisesRegex(ValueError, "finite tracked"): + documentation.source_hashes(inputs, source, checkout) + + def test_original_document_plan_matches_independent_governed_block_inventory(self): + fixture = json.loads((Path(__file__).parent / "fixtures/python_source_install_blocks.json").read_text()) + source = {"files": {name: {"gitBlob": item["gitBlob"]} for name, item in fixture["documents"].items()}} + for name, item in fixture["documents"].items(): + (self.root / name).write_text(item["text"]) + actual = documentation.plan(inputs, self.root, source) + self.assertEqual(len(actual), 3) + self.assertEqual(sum(len(item["invocations"]) for item in actual), 12) + for observed, expected in zip(actual, fixture["expectedBlocks"], strict=True): + self.assertEqual((observed["document"], observed["blockIndex"], observed["blockSha256"]), + (expected["document"], expected["index"], expected["sha256"])) + self.assertEqual([item["command"] for item in observed["invocations"]], expected["commands"]) + self.assertEqual([item["ordinal"] for item in observed["invocations"]], list(range(len(expected["commands"])))) + self.assertEqual(actual[1]["invocations"][0]["editable"], [True]) + self.assertEqual(actual[1]["invocations"][2]["requested"], ["pytest", "pytest-cov", "ruff", "mypy"]) + original = fixture["documents"]["README.md"]["text"] + for replacement in ("pip install --extra-index-url https://foreign.invalid packages/honua-sdk", + "pip install -e /foreign/source", "pip install packages/honua-sdk --no-compile"): + (self.root / "README.md").write_text(original.replace("pip install ./packages/honua-sdk", replacement, 1)) + with self.subTest(replacement=replacement), self.assertRaises(ValueError): + documentation.plan(inputs, self.root, source) + + def test_unknown_pip_transform_refuses_before_wheel_loading(self): + observed = {"files": {"module.py": "0" * 64}, "sizes": {"module.py": 1}, "generated": {}, + "sitePackages": str(self.root / "site"), "python": str(self.root / "bin/python")} + with patch.object(documentation.importlib.metadata, "version", return_value="99.0"), \ + patch.object(documentation.zipfile, "ZipFile") as archive: + with self.assertRaisesRegex(ValueError, "unreviewed pip installed-file"): + documentation.installed_wheel_bytes(inputs, observed, self.root / "unopened.whl") + archive.assert_not_called() + + def test_actual_default_pip_generated_file_positive_and_extra_cache_script_mutants(self): + if os.name == "nt": + # This production input policy is Linux-only; Windows must refuse it + # before attempting the POSIX installed-file transformation model. + policy = json.loads((Path(inputs.__file__).parent / "development-inputs/runtime.json").read_text()) + with self.assertRaisesRegex(ValueError, "runtime differs"): + inputs.runtime(policy) + return + directory = self.root / "wheelhouse" + directory.mkdir() + metadata_dir = "finite_probe-1.0.dist-info" + members = {"finite_probe.py": b"def main():\n return 0\n", + metadata_dir + "/METADATA": b"Metadata-Version: 2.3\nName: finite-probe\nVersion: 1.0\n", + metadata_dir + "/WHEEL": b"Wheel-Version: 1.0\nGenerator: owned-control\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + metadata_dir + "/entry_points.txt": b"[console_scripts]\nfinite-probe = finite_probe:main\n"} + record = io.StringIO(newline="") + writer = csv.writer(record) + for name, content in members.items(): + digest = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b"=").decode() + writer.writerow([name, "sha256=" + digest, len(content)]) + writer.writerow([metadata_dir + "/RECORD", "", ""]) + members[metadata_dir + "/RECORD"] = record.getvalue().encode() + archive = directory / "finite_probe-1.0-py3-none-any.whl" + with zipfile.ZipFile(archive, "w") as wheel: + for name, content in members.items(): + wheel.writestr(name, content) + environment = self.root / "environment" + inputs.run([inputs.sys.executable, "-m", "venv", "--without-pip", str(environment)]) + python = environment / "bin/python" + inputs.run([inputs.sys.executable, "-m", "pip", "--isolated", "--python", str(python), "install", "--no-index", "--no-deps", str(archive)]) + observed = documentation.inventory(inputs, python)["finite-probe"] + try: + documentation.installed_wheel_bytes(inputs, observed, archive) + except ValueError: + # Retain only this authored synthetic wheel's original bytes on the + # hosted failure path. The real verifier refusal remains unchanged. + filename = str(Path(observed["sitePackages"]) / "finite_probe.py") + cache = str(Path(importlib.util.cache_from_source(filename)).relative_to(observed["sitePackages"])) + raw = base64.b64decode(observed["generated"][cache], validate=True) + actual = marshal.loads(raw[16:]) # noqa: S302 -- only the authored synthetic wheel compiled by owned pip + expected = compile(members["finite_probe.py"], filename, "exec", dont_inherit=True, optimize=0) + fields = ["co_argcount", "co_posonlyargcount", "co_kwonlyargcount", "co_nlocals", "co_stacksize", "co_flags", + "co_code", "co_consts", "co_names", "co_varnames", "co_filename", "co_name", "co_qualname", + "co_firstlineno", "co_linetable", "co_exceptiontable", "co_freevars", "co_cellvars"] + def differences(left, right, prefix=""): + result = [] + for field in fields: + first, second = getattr(left, field), getattr(right, field) + if field == "co_consts": + if len(first) != len(second): + result.append(prefix + field + ".length") + for ordinal, (a, b) in enumerate(zip(first, second, strict=False)): + if isinstance(a, types.CodeType) and isinstance(b, types.CodeType): + result.extend(differences(a, b, prefix + "constant" + str(ordinal) + ".")) + elif type(a) is not type(b) or a != b: + result.append(prefix + field + "." + str(ordinal)) + elif type(first) is not type(second) or first != second: + result.append(prefix + field) + return result + diagnostic = {"actualFilename": actual.co_filename, "expectedFilename": filename, + "codeObjectEqual": actual == expected, "fieldDifferences": differences(actual, expected), + "actualMarshalSha256": inputs.digest(raw[16:]), "expectedMarshalSha256": inputs.digest(marshal.dumps(expected)), + "actualRemarshalEqual": marshal.dumps(actual) == marshal.dumps(expected), "header": raw[:16].hex()} + print("ORIGINAL_PIP_BYTECODE_DIAGNOSTIC " + json.dumps(diagnostic, sort_keys=True), flush=True) + if os.environ.get("RUNNER_TEMP"): + output = Path(os.environ["RUNNER_TEMP"]) / "development-pip-control" + output.mkdir(exist_ok=True) + (output / "original.pyc").write_bytes(raw) + (output / "expected.marshal").write_bytes(marshal.dumps(expected)) + (output / "verified-source.py").write_bytes(members["finite_probe.py"]) + (output / "diagnostic.json").write_text(json.dumps(diagnostic, indent=2) + "\n") + raise + for mutation in ("extra", "cache", "cache-code", "cache-filename", "script", "record", "installer"): + value = deepcopy(observed) + if mutation == "extra": + value["files"]["unmodeled.py"] = inputs.digest(b"EXTRA=True\n") + value["sizes"]["unmodeled.py"] = 11 + else: + key = next(k for k in value["generated"] if + (k.endswith(".pyc") if mutation.startswith("cache") else + k.startswith("../") if mutation == "script" else + k.endswith("/RECORD") if mutation == "record" else k.endswith("/INSTALLER"))) + data = base64.b64decode(value["generated"][key]) + if mutation in ("cache-code", "cache-filename"): + filename = str(Path(observed["sitePackages"]) / "finite_probe.py") + content = members["finite_probe.py"].replace(b"return 0", b"return 1") if mutation == "cache-code" else members["finite_probe.py"] + code = compile(content, filename + (".foreign" if mutation == "cache-filename" else ""), "exec", dont_inherit=True, optimize=0) + data = data[:16] + marshal.dumps(code) + else: + data += b"MUTATION" + value["generated"][key] = base64.b64encode(data).decode() + value["files"][key] = inputs.digest(data) + value["sizes"][key] = len(data) + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + documentation.installed_wheel_bytes(inputs, value, archive) + site = Path(observed["sitePackages"]) + (site / "unrecorded_extra.py").write_bytes(b"EXTRA = True\n") + with self.assertRaises(subprocess.CalledProcessError): + documentation.inventory(inputs, python) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_governed_source_inputs.py b/tests/test_governed_source_inputs.py new file mode 100644 index 0000000..7df2081 --- /dev/null +++ b/tests/test_governed_source_inputs.py @@ -0,0 +1,315 @@ +"""Offline daemon projection/failure controls; actual image execution is separate.""" +from __future__ import annotations + +from copy import deepcopy +from functools import partial +import hashlib +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from scripts import run_governed_source_inputs as runner + + +class GovernedExecutionTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + self.container, self.owner, self.image = "a" * 64, "b" * 32, "sha256:" + "c" * 64 + self.arguments = ["/usr/local/bin/python3.12", "-I", "/implementation/scripts/governed_source_inputs_bootstrap.py"] + self.mounts = [(self.root / "implementation", "/implementation", True), + (self.root / "subject", "/subject", True), + (self.root / "packaging.whl", "/implementation-tools/packaging.whl", True), + (self.root / "output", "/output", False)] + self.tmpfs = "/tmp:rw,exec,mode=0700,uid=1000,gid=1000" # noqa: S108 -- offline private daemon model + self.observed = {"id": self.container, "ownerNonce": self.owner, "user": "1000:1000", "imageId": self.image, + "declaredImage": runner.IMAGE, "readonlyRootfs": True, "path": self.arguments[0], "args": self.arguments[1:], + "tmpfs": {"/tmp": self.tmpfs.split(":", 1)[1]}, # noqa: S108 -- offline daemon model + "mounts": [{"Type": "bind", "Source": str(source), "Destination": target, "RW": not readonly} + for source, target, readonly in self.mounts], + "exitCode": 0, "status": "created", "running": False, + "startedAt": "0001-01-01T00:00:00Z", "finishedAt": "0001-01-01T00:00:00Z"} + self.validate = partial(runner.validate_container, owner=self.owner, image_id=self.image, + arguments=self.arguments, mounts=self.mounts, user="1000:1000", tmpfs=self.tmpfs) + + def terminal(self, code=0): + observed = deepcopy(self.observed) + observed.update(status="exited", running=False, exitCode=code, + startedAt="2026-10-20T00:00:00Z", finishedAt="2026-10-20T00:01:00Z") + return observed + + def live(self): + observed = deepcopy(self.observed) + observed.update(status="running", running=True, startedAt="2026-10-20T00:00:00Z") + return observed + + def test_full_projection_positive_legacy_and_explicit_tmpfs_and_independent_refusals(self): + self.validate(self.observed, self.container) + explicit = deepcopy(self.observed) + explicit["mounts"].append({"Type": "tmpfs", "Source": "", "Destination": "/tmp", "RW": True}) # noqa: S108 -- offline daemon model + self.validate(explicit, self.container) + for mutation in ("owner", "image", "readonly", "command", "user", "source", "extra", "typed-rw", "tmpfs", "duplicate-tmpfs"): + observed = deepcopy(explicit) + if mutation == "owner": + observed["ownerNonce"] = "foreign" + elif mutation == "image": + observed["imageId"] = "sha256:" + "d" * 64 + elif mutation == "readonly": + observed["readonlyRootfs"] = False + elif mutation == "command": + observed["args"] += ["--foreign"] + elif mutation == "user": + observed["user"] = "0:0" + elif mutation == "source": + observed["mounts"][0]["Source"] = str(self.root / "foreign") + elif mutation == "extra": + observed["mounts"].append({"Type": "volume", "Source": "foreign", "Destination": "/foreign", "RW": True}) + elif mutation == "typed-rw": + observed["mounts"][0]["RW"] = 0 + elif mutation == "tmpfs": + observed["tmpfs"] = {} + else: + observed["mounts"].append(deepcopy(observed["mounts"][-1])) + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + self.validate(observed, self.container) + + def test_original_failure_stdout_stderr_and_terminal_state_retained_before_refusal(self): + result = subprocess.CompletedProcess(["docker"], 1, b"original failure stdout", b"original failure stderr") + with patch.object(runner.subprocess, "run", return_value=result), patch.object(runner, "inspect_container", return_value=self.terminal(1)): + with self.assertRaisesRegex(ValueError, "original streams and terminal state retained"): + runner.capture_action(self.container, "generate", self.root, self.validate) + self.assertEqual((self.root / "generate.stdout").read_bytes(), result.stdout) + self.assertEqual((self.root / "generate.stderr").read_bytes(), result.stderr) + self.assertEqual(json.loads((self.root / "generate.container-after.json").read_bytes())["exitCode"], 1) + self.assertEqual(json.loads((self.root / "generate.attach.json").read_bytes())["returnCode"], 1) + + def test_safe_original_state_retained_before_identity_refusal(self): + result = subprocess.CompletedProcess(["docker"], 0, b"original stdout", b"original stderr") + foreign = self.terminal() + foreign["ownerNonce"] = "foreign" + with patch.object(runner.subprocess, "run", return_value=result), patch.object(runner, "inspect_container", return_value=foreign): + with self.assertRaisesRegex(ValueError, "actual container identity"): + runner.capture_action(self.container, "generate", self.root, self.validate) + self.assertEqual((self.root / "generate.stdout").read_bytes(), result.stdout) + self.assertEqual((self.root / "generate.stderr").read_bytes(), result.stderr) + self.assertEqual(json.loads((self.root / "generate.container-after.json").read_bytes())["ownerNonce"], "foreign") + + def test_transport_failure_waits_for_live_work_and_remains_failure_after_terminal(self): + result = subprocess.CompletedProcess(["docker"], 1, b"partial original stdout", b"transport lost") + with patch.object(runner.subprocess, "run", return_value=result) as command, \ + patch.object(runner, "inspect_container", side_effect=[self.live(), self.live(), self.terminal()]) as inspect, \ + patch.object(runner.time, "sleep"): + with self.assertRaisesRegex(ValueError, "original streams and terminal state retained"): + runner.capture_action(self.container, "generate", self.root, self.validate) + self.assertEqual(inspect.call_count, 3) + self.assertEqual(command.call_args.args[0], ["docker", "start", "--attach", self.container]) + self.assertTrue(json.loads((self.root / "generate.pending.json").read_bytes())["container"]["running"]) + self.assertEqual(json.loads((self.root / "generate.container-after.json").read_bytes())["status"], "exited") + self.assertEqual((self.root / "generate.stdout").read_bytes(), result.stdout) + + def test_owned_live_cleanup_waits_then_normal_removes_exact_terminal_identity(self): + def command(arguments): + if arguments[:2] == ["docker", "ps"]: + return (self.container + "\n").encode() + self.assertEqual(arguments, ["docker", "rm", self.container]) + self.assertEqual(inspect.call_count, 2) + return (self.container + "\n").encode() + with patch.object(runner, "run", side_effect=command), \ + patch.object(runner, "inspect_container", side_effect=[self.live(), self.terminal()]) as inspect, \ + patch.object(runner.time, "sleep"): + result = runner.cleanup_owned(self.owner, "generate", self.root, self.validate) + self.assertTrue(result["removed"]) + self.assertFalse(result["sourceAdmitted"]) + self.assertEqual(result["originalState"]["status"], "exited") + + def test_foreign_owner_cleanup_never_mutates(self): + foreign = self.terminal() + foreign["ownerNonce"] = "foreign" + with patch.object(runner, "run", return_value=(self.container + "\n").encode()) as command, \ + patch.object(runner, "inspect_container", return_value=foreign): + with self.assertRaisesRegex(ValueError, "actual container identity"): + runner.cleanup_owned(self.owner, "generate", self.root, self.validate) + self.assertEqual(command.call_count, 1) + self.assertEqual(command.call_args.args[0][:2], ["docker", "ps"]) + self.assertEqual(json.loads((self.root / "generate.cleanup-observed.json").read_bytes())["ownerNonce"], "foreign") + + def test_created_never_started_cleanup_is_normal_and_unknown_state_refuses(self): + def command(arguments): + self.assertNotIn("--force", arguments) + return (self.container + "\n").encode() + with patch.object(runner, "run", side_effect=command), patch.object(runner, "inspect_container", return_value=self.observed): + self.assertTrue(runner.cleanup_owned(self.owner, "generate", self.root, self.validate)["removed"]) + unknown = deepcopy(self.observed) + unknown["status"] = "unknown" + with patch.object(runner, "run", side_effect=command) as invoke, patch.object(runner, "inspect_container", return_value=unknown): + with self.assertRaisesRegex(ValueError, "terminal or never-started"): + runner.cleanup_owned(self.owner, "verify", self.root, self.validate) + self.assertEqual(invoke.call_count, 1) + + def outer_vector(self): + """Complete offline original-byte vector; never a candidate/runtime claim.""" + policy = json.loads((Path(runner.__file__).parent / 'development-inputs/governed-runtime.json').read_bytes()) + binding = policy['executionBinding'] + image = {'id': self.image, 'repoDigests': ['python@'+runner.IMAGE.split('@')[1]], 'declaredImage': runner.IMAGE, + 'os': 'linux', 'architecture': 'amd64'} + context = {'mountSources': {'implementation': str(self.root/'implementation'), 'subject': str(self.root/'subject'), + 'tool': str(self.root/binding['implementationPackagingFilename']), 'output': str(self.root/'output')}, + 'user': '1000:1000', 'tmpfs': self.tmpfs, 'image': image} + declarations = [] + for number, count in enumerate((3, 3, 6)): + declarations.append({'document': 'README.md' if number < 2 else 'INSTALL.md', 'blockIndex': number, + 'blockSha256': str(number)*64, 'invocations': [ + {'ordinal': ordinal, 'command': f'pip install fixture-{number}-{ordinal}', + 'commandSha256': hashlib.sha256(f'pip install fixture-{number}-{ordinal}'.encode()).hexdigest(), + 'requested': [f'fixture-{number}-{ordinal}'], 'editable': [False]} for ordinal in range(count)]}) + runtime = {key: value for key, value in policy.items() if key != 'executionBinding'} + runtime.update(policySha256=hashlib.sha256(json.dumps(policy, sort_keys=True).encode()).hexdigest(), + executableSha256=binding['pythonExecutableSha256'], toolFilesSha256={'pip': binding['pipToolFilesSha256']}, + executionBinding=deepcopy(binding)) + proposal = {'source': {'sourceSha': 'e'*40}, 'implementation': {'sourceSha': 'f'*40}, 'runtime': runtime, + 'sourceAdmitted': False, 'candidateQualified': False, 'publishedWheelIdentity': False, + 'documentationSequences': [{**item, 'steps': deepcopy(item['invocations'])} for item in declarations]} + proposal_bytes = json.dumps(proposal).encode() + receipt = {'schema': 'honua.python-governed-input-execution/v1', 'implementationSha': 'f'*40, 'subjectSourceSha': 'e'*40, + 'image': image, 'implementationPackaging': {'filename': binding['implementationPackagingFilename'], + 'sha256': binding['implementationPackagingSha256'], 'installed': False}, 'commands': [], 'containers': [], + 'innerProposalSha256': hashlib.sha256(proposal_bytes).hexdigest(), + 'sourceAdmitted': False, 'candidateQualified': False, 'publishedWheelIdentity': False} + stdout_values = {'probe': json.dumps(runtime).encode(), 'generate': b'', 'verify': b'', + 'probe-refusals': json.dumps({'actualRefusals': ['wheel-bytes', 'wheel-name', 'image', 'interpreter', 'pip', + 'pip-entrypoint', 'wheel-policy'], 'sourceAdmitted': False, 'candidateQualified': False}).encode(), + 'mutations': json.dumps({'completeOriginalPositive': 'PASS', 'mutationsRejected': ['context-loss', 'step-loss', + 'step-reorder', 'ordinal', 'command', 'block', 'extras-superset', 'editable', 'source-before', 'source-resolved', + 'source-after', 'installed-before', 'installed-after', 'derived-command', 'install-command', 'raw-report', + 'backend-command', 'editable-identity', 'published', 'admitted', 'candidate'], + 'sourceAdmitted': False, 'candidateQualified': False, 'publishedWheelIdentity': False}).encode()} + files = {'execution-context.json': json.dumps(context).encode()} + receipt['executionContextSha256'] = hashlib.sha256(files['execution-context.json']).hexdigest() + for number, action in enumerate(runner.ACTIONS, 1): + arguments, mounts, user, tmpfs = runner.action_inputs(action, 'e'*40, context, policy) + before = deepcopy(self.observed) + before.update(id=str(number)*64, ownerNonce=str(number)*32, path=arguments[0], args=arguments[1:], + mounts=[{'Type': 'bind', 'Source': source, 'Destination': target, 'RW': not readonly} + for source, target, readonly in mounts]) + after = deepcopy(before) + after.update(status='exited', running=False, startedAt='2026-10-20T00:00:00Z', finishedAt='2026-10-20T00:01:00Z') + cleanup = {'ownedContainerId': before['id'], 'ownerNonce': before['ownerNonce'], + 'originalState': after, 'removed': True, 'sourceAdmitted': False} + originals = {'stdout': stdout_values[action], 'stderr': b'', + 'attach.json': json.dumps({'returnCode': 0, 'sourceAdmitted': False}).encode(), + 'container-before.json': json.dumps(before).encode(), 'container-after.json': json.dumps(after).encode(), + 'cleanup-observed.json': json.dumps(after).encode(), 'cleanup.json': json.dumps(cleanup).encode()} + files.update({action+'.'+suffix: value for suffix, value in originals.items()}) + receipt['commands'].append({'action': action, 'command': runner.create_command(before['ownerNonce'], arguments, mounts, user, tmpfs), + 'stdoutSha256': hashlib.sha256(originals['stdout']).hexdigest(), 'stderrSha256': hashlib.sha256(originals['stderr']).hexdigest()}) + receipt['containers'].append({'action': action, 'before': before, 'after': after, 'cleanup': cleanup}) + return receipt, {'implementation_sha': 'f'*40, 'subject_sha': 'e'*40, 'policy': policy, 'proposal_bytes': proposal_bytes, + 'expected_documentation': declarations, 'execution_context': context, 'original_files': files} + + def test_complete_outer_original_bytes_positive_and_finite_mutations(self): + receipt, arguments = self.outer_vector() + result = runner.verify_execution_receipt(receipt, **arguments) + self.assertTrue(result['innerArtifactVerificationRequired']) + self.assertFalse(result['sourceAdmitted']) + # Actual Docker returns the same mount census in different array order + # on repeated inspect. Preserve raw originals and every row field. + reordered, reordered_inputs = deepcopy(receipt), deepcopy(arguments) + for item in reordered['containers']: + item['cleanup']['originalState'] = deepcopy(item['cleanup']['originalState']) + item['cleanup']['originalState']['mounts'].reverse() + self.assertNotEqual(item['after'], item['cleanup']['originalState']) + action = item['action'] + reordered_inputs['original_files'][action+'.cleanup-observed.json'] = json.dumps(item['cleanup']['originalState']).encode() + reordered_inputs['original_files'][action+'.cleanup.json'] = json.dumps(item['cleanup']).encode() + runner.verify_execution_receipt(reordered, **reordered_inputs) + cases = ('missing-file', 'extra-file', 'action-loss', 'action-order', 'foreign-subject', 'foreign-implementation', + 'foreign-image', 'foreign-mount', 'command', 'owner', 'terminal', 'cleanup', 'stream', 'coherent-probe', + 'coherent-refusals', 'coherent-mutants', 'coherent-proposal', 'coherent-step', 'published', 'attach', + 'unsafe-projection', 'typed-attach', 'coherent-cleanup-mount', 'context-bytes', 'coherent-context', 'platform') + for case in cases: + value, inputs = deepcopy(receipt), deepcopy(arguments) + if case == 'missing-file': + del inputs['original_files']['probe.stdout'] + elif case == 'extra-file': + inputs['original_files']['foreign.stdout'] = b'' + elif case == 'action-loss': + value['containers'].pop() + elif case == 'action-order': + value['commands'].reverse() + elif case == 'foreign-subject': + value['subjectSourceSha'] = 'a'*40 + elif case == 'foreign-implementation': + value['implementationSha'] = 'a'*40 + elif case == 'foreign-image': + value['image']['id'] = 'sha256:'+'a'*64 + elif case == 'platform': + value['image']['architecture'] = 'arm64' + elif case == 'foreign-mount': + value['containers'][0]['before']['mounts'][0]['Source'] += '-foreign' + elif case == 'command': + value['commands'][0]['command'].append('--foreign') + elif case == 'owner': + value['containers'][0]['before']['ownerNonce'] = '0'*32 + elif case == 'terminal': + value['containers'][0]['after']['running'] = True + elif case == 'cleanup': + value['containers'][0]['cleanup']['removed'] = False + elif case == 'stream': + inputs['original_files']['probe.stderr'] = b'foreign-original' + elif case.startswith('coherent-') and case not in ('coherent-proposal', 'coherent-step', 'coherent-cleanup-mount', 'coherent-context'): + action = {'coherent-probe': 'probe', 'coherent-refusals': 'probe-refusals', 'coherent-mutants': 'mutations'}[case] + inputs['original_files'][action+'.stdout'] = b'{}' + next(item for item in value['commands'] if item['action'] == action)['stdoutSha256'] = hashlib.sha256(b'{}').hexdigest() + elif case in ('coherent-proposal', 'coherent-step'): + proposal = json.loads(inputs['proposal_bytes']) + if case == 'coherent-proposal': + proposal['source']['sourceSha'] = 'a'*40 + else: + proposal['documentationSequences'][0]['steps'][0]['requested'] = ['foreign'] + inputs['proposal_bytes'] = json.dumps(proposal).encode() + value['innerProposalSha256'] = hashlib.sha256(inputs['proposal_bytes']).hexdigest() + elif case == 'published': + value['publishedWheelIdentity'] = True + elif case == 'unsafe-projection': + value['containers'][0]['before']['environment'] = [] + elif case == 'typed-attach': + inputs['original_files']['probe.attach.json'] = b'{"returnCode":false,"sourceAdmitted":false}' + elif case == 'coherent-cleanup-mount': + cleanup = value['containers'][0]['cleanup'] + cleanup['originalState'] = deepcopy(cleanup['originalState']) + cleanup['originalState']['mounts'][0]['Source'] += '-foreign' + inputs['original_files']['probe.cleanup-observed.json'] = json.dumps(cleanup['originalState']).encode() + inputs['original_files']['probe.cleanup.json'] = json.dumps(cleanup).encode() + elif case == 'context-bytes': + inputs['original_files']['execution-context.json'] += b'\n' + elif case == 'coherent-context': + foreign_context = deepcopy(inputs['execution_context']) + foreign_context['mountSources']['subject'] += '-foreign' + inputs['original_files']['execution-context.json'] = json.dumps(foreign_context).encode() + value['executionContextSha256'] = hashlib.sha256(inputs['original_files']['execution-context.json']).hexdigest() + else: + inputs['original_files']['probe.attach.json'] = b'{"returnCode":1,"sourceAdmitted":false}' + with self.subTest(case=case), self.assertRaises(ValueError): + runner.verify_execution_receipt(value, **inputs) + runner.verify_execution_receipt(receipt, **arguments) + + def test_retained_actual_docker_mount_order_census_and_changed_row_refusal(self): + vector = json.loads((Path(__file__).parent / 'fixtures/governed_mount_order.json').read_bytes()) + after, cleanup = vector['after'], vector['cleanupObserved'] + self.assertNotEqual(after, cleanup) + for observed in (after, cleanup): + runner.validate_container(observed, after['id'], **vector['expected']) + self.assertEqual(runner.canonical_container_state(after), runner.canonical_container_state(cleanup)) + foreign = deepcopy(cleanup) + foreign['mounts'][0]['Source'] += '-foreign' + self.assertNotEqual(runner.canonical_container_state(after), runner.canonical_container_state(foreign)) + with self.assertRaisesRegex(ValueError, 'readonly bind mount census'): + runner.validate_container(foreign, after['id'], **vector['expected']) + + +if __name__ == "__main__": + unittest.main()