diff --git a/.github/workflows/a2a-tck.yml b/.github/workflows/a2a-tck.yml new file mode 100644 index 00000000..02a0cadd --- /dev/null +++ b/.github/workflows/a2a-tck.yml @@ -0,0 +1,49 @@ +name: Pinned A2A ordinary-path TCK +on: + workflow_dispatch: + pull_request: + paths: + - 'experiments/a2a-tck/**' + - 'examples/a2a-sdk/**' + - 'src/**' + - 'requirements/**' + - 'pyproject.toml' + - '.github/workflows/a2a-tck.yml' +permissions: + contents: read +jobs: + differential: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Prepare isolated pinned environments + run: | + python -m pip install uv==0.12.23 + uv venv --python 3.12 ../ca2a-tck-sut + uv pip install --python ../ca2a-tck-sut/bin/python -r experiments/a2a-tck/environment.txt + ../ca2a-tck-sut/bin/python -m pip check + ../ca2a-tck-sut/bin/python -m pip_audit + git clone https://github.com/a2aproject/a2a-tck.git ../a2a-tck + git -C ../a2a-tck checkout 5daf0144b4a4adbdefb62180739b13c5bd7d8196 + cd ../a2a-tck + UV_PROJECT_ENVIRONMENT="$RUNNER_TEMP/tck-env" uv sync --frozen + - name: Run paired TCK and protected cA2A tests + env: + PYTHONPATH: src + UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/tck-env + run: | + ../ca2a-tck-sut/bin/python experiments/a2a-tck/run.py --tck ../a2a-tck --output tck-baseline-results --mode baseline + ../ca2a-tck-sut/bin/python experiments/a2a-tck/run.py --tck ../a2a-tck --output tck-extension-results --mode extension + python experiments/a2a-tck/compare.py tck-baseline-results tck-extension-results + ../ca2a-tck-sut/bin/python -m pytest experiments/a2a-tck/test_compare.py tests/unit/test_a2a_sdk_bridge.py tests/unit/test_a2a_sdk_loopback.py + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.0 + if: always() + with: + name: a2a-tck-evidence + path: | + tck-baseline-results + tck-extension-results diff --git a/CHANGELOG.md b/CHANGELOG.md index caaf5e60..e706403d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Raise the A2A SDK and development extras' `pyasn1` security floor to 0.6.5, + alongside the development/CI hash lock, clearing four dependency-audit + findings for 0.6.4. + - Sealing to a low-order X25519 peer key raises `SealedChannelError`, consistent with other invalid channel keys, instead of leaking a backend `ValueError`. These keys were already refused; no ciphertext format or acceptance rule changes. diff --git a/experiments/a2a-tck/README.md b/experiments/a2a-tck/README.md new file mode 100644 index 00000000..64faa046 --- /dev/null +++ b/experiments/a2a-tck/README.md @@ -0,0 +1,68 @@ +# Pinned A2A ordinary-path compatibility experiment + +The runner serves the existing cA2A SDK loopback adapter with the official TCK's +scenario executor for ordinary messages. Protected messages retain the existing +cA2A executor. Both modes are cA2A-aware, and both enable streaming for the pinned scenario executor. +The differential between the two modes is only the extension advertisement. This isolates advertisement compatibility; it is not a comparison +against a separate, unmodified SDK server or a protected-path TCK certification. + +Test dependencies and upstream scenario code are isolated from runtime packages. +Linux and Python 3.12 are required. From the repository root: + +```sh +git clone https://github.com/a2aproject/a2a-tck.git ../a2a-tck +git -C ../a2a-tck checkout 5daf0144b4a4adbdefb62180739b13c5bd7d8196 +uv venv --python 3.12 ../ca2a-tck-sut +uv pip install --python ../ca2a-tck-sut/bin/python -r experiments/a2a-tck/environment.txt +export UV_PROJECT_ENVIRONMENT="$(cd .. && pwd)/tck-clean-env" +(cd ../a2a-tck && uv sync --frozen) +PYTHONPATH=src ../ca2a-tck-sut/bin/python experiments/a2a-tck/run.py --tck ../a2a-tck --output ../tck-baseline-results --mode baseline +PYTHONPATH=src ../ca2a-tck-sut/bin/python experiments/a2a-tck/run.py --tck ../a2a-tck --output ../tck-extension-results --mode extension +python experiments/a2a-tck/compare.py ../tck-baseline-results ../tck-extension-results +PYTHONPATH=src ../ca2a-tck-sut/bin/python -m pytest tests/unit/test_a2a_sdk_bridge.py tests/unit/test_a2a_sdk_loopback.py +``` + +Output directories must be new. Each run retains logs, JUnit, HTML, compatibility +report, dependency/source identifiers, timeout status and protected-handler count. +The comparison rejects unsuccessful runs, changed test identities/outcomes and +mismatched provenance, modified tracked cA2A source, and any protected-handler +count other than integer zero on either side. The counter measures successful +protected `_lookup()` invocations after authentication and payload checks; it +does not count early protected-path refusals or prove that path was never entered. +Skips remain skips. The TCK checkout must have no tracked or untracked changes. +Inherited proxies and `PYTEST_*` settings are cleared for the local TCK child. Timeout cleanup terminates its process group. + +## Observed result + +On cA2A `806d1147c0bc834ee8aabb698b341c926cd8531e`, A2A SDK 1.2.2, +Python 3.12.14 and the pinned TCK above: + +| Check | Result | +| --- | --- | +| Baseline JSON-RPC MUST | 79 passed, 156 skipped, 30 deselected | +| Extension-advertised JSON-RPC MUST | 79 passed, 156 skipped, 30 deselected | +| Per-test identity/outcome comparison | identical | +| Protected handler calls in either TCK run | 0 | +| Separate cA2A bridge + signed loopback suite | 55 passed | +| Dependency consistency | `uv pip check` passed | + +Initial runs using the immediate-response inventory demo instead of the upstream +scenario executor had five task/artifact/text failures in both modes. They were +fixture coverage gaps; this change makes no production compatibility fix. + +This does not cover HTTP+JSON, gRPC, SHOULD/MAY, real hardware attestation, +independent Go/JavaScript SDKs or cross-language cryptographic interoperability. +The existing signed loopback tests are separate evidence, not TCK coverage. + +The table above is historical evidence at +`806d1147c0bc834ee8aabb698b341c926cd8531e`, not a result for the current +environment. The experimental SUT now uses `pyasn1==0.6.5`; #303 did not cover +this installation path. The [dependency triage](dependency-triage.md) records +four further security pin updates. Other pins, including `cedarpy` and action +SHAs, are unchanged. These pins are not claims to track the latest versions. + +The workflow checks installed dependency consistency and runs `pip-audit` on +the SUT environment before executing the experiment. An audit failure stops +the job; no advisory is ignored. This check does not audit the separate upstream +TCK environment or establish absence of unknown vulnerabilities. The SUT file +contains version pins, not a hash lock. diff --git a/experiments/a2a-tck/compare.py b/experiments/a2a-tck/compare.py new file mode 100644 index 00000000..f3925f1d --- /dev/null +++ b/experiments/a2a-tck/compare.py @@ -0,0 +1,52 @@ +"""Fail on missing, failed, or changed per-test outcomes in paired TCK runs.""" +import argparse +import json +import sys +import xml.etree.ElementTree as ET +from collections import Counter +from pathlib import Path + + +def outcomes(directory): + manifest = json.loads((directory / 'manifest.json').read_text()) + if manifest['exit_code'] != 0 or manifest['timed_out']: + raise ValueError('TCK run did not complete successfully') + rows = {} + for case in ET.parse(directory / 'junit.xml').iter('testcase'): + key = case.attrib['classname'] + '::' + case.attrib['name'] + if key in rows: + raise ValueError('duplicate test identity: ' + key) + rows[key] = next((tag for tag in ('failure', 'error', 'skipped') if case.find(tag) is not None), 'passed') + if not rows or 'passed' not in rows.values(): + raise ValueError('empty test coverage') + if any(status in {'failure', 'error'} for status in rows.values()): + raise ValueError('failed tests') + return manifest, rows + + +def compare(baseline, extension): + left, a = outcomes(baseline) + right, b = outcomes(extension) + if left['mode'] != 'baseline' or right['mode'] != 'extension': + raise ValueError('expected baseline then extension') + for key in ('ca2a_commit', 'tck_commit', 'tck_lock_sha256', 'runner_sha256', 'sdk'): + if left[key] != right[key]: + raise ValueError('mismatched provenance: ' + key) + for manifest in (left, right): + count = manifest.get('protected_handler_invocations') + if type(count) is not int or count != 0: + raise ValueError('protected-handler count must be integer zero') + if manifest.get('ca2a_tracked_changes') != '': + raise ValueError('cA2A tracked files must be unchanged') + if a != b: + raise ValueError('per-test outcomes differ') + return {'identical_test_outcomes': True, 'counts': dict(Counter(a.values())), + 'protected_handler_invocations': [left['protected_handler_invocations'], right['protected_handler_invocations']]} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('baseline', type=Path) + parser.add_argument('extension', type=Path) + args = parser.parse_args() + sys.stdout.write(json.dumps(compare(args.baseline, args.extension), indent=2) + "\n") diff --git a/experiments/a2a-tck/dependency-triage.md b/experiments/a2a-tck/dependency-triage.md new file mode 100644 index 00000000..3b93a2d3 --- /dev/null +++ b/experiments/a2a-tck/dependency-triage.md @@ -0,0 +1,82 @@ +# Experimental SUT dependency triage — 3 November 2026 + +An installed-environment audit with pip-audit 2.10.1 on Linux/Python 3.12.14 +reproduced 15 advisory findings in four packages from `environment.txt` at +`3fbd9a3f2a7b3a65a1f2c59651df202614f8d0f1`. No pyasn1 finding remained. +This is dependency exposure triage, not a demonstrated cA2A exploit. + +| Package | Previous pin | Replacement | Findings | Relevant boundary | +| --- | --- | --- | --- | --- | +| filelock | 3.32.5 | 4.2.1 | 12 | Local filesystem/lock-directory attacks across several lock implementations; some are Windows-specific. CacheControl's file-cache extra declares this dependency. Shared writable directories are not treated as safe. | +| msgpack | 1.2.2 | 1.2.3 | 1 | Repeated rejected map keys can leak memory in the C decoder. CacheControl declares msgpack. No attacker-controlled MessagePack path was demonstrated in this experiment. | +| opentelemetry-api | 1.44.0 | 1.45.1 | 1 | Excessive CPU in malformed inbound tracestate parsing. google-api-core declares it; A2A also exposes optional telemetry dependencies. No remote exploit test was performed. | +| setuptools | 84.0.0 | 84.1.0 | 1 | Windows legacy wheel installation can escape the destination through namespace metadata. This Linux experiment does not demonstrate that path; grpcio-tools declares setuptools. | + +The installed metadata dependency edges above explain why simply deleting these +pins is not a complete repair. All four pins move to the versions identified as +fixed by the audit. filelock crosses a major-version boundary, so installation +consistency and experiment validation are required; this is not a claim of +universal compatibility. Runtime dependency declarations and upstream TCK's +independent lock are unchanged. + +## Advisory inventory + +Filelock findings (all addressed by 4.2.1; GHSA-j8f7-rjxc-mr56 also lists 4.0.9): + +- [GHSA-v4x8-3wr5-3wq8](https://github.com/tox-dev/filelock/security/advisories/GHSA-v4x8-3wr5-3wq8) +- [GHSA-9vhv-jxww-xfm5](https://github.com/tox-dev/filelock/security/advisories/GHSA-9vhv-jxww-xfm5) +- [GHSA-567q-5qvw-pf92](https://github.com/tox-dev/filelock/security/advisories/GHSA-567q-5qvw-pf92) +- [GHSA-m462-7rxw-wvpg](https://github.com/tox-dev/filelock/security/advisories/GHSA-m462-7rxw-wvpg) +- [GHSA-j8f7-rjxc-mr56](https://github.com/tox-dev/filelock/security/advisories/GHSA-j8f7-rjxc-mr56) +- [GHSA-8mx5-7m5m-r77g](https://github.com/tox-dev/filelock/security/advisories/GHSA-8mx5-7m5m-r77g) +- [GHSA-69hj-f68j-2879](https://github.com/tox-dev/filelock/security/advisories/GHSA-69hj-f68j-2879) +- [GHSA-9c4x-gm8v-wgc4](https://github.com/tox-dev/filelock/security/advisories/GHSA-9c4x-gm8v-wgc4) +- [GHSA-rfpc-qcw5-3pc4](https://github.com/tox-dev/filelock/security/advisories/GHSA-rfpc-qcw5-3pc4) +- [GHSA-cwqr-hh33-gh44](https://github.com/tox-dev/filelock/security/advisories/GHSA-cwqr-hh33-gh44) +- [GHSA-fq56-3cx3-fjxp](https://github.com/tox-dev/filelock/security/advisories/GHSA-fq56-3cx3-fjxp) +- [GHSA-6627-m3xw-93mp](https://github.com/tox-dev/filelock/security/advisories/GHSA-6627-m3xw-93mp) + +Other findings: + +- msgpack: [GHSA-j586-36cw-2gc2](https://github.com/msgpack/msgpack-python/security/advisories/GHSA-j586-36cw-2gc2) +- opentelemetry-api: [GHSA-xcq7-p3pf-p88h](https://github.com/open-telemetry/opentelemetry-python/security/advisories/GHSA-xcq7-p3pf-p88h) +- setuptools: [GHSA-xmj3-9gf7-h52w](https://github.com/pypa/setuptools/security/advisories/GHSA-xmj3-9gf7-h52w) + +## Reproduction and limits + +Install the current `environment.txt` in a new Python 3.12 virtual environment, +then run that environment's `python -m pip check` and `python -m pip_audit`. +The TCK workflow now runs both before the paired experiment. Neither check +disables findings or exempts these dependencies. Audit results are dated +observations of the advisory database, not a permanent clean bill of health. + +The README's original TCK table remains historical. Current-environment runs +must retain their own source revision and dependency provenance. This change +does not establish protected-path certification, Windows execution, a full +hash lock, or an audit of the separately installed upstream TCK environment. + +## Validation on the updated environment + +At local source `29d0d5087ed46ecd17dafd5a945bd76a7386b7f1`, a newly created SUT +virtual environment installed all 97 pinned packages. `pip check` passed and +`pip-audit` reported zero known findings. The 15 comparator and 55 bridge/loopback +tests passed. Both fresh official TCK JSON-RPC MUST runs produced 79 passed, +156 skipped and 30 deselected. The comparator accepted identical per-test +outcomes and integer-zero protected-handler counts on both sides. Neither run +timed out; both recorded clean tracked cA2A source. + +The published equivalent source commit is +`98f4fd69d789509ea81be8aa2a43870d586d91cb`; its tree +`71ae1b78eacbb3f74b0e680ed1948ab876c136b5` exactly matches the locally +tested tree. The retained manifests identify the local commit. In a separate +hash-locked development environment, 796 unit tests passed and two skipped; +Ruff, mypy (47 files) and configured Bandit passed. + +The SDK emitted context-detach and pending-task cleanup diagnostics during the +runs. This is a passing test/comparator result, not a warning-free execution or +proof those diagnostics are harmless. Structured revision, environment digest, +audit and result observations are retained in +[dependency-validation-2026-11-03.json](dependency-validation-2026-11-03.json). +These observations do not replace the original historical table. + +Prepared with OpenAI Codex. diff --git a/experiments/a2a-tck/dependency-validation-2026-11-03.json b/experiments/a2a-tck/dependency-validation-2026-11-03.json new file mode 100644 index 00000000..ebd79a99 --- /dev/null +++ b/experiments/a2a-tck/dependency-validation-2026-11-03.json @@ -0,0 +1,99 @@ +{ + "date": "2026-11-03", + "python": "3.12.14", + "environment_sha256": "68014f5dcb26a0900549e044c94a76a94b05491583c246bbd7a448bc0237183c", + "audit_tool": "pip-audit 2.10.1", + "audit_before": [ + { + "package": "filelock", + "version": "3.32.5", + "advisories": [ + "CVE-2026-112940", + "CVE-2026-112913", + "CVE-2026-112907", + "CVE-2026-112938", + "CVE-2026-112936", + "CVE-2026-112910", + "CVE-2026-112909", + "CVE-2026-112911", + "CVE-2026-112939", + "CVE-2026-112915", + "CVE-2026-112935", + "CVE-2026-112908" + ] + }, + { + "package": "msgpack", + "version": "1.2.2", + "advisories": [ + "CVE-2026-114613" + ] + }, + { + "package": "opentelemetry-api", + "version": "1.44.0", + "advisories": [ + "CVE-2026-114603" + ] + }, + { + "package": "setuptools", + "version": "84.0.0", + "advisories": [ + "GHSA-xmj3-9gf7-h52w" + ] + } + ], + "audit_after": { + "packages": 97, + "findings": 0 + }, + "focused_tests_passed": 70, + "tck": { + "baseline": { + "ca2a_commit": "29d0d5087ed46ecd17dafd5a945bd76a7386b7f1", + "ca2a_tracked_changes": "", + "tck_commit": "5daf0144b4a4adbdefb62180739b13c5bd7d8196", + "tck_lock_sha256": "7650dafd015617312c49f87bfafaa17e96f6af5d7d73540ce0ef4a2f46dfe404", + "runner_sha256": "94c6c29bc669422fee9ca005ae64ee54d5e917bdb9d585c7df195f96c9db6750", + "sdk": "1.2.2", + "exit_code": 0, + "timed_out": false, + "protected_handler_invocations": 0, + "passed": 79, + "skipped": 156, + "deselected": 30 + }, + "extension": { + "ca2a_commit": "29d0d5087ed46ecd17dafd5a945bd76a7386b7f1", + "ca2a_tracked_changes": "", + "tck_commit": "5daf0144b4a4adbdefb62180739b13c5bd7d8196", + "tck_lock_sha256": "7650dafd015617312c49f87bfafaa17e96f6af5d7d73540ce0ef4a2f46dfe404", + "runner_sha256": "94c6c29bc669422fee9ca005ae64ee54d5e917bdb9d585c7df195f96c9db6750", + "sdk": "1.2.2", + "exit_code": 0, + "timed_out": false, + "protected_handler_invocations": 0, + "passed": 79, + "skipped": 156, + "deselected": 30 + } + }, + "limits": [ + "Local Linux software experiment; no protected-path TCK certification", + "SUT audit only; upstream TCK environment is separate", + "SDK/OpenTelemetry context-detach and pending-task cleanup diagnostics occurred; not a warning-free run" + ], + "local_source_commit": "29d0d5087ed46ecd17dafd5a945bd76a7386b7f1", + "published_equivalent_source_commit": "98f4fd69d789509ea81be8aa2a43870d586d91cb", + "source_tree": "71ae1b78eacbb3f74b0e680ed1948ab876c136b5", + "publication_note": "The GitHub API-created commit has the identical tested source tree; retained TCK manifests identify the local commit.", + "repository_checks": { + "environment": "separate hash-locked requirements/dev.txt environment with editable checkout", + "unit_passed": 796, + "unit_skipped": 2, + "ruff": "pass", + "mypy": "47 source files pass", + "bandit": "pass" + } +} diff --git a/experiments/a2a-tck/environment.txt b/experiments/a2a-tck/environment.txt new file mode 100644 index 00000000..1a13d96c --- /dev/null +++ b/experiments/a2a-tck/environment.txt @@ -0,0 +1,97 @@ +a2a-sdk==1.2.2 +agent-manifest==0.15.0 +agentrust-trace==0.11.0 +agentrust-trace-tests==0.6.2 +aiohappyeyeballs==2.7.1 +aiohttp==3.14.5 +aiologic==0.17.1 +aiosignal==1.4.0 +annotated-types==0.8.0 +anyio==4.15.1 +ast-serialize==0.13.0 +attrs==26.1.0 +bandit==1.9.4 +boolean-py==5.0 +cachecontrol==0.14.4 +cbor2==6.1.4 +cedarpy==4.12.1 +certifi==2026.10.19 +cffi==2.1.1 +charset-normalizer==3.5.1 +click==8.5.0 +coverage==7.16.0 +cryptography==50.0.1 +culsans==0.11.0 +cyclonedx-python-lib==11.12.0 +defusedxml==0.7.1 +filelock==4.2.1 +frozenlist==1.8.0 +google-api-core==2.36.0 +google-auth==2.57.1 +googleapis-common-protos==1.75.3 +grpcio==1.84.0 +grpcio-reflection==1.84.0 +grpcio-tools==1.84.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.19 +iniconfig==2.3.0 +json-rpc==1.15.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +librt==0.16.0 +license-expression==30.4.4 +markdown-it-py==4.2.0 +mdurl==0.1.2 +msgpack==1.2.3 +multidict==6.9.1 +mypy==2.4.0 +mypy-extensions==1.1.0 +opentelemetry-api==1.45.1 +packageurl-python==0.17.6 +packaging==26.3 +pathspec==1.1.1 +pip==26.2.1 +pip-api==0.0.34 +pip-audit==2.10.1 +pip-requirements-parser==32.0.1 +platformdirs==4.13.1 +pluggy==1.6.0 +propcache==0.5.2 +proto-plus==1.28.4 +protobuf==7.36.2 +py-serializable==2.1.0 +pyasn1==0.6.5 +pyasn1-modules==0.4.2 +pycparser==3.0 +pydantic==2.13.5 +pydantic-core==2.46.5 +pygments==2.21.0 +pyparsing==3.3.2 +pytest==9.1.1 +pytest-asyncio==1.4.0 +pytest-cov==7.1.0 +pyyaml==6.0.3 +referencing==0.37.0 +requests==2.34.2 +rfc3986-validator==0.1.1 +rfc8785==0.1.4 +rich==15.0.0 +rpds-py==2026.6.3 +ruff==0.17.1 +setuptools==84.1.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +sse-starlette==3.4.11 +starlette==1.6.0 +stevedore==5.9.1 +tomli==2.4.1 +tomli-w==1.2.0 +types-pyyaml==6.0.12.20260906 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +urllib3==2.8.0 +uvicorn==0.54.0 +wrapt==2.4.0 +yarl==1.25.1 diff --git a/experiments/a2a-tck/run.py b/experiments/a2a-tck/run.py new file mode 100644 index 00000000..729fdb98 --- /dev/null +++ b/experiments/a2a-tck/run.py @@ -0,0 +1,118 @@ +"""Pinned, Linux-only differential TCK characterization of the existing SDK demo.""" +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import importlib.metadata +import importlib.util +import json +import os +import signal +import subprocess +import sys +from pathlib import Path + +TCK_SHA = "5daf0144b4a4adbdefb62180739b13c5bd7d8196" +ROOT = Path(__file__).resolve().parents[2] + + +def git(path: Path, *args: str) -> str: + return subprocess.check_output(["git", "-C", str(path), *args], text=True).strip() + + +async def run(args: argparse.Namespace) -> int: + if sys.platform != "linux": + raise SystemExit("This characterization runner requires Linux process groups") + tck = args.tck.resolve() + if git(tck, "rev-parse", "HEAD") != TCK_SHA: + raise SystemExit(f"TCK must be checked out at {TCK_SHA}") + if git(tck, "status", "--porcelain", "--untracked-files=all"): + raise SystemExit("TCK checkout must have no tracked or untracked changes") + out = args.output.resolve() + out.mkdir(parents=True, exist_ok=False) + spec = importlib.util.spec_from_file_location("sdk_demo", ROOT / "examples/a2a-sdk/loopback.py") + assert spec and spec.loader + demo = importlib.util.module_from_spec(spec) + spec.loader.exec_module(demo) + original_card = demo.operator_card + scenario_spec = importlib.util.spec_from_file_location( + "tck_scenario", tck / "sut/a2a-python/sut_agent.py" + ) + assert scenario_spec and scenario_spec.loader + scenario = importlib.util.module_from_spec(scenario_spec) + scenario_spec.loader.exec_module(scenario) + + class ScenarioExecutor(demo.InventoryExecutor): + async def execute(self, context, event_queue): + message = context.message + metadata = demo.metadata_from_sdk_message(message) if message else {} + opted = message is not None and demo.EXTENSION_URI in message.extensions + if (demo.has_ca2a_metadata(metadata) or opted + or demo.EXTENSION_URI in context.requested_extensions): + await super().execute(context, event_queue) + else: + await scenario.TckAgentExecutor().execute(context, event_queue) + + async def cancel(self, context, event_queue): + await scenario.TckAgentExecutor().cancel(context, event_queue) + + def card(url, node): + value = original_card(url, node) + value.capabilities.streaming = True + if args.mode == "baseline": + del value.capabilities.extensions[:] + return value + + demo.operator_card = card + demo.InventoryExecutor = ScenarioExecutor + chain, _ = demo.demo_credentials() + node = demo.PeerNode(demo.LocalPolicy.of({demo.READ}), trusted_root_issuers={chain[0].issuer}) + env = {key: value for key, value in os.environ.items() + if key.lower() not in {"http_proxy", "https_proxy", "all_proxy"} + and not key.startswith("PYTEST_")} + env["NO_PROXY"] = env["no_proxy"] = "127.0.0.1,localhost" + command = ["uv", "run", "--frozen", "python", "run_tck.py"] + async with demo.serve(node) as (url, executor): + command += ["--sut-host", url, "--transport", "jsonrpc", "--level", "must", "--", + f"--compatibility-report={out / 'compatibility'}", + f"--html={out / 'pytest.html'}", f"--junitxml={out / 'junit.xml'}"] + timed_out = False + with (out / "run.log").open("wb") as log: + process = await asyncio.create_subprocess_exec( + *command, cwd=tck, env=env, stdout=log, stderr=asyncio.subprocess.STDOUT, + start_new_session=True, + ) + try: + code = await asyncio.wait_for(process.wait(), timeout=args.timeout) + except TimeoutError: + timed_out = True + os.killpg(process.pid, signal.SIGTERM) + try: + await asyncio.wait_for(process.wait(), timeout=5) + except TimeoutError: + os.killpg(process.pid, signal.SIGKILL) + await process.wait() + code = 124 + manifest = { + "mode": args.mode, "ca2a_commit": git(ROOT, "rev-parse", "HEAD"), + "ca2a_tracked_changes": git(ROOT, "status", "--porcelain", "--untracked-files=no"), + "runner_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "tck_commit": TCK_SHA, + "tck_lock_sha256": hashlib.sha256((tck / "uv.lock").read_bytes()).hexdigest(), + "python": sys.version, "sdk": importlib.metadata.version("a2a-sdk"), + "command": command, "exit_code": code, "timed_out": timed_out, + "protected_handler_invocations": executor.invocations, + "scope": "JSON-RPC MUST with pinned upstream scenario executor; cA2A ordinary-path differential", + } + (out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + return code + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tck", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--mode", choices=["baseline", "extension"], required=True) + parser.add_argument("--timeout", type=float, default=180) + raise SystemExit(asyncio.run(run(parser.parse_args()))) diff --git a/experiments/a2a-tck/test_compare.py b/experiments/a2a-tck/test_compare.py new file mode 100644 index 00000000..aaf5b6d3 --- /dev/null +++ b/experiments/a2a-tck/test_compare.py @@ -0,0 +1,69 @@ +"""Negative controls for the evidence comparator.""" +import json +from pathlib import Path + +import pytest +from compare import compare + + +def write_run(root: Path, mode: str): + root.mkdir() + (root / 'manifest.json').write_text(json.dumps({ + 'mode': mode, 'exit_code': 0, 'timed_out': False, 'ca2a_tracked_changes': '', + 'ca2a_commit': 'ca2a', 'tck_commit': 'tck', 'tck_lock_sha256': 'lock', + 'runner_sha256': 'runner', 'sdk': '1.2.2', 'protected_handler_invocations': 0, + })) + (root / 'junit.xml').write_text('') + + +@pytest.mark.parametrize('mutation', ['failure', 'missing', 'rename', 'skip', 'provenance', 'exit']) +def test_rejects_false_equivalence(tmp_path, mutation): + left, right = tmp_path / 'baseline', tmp_path / 'extension' + write_run(left, 'baseline') + write_run(right, 'extension') + assert compare(left, right)['identical_test_outcomes'] + path = right / 'junit.xml' + if mutation == 'failure': + path.write_text('') + elif mutation == 'missing': + path.write_text('') + elif mutation == 'rename': + path.write_text(path.read_text().replace('one', 'two')) + elif mutation == 'skip': + path.write_text('') + else: + path = right / 'manifest.json' + data = json.loads(path.read_text()) + data['sdk' if mutation == 'provenance' else 'exit_code'] = 'changed' + path.write_text(json.dumps(data)) + with pytest.raises(ValueError): + compare(left, right) + + +@pytest.mark.parametrize('counts', [(0, 7), (7, 0), (7, 7), (False, 0), (0, 0.0), (0, None), (0, -1)]) +def test_protected_handler_zero_is_enforced(tmp_path, counts): + left, right = tmp_path / 'baseline', tmp_path / 'extension' + write_run(left, 'baseline') + write_run(right, 'extension') + assert compare(left, right)['protected_handler_invocations'] == [0, 0] + for root, count in zip((left, right), counts, strict=True): + path = root / 'manifest.json' + data = json.loads(path.read_text()) + data['protected_handler_invocations'] = count + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match='protected-handler'): + compare(left, right) + + +@pytest.mark.parametrize('mode', ['baseline', 'extension']) +def test_dirty_source_cannot_compare_equal(tmp_path, mode): + left, right = tmp_path / 'baseline', tmp_path / 'extension' + write_run(left, 'baseline') + write_run(right, 'extension') + root = left if mode == 'baseline' else right + path = root / 'manifest.json' + data = json.loads(path.read_text()) + data['ca2a_tracked_changes'] = ' M src/ca2a_runtime/peer.py' + path.write_text(json.dumps(data)) + with pytest.raises(ValueError, match='tracked files'): + compare(left, right) diff --git a/pyproject.toml b/pyproject.toml index 4cf4691a..f6515ffe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ tpm = [ a2a-sdk = [ "a2a-sdk>=1.1,<2", "aiohttp>=3.14.3", + "pyasn1>=0.6.5", ] dev = [ "pytest>=8.0", @@ -66,6 +67,7 @@ dev = [ "a2a-sdk[http-server]==1.2.2", "uvicorn==0.54.0", "aiohttp>=3.14.3", + "pyasn1>=0.6.5", ] [project.scripts] diff --git a/requirements/dev.in b/requirements/dev.in index ad636eff..36a848d0 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -25,3 +25,6 @@ a2a-sdk[http-server]==1.2.2 uvicorn==0.54.0 aiohttp>=3.14.3 setuptools>=83.0.0 + +# Security floor for google-auth's transitive ASN.1 dependency (four audit findings). +pyasn1>=0.6.5 diff --git a/requirements/dev.txt b/requirements/dev.txt index a57d254f..981e67dc 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1710,10 +1710,12 @@ py-serializable==2.1.0 \ --hash=sha256:9d5db56154a867a9b897c0163b33a793c804c80cee984116d02d49e4578fc103 \ --hash=sha256:b56d5d686b5a03ba4f4db5e769dc32336e142fc3bd4d68a8c25579ebb0a67304 # via cyclonedx-python-lib -pyasn1==0.6.4 \ - --hash=sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81 \ - --hash=sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b - # via pyasn1-modules +pyasn1==0.6.5 \ + --hash=sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587 \ + --hash=sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47 + # via + # -r requirements/dev.in + # pyasn1-modules pyasn1-modules==0.4.2 \ --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6