From 4db80f610e9ff7af580f40954c9ef6263919abe8 Mon Sep 17 00:00:00 2001 From: Seongjae Date: Sat, 19 Sep 2026 21:12:10 +0900 Subject: [PATCH] Add reproducible upstream dependency and contract validation --- .github/workflows/ci.yml | 130 +++++++++++++------------ .gitignore | 1 + crates/pty/src/lib.rs | 33 +++++-- docs/ko/upstream-update.md | 33 +++++++ docs/translations.json | 6 +- docs/upstream-update.md | 35 +++++++ scripts/check-upstream-pin.sh | 3 +- scripts/tests/test_upstream.py | 116 ++++++++++++++++++++++ scripts/upstream_dependencies.py | 122 +++++++++++++++++++++++ scripts/validate-upstream.py | 161 +++++++++++++++++++++++++++++++ 10 files changed, 567 insertions(+), 73 deletions(-) create mode 100644 scripts/tests/test_upstream.py create mode 100644 scripts/upstream_dependencies.py create mode 100644 scripts/validate-upstream.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba2848a..abaf372 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,13 @@ jobs: - name: Core Codex / model dep scan env: SCAN_BASE: ${{ github.event.pull_request.base.sha || github.event.before }} - run: ./scripts/check-no-model-deps.sh + run: python3 scripts/validate-upstream.py policy python --output target/upstream-reports/policy + - uses: actions/upload-artifact@v4 + if: always() + with: + name: upstream-${{ github.job }}-${{ matrix.name || 'single' }}-${{ github.run_attempt }} + path: target/upstream-reports/ + if-no-files-found: warn rust-format: name: Rust / Format + upstream pin @@ -28,16 +34,14 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: components: rustfmt - - name: Upstream pin - run: PIN_ONLY=1 ./scripts/check-upstream-pin.sh - - name: Format - run: | - cargo fmt --check - cargo fmt --check --manifest-path crates/patch/Cargo.toml - cargo fmt --check --manifest-path crates/codex-runtime/Cargo.toml - cargo fmt --check --manifest-path crates/pty/Cargo.toml - cargo fmt --check --manifest-path crates/file-system/Cargo.toml - cargo fmt --check --manifest-path crates/linux-sandbox/Cargo.toml + - name: Format and pin + run: python3 scripts/validate-upstream.py pin format --output target/upstream-reports/format + - uses: actions/upload-artifact@v4 + if: always() + with: + name: upstream-${{ github.job }}-${{ matrix.name || 'single' }}-${{ github.run_attempt }} + path: target/upstream-reports/ + if-no-files-found: warn rust-clippy: name: Rust / Clippy (${{ matrix.name }}) @@ -47,27 +51,11 @@ jobs: matrix: include: - name: root - command: | - set -euo pipefail - cargo clippy --locked --all-targets -- -D warnings - fail=0 - for pkg in codespace-linux-sandbox codex-linux-sandbox; do - if cargo tree -p codespace-runner --locked --edges normal -i "$pkg" --prefix none >/dev/null 2>&1; then - echo "codespace-runner graph must not include $pkg" >&2 - cargo tree -p codespace-runner --locked --edges normal -i "$pkg" --prefix none >&2 || true - fail=1 - fi - done - test "$fail" -eq 0 + stage: clippy-root dependencies - name: adapters - command: | - cargo clippy --locked --manifest-path crates/patch/Cargo.toml --all-targets -- -D warnings - cargo clippy --locked --manifest-path crates/pty/Cargo.toml --all-targets -- -D warnings - cargo clippy --locked --manifest-path crates/file-system/Cargo.toml --all-targets -- -D warnings + stage: clippy-adapters - name: codex-adapters - command: | - cargo clippy --locked --manifest-path crates/codex-runtime/Cargo.toml --all-targets -- -D warnings - cargo clippy --locked --manifest-path crates/linux-sandbox/Cargo.toml --all-targets -- -D warnings + stage: clippy-codex steps: - uses: actions/checkout@v4 with: @@ -83,7 +71,13 @@ jobs: ~/.cargo/git key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} - name: Clippy - run: ${{ matrix.command }} + run: python3 scripts/validate-upstream.py ${{ matrix.stage }} --output target/upstream-reports/${{ matrix.name }} + - uses: actions/upload-artifact@v4 + if: always() + with: + name: upstream-${{ github.job }}-${{ matrix.name || 'single' }}-${{ github.run_attempt }} + path: target/upstream-reports/ + if-no-files-found: warn rust-unit: name: Rust / Unit (${{ matrix.name }}) @@ -91,19 +85,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - name: patch - command: cargo test --locked --manifest-path crates/patch/Cargo.toml - - name: codex-runtime - command: cargo test --locked --manifest-path crates/codex-runtime/Cargo.toml - - name: pty - command: cargo test --locked --manifest-path crates/pty/Cargo.toml - - name: file-system - command: cargo test --locked --manifest-path crates/file-system/Cargo.toml - - name: linux-sandbox-protocol - command: cargo test --locked -p codespace-linux-sandbox-protocol - - name: linux-sandbox - command: cargo test --locked --manifest-path crates/linux-sandbox/Cargo.toml --bins --test cli + name: [patch, codex-runtime, pty, file-system, linux-sandbox-protocol, linux-sandbox] steps: - uses: actions/checkout@v4 with: @@ -117,7 +99,13 @@ jobs: ~/.cargo/git key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} - name: Unit tests - run: ${{ matrix.command }} + run: python3 scripts/validate-upstream.py unit-${{ matrix.name }} --output target/upstream-reports/unit-${{ matrix.name }} + - uses: actions/upload-artifact@v4 + if: always() + with: + name: upstream-${{ github.job }}-${{ matrix.name || 'single' }}-${{ github.run_attempt }} + path: target/upstream-reports/ + if-no-files-found: warn rust-linux-isolation: name: Rust / Linux isolation @@ -154,12 +142,14 @@ jobs: echo "Disabling kernel.apparmor_restrict_unprivileged_userns for bubblewrap." sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 fi - - name: Build Linux sandbox helper - run: cargo build --locked --manifest-path crates/linux-sandbox/Cargo.toml --bin codespace-linux-sandbox - name: Linux isolation tests - env: - CODESPACE_REQUIRE_LINUX_SANDBOX: "1" - run: cargo test --locked --manifest-path crates/linux-sandbox/Cargo.toml --test isolation + run: python3 scripts/validate-upstream.py linux-isolation --output target/upstream-reports/isolation + - uses: actions/upload-artifact@v4 + if: always() + with: + name: upstream-${{ github.job }}-${{ matrix.name || 'single' }}-${{ github.run_attempt }} + path: target/upstream-reports/ + if-no-files-found: warn rust-integration: name: Rust / Integration @@ -191,17 +181,33 @@ jobs: if [ -n "$current_apparmor" ] && [ "$current_apparmor" != "0" ]; then sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 fi - - name: Build helpers - run: | - cargo build --locked --manifest-path crates/patch/Cargo.toml --bin codespace-patch - cargo build --locked --manifest-path crates/codex-runtime/Cargo.toml --bin codespace-codex-runtime - cargo build --locked --manifest-path crates/linux-sandbox/Cargo.toml --bin codespace-linux-sandbox - name: Workspace integration tests - env: - CODESPACE_PATCH_BIN: ${{ github.workspace }}/crates/patch/target/debug/codespace-patch - CODESPACE_RUNTIME_BIN: ${{ github.workspace }}/crates/codex-runtime/target/debug/codespace-codex-runtime - CODESPACE_LINUX_SANDBOX_BIN: ${{ github.workspace }}/crates/linux-sandbox/target/debug/codespace-linux-sandbox - run: cargo test --locked --workspace + run: python3 scripts/validate-upstream.py integration --output target/upstream-reports/integration + - uses: actions/upload-artifact@v4 + if: always() + with: + name: upstream-${{ github.job }}-${{ matrix.name || 'single' }}-${{ github.run_attempt }} + path: target/upstream-reports/ + if-no-files-found: warn + + rust-macos: + name: Rust / macOS contracts + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - name: macOS contracts and dependencies + run: python3 scripts/validate-upstream.py macos-core dependencies --output target/upstream-reports/macos + - uses: actions/upload-artifact@v4 + if: always() + with: + name: upstream-${{ github.job }}-${{ matrix.name || 'single' }}-${{ github.run_attempt }} + path: target/upstream-reports/ + if-no-files-found: warn rust: name: rust @@ -212,6 +218,8 @@ jobs: - rust-unit - rust-linux-isolation - rust-integration + - rust-macos + - policy-scan runs-on: ubuntu-latest steps: - name: Require all Rust checks @@ -221,3 +229,5 @@ jobs: test "${{ needs.rust-unit.result }}" = "success" test "${{ needs.rust-linux-isolation.result }}" = "success" test "${{ needs.rust-integration.result }}" = "success" + test "${{ needs.rust-macos.result }}" = "success" + test "${{ needs.policy-scan.result }}" = "success" diff --git a/.gitignore b/.gitignore index 1f87eb6..5726d28 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ compile-out/ .turbo/ .direnv/ .local/ +__pycache__/ diff --git a/crates/pty/src/lib.rs b/crates/pty/src/lib.rs index c1e90a3..539791c 100644 --- a/crates/pty/src/lib.rs +++ b/crates/pty/src/lib.rs @@ -88,16 +88,29 @@ mod tests { assert_eq!(env!("CARGO_PKG_NAME"), "codespace-pty"); } - #[test] - fn default_size_is_24x80_and_matches_upstream() { - assert_eq!(DEFAULT_ROWS, 24); - assert_eq!(DEFAULT_COLS, 80); - let upstream = codex_utils_pty::TerminalSize::default(); - assert_eq!( - (upstream.rows, upstream.cols), - (DEFAULT_ROWS, DEFAULT_COLS), - "upstream TerminalSize::default drifted from advertised PTY size" - ); + #[tokio::test] + async fn created_terminal_has_advertised_size() { + let dir = tempfile::tempdir().unwrap(); + let env = HashMap::from([("PATH".into(), "/usr/bin:/bin".into())]); + let mut session = spawn("/bin/stty", &["size".into()], dir.path(), &env) + .await + .expect("spawn stty"); + let mut output = session.take_stdout().expect("stdout"); + let code = tokio::time::timeout(Duration::from_secs(5), session.take_exit().unwrap()) + .await + .expect("exit timeout") + .expect("exit receiver"); + assert_eq!(code, 0); + let bytes = tokio::time::timeout(Duration::from_secs(5), async { + let mut bytes = Vec::new(); + while let Some(chunk) = output.recv().await { + bytes.extend(chunk); + } + bytes + }) + .await + .expect("output timeout"); + assert_eq!(String::from_utf8(bytes).unwrap().trim(), "24 80"); } #[tokio::test] diff --git a/docs/ko/upstream-update.md b/docs/ko/upstream-update.md index 7fd2d8f..82a09e3 100644 --- a/docs/ko/upstream-update.md +++ b/docs/ko/upstream-update.md @@ -36,3 +36,36 @@ ## 반영과 되돌리기 PR에 이전·새 SHA, 동작 변경, 테스트 근거를 기록합니다. 호환성 검사가 실패한 채 병합하거나 다른 패치 엔진으로 조용히 대체하지 않습니다. 되돌릴 때는 서브모듈, 어댑터 잠금 파일, 필요한 Cargo 패치, 문서를 함께 되돌리고 영향받는 검사를 다시 수행합니다. 고정 커밋 불일치는 경고가 아닌 오류입니다. + +## 재현 가능한 검증 보고서 + +전체 로컬 검증은 `python3 scripts/validate-upstream.py all`로 실행합니다. +CI도 같은 이름의 단계를 병렬 실행하며 목록은 `--help`로 확인합니다. +보고서와 명령 로그는 기본적으로 `target/upstream-reports/local`에 생성됩니다. +시도마다 `--output`으로 다른 Git 무시 디렉터리를 지정하거나 이전 결과를 보관합니다. +보고서에는 소스 HEAD, Codex SHA, Rust 호스트, 소스와 lockfile 해시가 기록되며 +실행 중 입력이 바뀌면 실패합니다. `passed`는 기록된 단계만의 통과를 뜻합니다. +macOS에서는 Linux 격리를 `not_run`, 전체 결과를 `incomplete`로 표시합니다. +Linux CI 근거가 별도로 필요하며 macOS CI는 PTY와 파일 시스템 계약도 검사합니다. +한 플랫폼 결과만으로 다른 플랫폼 검증을 대체하지 않습니다. + +의존성 검사는 `--locked`와 대상 플랫폼 필터를 사용한 Cargo metadata에서 +제품 root의 일반·빌드 의존성을 탐색하고 개발용 관계는 제외합니다. +에이전트·제품 crate 및 Runner에서 샌드박스 라이브러리로 향하는 경로는 실패합니다. +이는 패키지 도달 가능성 검사이며 모든 API가 실제 실행됨을 뜻하지 않습니다. +Cargo feature 통합으로 선택적 관계가 보수적으로 포함될 수 있습니다. + +동일한 Rust target의 보관된 결과와 후보 결과를 비교할 수 있습니다. +기준 결과를 자동 갱신하지 않습니다. + +```bash +python3 scripts/upstream_dependencies.py --target x86_64-unknown-linux-gnu \ + --compare target/baseline/dependencies.json \ + --output target/candidate/dependencies.json +``` + +보고서는 추가·삭제된 패키지와 관계를 나열합니다. 버전·출처 변경은 삭제와 추가로 +표시됩니다. 일반 변화는 검토 대상이며 금지 의존성, 잘못된 metadata, 누락된 root, +Cargo 실패는 검증 실패입니다. 경로 식별자는 저장소 상대 경로를 사용합니다. +CI는 실패 시에도 보고서와 로그를 업로드합니다. 기존 pin 검사 스크립트는 +SHA와 패치 테스트만 확인하며 전체 검증 명령을 대체하지 않습니다. diff --git a/docs/translations.json b/docs/translations.json index af8f671..d78bbd6 100644 --- a/docs/translations.json +++ b/docs/translations.json @@ -599,6 +599,7 @@ "forbidden", "local-gate", "release-and-rollback", + "reproducible-validation-reports", "review-the-candidate", "updating-codex-dependencies", "upstream-pin-update", @@ -611,12 +612,13 @@ "로컬-게이트", "반영과-되돌리기", "업스트림-핀-갱신", + "재현-가능한-검증-보고서", "점검-목록", "제출-전-검증", "후보-검토" ], - "source_sha256": "efe8b1d11db41148ec6dfd08a67b52fd72d2e32a480123f27668feff9e39f8da", - "translation_sha256": "fe4f0732755820b3b2db72190770359663ca6c6f2fcbc54227f714152ef7682d" + "source_sha256": "dcac8e5362953f5ade48326c69d48f4a448509853ec2eb6034fb6400fe0418b3", + "translation_sha256": "692bccd11d607df49a0802b0a37df204af071a23ec826d6e72f8bdd2973fa6ff" }, { "id": "documentation", diff --git a/docs/upstream-update.md b/docs/upstream-update.md index 5221eba..14ea331 100644 --- a/docs/upstream-update.md +++ b/docs/upstream-update.md @@ -32,3 +32,38 @@ A passing patch subset is insufficient for an update that also affects execution ## Release and rollback Open a PR with the old/new SHA, behavioral changes, and test evidence. Do not merge a failed compatibility gate or silently fall back to another patch engine. If the update must be reverted, revert the submodule, adapter locks, required Cargo patches, and documentation together; then rerun the affected gates. A pin mismatch is an error, not a warning. + +## Reproducible validation reports + +Run `python3 scripts/validate-upstream.py all` for the complete local sequence. +CI calls the same named stages in parallel; use `--help` to list them. +Reports and command logs default to `target/upstream-reports/local`. +Choose a different ignored directory with `--output` for each attempt; preserve +previous reports before repeating a run. Reports record the source HEAD, Codex +SHA, Rust host, and source/lockfile hashes. A changed input invalidates the run. +`passed` applies only to the listed stages, not to all release gates. +On macOS, Linux isolation is explicitly `not_run` and the overall result is +`incomplete`; Linux CI evidence is still required. macOS CI additionally checks +PTY and filesystem contracts. Neither result alone replaces the other platform. + +The dependency stage uses locked, target-filtered Cargo metadata and follows +normal/build edges from product roots, excluding development edges. It rejects +agent/product crates and Runner-to-sandbox-library edges with a dependency path. +This checks package reachability, not whether a binary executes every linked API. +Cargo's resolved feature unification can conservatively include optional edges. + +Generate a candidate report and compare it with an archived report for the same +Rust target (the baseline is never updated automatically): + +```bash +python3 scripts/upstream_dependencies.py --target x86_64-unknown-linux-gnu \ + --compare target/baseline/dependencies.json \ + --output target/candidate/dependencies.json +``` + +The comparison lists added/removed package identities and edges. A version or +source change appears as removal plus addition. Ordinary changes require review; +forbidden dependencies, malformed metadata, missing roots, or Cargo failure fail +the gate. Source paths are repository-relative, never machine-specific identities. +CI uploads reports/logs even on failed validation. The legacy pin script still +checks only SHA and patch tests; it is not a complete qualification command. diff --git a/scripts/check-upstream-pin.sh b/scripts/check-upstream-pin.sh index f4faae3..d8c545d 100755 --- a/scripts/check-upstream-pin.sh +++ b/scripts/check-upstream-pin.sh @@ -45,4 +45,5 @@ if [[ "${PIN_ONLY:-}" == "1" ]]; then exit 0 fi -cargo test --manifest-path crates/patch/Cargo.toml +echo "This gate covers SHA + patch only; full qualification: python3 scripts/validate-upstream.py all" +cargo test --locked --manifest-path crates/patch/Cargo.toml diff --git a/scripts/tests/test_upstream.py b/scripts/tests/test_upstream.py new file mode 100644 index 0000000..9f3f815 --- /dev/null +++ b/scripts/tests/test_upstream.py @@ -0,0 +1,116 @@ +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import upstream_dependencies as deps +spec = importlib.util.spec_from_file_location('validation', Path(__file__).resolve().parents[1] / 'validate-upstream.py') +validation = importlib.util.module_from_spec(spec) +spec.loader.exec_module(validation) + + +def fixture(names, edges): + return {'packages': [{'id': n, 'name': n, 'version': '1', 'source': None, + 'manifest_path': str(deps.ROOT / 'crates' / n / 'Cargo.toml')} for n in names], + 'resolve': {'nodes': [{'id': n, 'deps': [ + {'name': 'renamed_alias', 'pkg': child, 'dep_kinds': [{'kind': kind, 'target': None}]} + for parent, child, kind in edges if parent == n]} for n in names]}} + + +class GraphTests(unittest.TestCase): + def test_transitive_and_renamed_build_edge(self): + data = fixture(['app', 'adapter', 'codex-core'], [('app', 'adapter', None), ('adapter', 'codex-core', 'build')]) + self.assertEqual(deps.graph(data, ['app'])['violations'], ['app -> adapter -> codex-core']) + + def test_development_not_product(self): + data = fixture(['app', 'codex-core'], [('app', 'codex-core', 'dev')]) + self.assertFalse(deps.graph(data, ['app'])['violations']) + self.assertEqual(len(deps.graph(data, ['app'])['packages']), 1) + + def test_runner_boundary(self): + data = fixture(['codespace-runner', 'codex-linux-sandbox'], [('codespace-runner', 'codex-linux-sandbox', None)]) + self.assertTrue(deps.graph(data, ['codespace-runner'])['violations']) + + def test_direct_and_missing_roots(self): + data = fixture(['app', 'codex-login'], [('app', 'codex-login', None)]) + self.assertTrue(deps.graph(data, ['app'])['violations']) + with self.assertRaises(ValueError): + deps.graph(data, ['absent']) + with self.assertRaises(ValueError): + deps.graph(data, []) + + def test_target_forwarded_and_failure(self): + for target in ('x86_64-unknown-linux-gnu', 'aarch64-apple-darwin'): + with patch.object(deps.subprocess, 'check_output', return_value=b'{}') as call: + deps.metadata(Path('Cargo.toml'), target) + self.assertIn(target, call.call_args.args[0]) + self.assertIn('--locked', call.call_args.args[0]) + with patch.object(deps.subprocess, 'check_output', side_effect=subprocess.CalledProcessError(1, 'cargo')): + with self.assertRaises(subprocess.CalledProcessError): + deps.metadata(Path('Cargo.toml'), 'target') + with patch.object(deps.subprocess, 'check_output', return_value=b'bad json'): + with self.assertRaises(ValueError): + deps.metadata(Path('Cargo.toml'), 'target') + + def test_comparison_deterministic(self): + data = fixture(['app', 'dep'], [('app', 'dep', None)]) + first = deps.graph(data, ['app']) + data['packages'].reverse() + self.assertEqual(first, deps.graph(data, ['app'])) + old = {'schema': 1, 'target': 'linux', 'graphs': {'root': first}} + new = json.loads(json.dumps(old)) + new['graphs']['root']['packages'].append('["new","2","registry"]') + diff = deps.difference(old, new) + self.assertEqual(len(diff['root']['packages']['added']), 1) + new['target'] = 'macos' + with self.assertRaises(ValueError): + deps.difference(old, new) + + +class RunnerTests(unittest.TestCase): + def test_command_failure(self): + with patch.object(validation.subprocess, 'run') as call: + call.return_value.returncode = 7 + with self.assertRaises(RuntimeError): + validation.execute([['false']], {}, io.StringIO()) + + def test_helper_paths_absolute(self): + env = validation.helper_env(Path('/tmp/build')) + self.assertEqual(env['CODESPACE_PATCH_BIN'], '/tmp/build/debug/codespace-patch') + + def test_report_and_platform_skip(self): + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(validation, 'fingerprint', return_value={'head': 'a', 'codex_sha': 'pin'}), \ + patch.object(validation, 'capture', return_value='host: aarch64-apple-darwin'), \ + patch.object(validation.platform, 'system', return_value='Darwin'): + self.assertEqual(validation.run(['linux-isolation'], Path(tmp)), 0) + report = json.loads((Path(tmp) / 'report.json').read_text()) + self.assertEqual(report['status'], 'incomplete') + self.assertFalse(report['full_linux_qualification']) + + def test_changed_inputs_fail(self): + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(validation, 'fingerprint', side_effect=[{'head': 'a', 'codex_sha': 'pin'}, {'head': 'b', 'codex_sha': 'pin'}]), \ + patch.object(validation, 'capture', return_value='host: x86_64-unknown-linux-gnu'): + self.assertEqual(validation.run([], Path(tmp)), 1) + report = json.loads((Path(tmp) / 'report.json').read_text()) + self.assertIn('changed', report['error']) + + def test_failure_report(self): + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(validation, 'fingerprint', return_value={'head': 'a', 'codex_sha': 'pin'}), \ + patch.object(validation, 'capture', return_value='host: x86_64-unknown-linux-gnu'), \ + patch.object(validation, 'execute', side_effect=RuntimeError('failure')): + self.assertEqual(validation.run(['pin'], Path(tmp)), 1) + report = json.loads((Path(tmp) / 'report.json').read_text()) + self.assertEqual(report['stages'][0]['status'], 'failed') + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/upstream_dependencies.py b/scripts/upstream_dependencies.py new file mode 100644 index 0000000..f956566 --- /dev/null +++ b/scripts/upstream_dependencies.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Inspect target-filtered Cargo product graphs, excluding development edges.""" +import argparse +from collections import deque +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +ADAPTERS = ('patch', 'codex-runtime', 'pty', 'file-system', 'linux-sandbox') +PRODUCTS = {'root': {'codespace-domain', 'codespace-policy', 'codespace-runner', 'codespace-store', 'codespace-server', 'codespace-linux-sandbox-protocol'}, + 'patch': {'codespace-patch'}, 'codex-runtime': {'codespace-codex-runtime'}, + 'pty': {'codespace-pty'}, 'file-system': {'codespace-fs'}, 'linux-sandbox': {'codespace-linux-sandbox'}} +FORBIDDEN = {'codex-core', 'codex-exec', 'codex-app-server', 'codex-login'} +RUNNER_FORBIDDEN = {'codespace-linux-sandbox', 'codex-linux-sandbox'} + + +def metadata(manifest, target): + return json.loads(subprocess.check_output( + ['cargo', 'metadata', '--locked', '--format-version', '1', + '--filter-platform', target, '--manifest-path', str(manifest)], cwd=ROOT)) + + +def graph(data, roots): + packages = {p['id']: p for p in data['packages']} + nodes = {n['id']: n for n in data['resolve']['nodes']} + if not roots or any(r not in packages or r not in nodes for r in roots): + raise ValueError('missing product root or resolve node') + + def key(pid): + p = packages[pid] + source = p['source'] + if source is None: + path = Path(p['manifest_path']).resolve().parent + source = 'path:' + path.relative_to(ROOT).as_posix() + return json.dumps([p['name'], p['version'], source], separators=(',', ':')) + + found, edges, violations = set(), set(), [] + for root in roots: + queue, seen = deque([(root, [root])]), set() + banned = FORBIDDEN | (RUNNER_FORBIDDEN if packages[root]['name'] == 'codespace-runner' else set()) + while queue: + pid, trail = queue.popleft() + if pid in seen: + continue + seen.add(pid) + found.add(key(pid)) + if packages[pid]['name'] in banned: + violations.append(' -> '.join(packages[x]['name'] for x in trail)) + for dep in nodes[pid]['deps']: + for kind in dep['dep_kinds']: + if kind['kind'] not in (None, 'normal', 'build'): + continue + child = dep['pkg'] + if child not in nodes or child not in packages: + raise ValueError('unresolved dependency node') + edges.add((key(pid), key(child), kind['kind'] or 'normal', kind.get('target') or '')) + queue.append((child, trail + [child])) + return {'packages': sorted(found), 'edges': [list(e) for e in sorted(edges)], + 'violations': sorted(set(violations))} + + +def difference(old, new): + if old.get('schema') != new.get('schema') or old.get('target') != new.get('target'): + raise ValueError('baseline schema/target mismatch') + if old['graphs'].keys() != new['graphs'].keys(): + raise ValueError('baseline product roots mismatch') + result = {} + for name, current in new['graphs'].items(): + previous = old['graphs'][name] + changes = {} + for field in ('packages', 'edges'): + before = {json.dumps(v, sort_keys=True) for v in previous[field]} + after = {json.dumps(v, sort_keys=True) for v in current[field]} + changes[field] = {'added': [json.loads(v) for v in sorted(after-before)], + 'removed': [json.loads(v) for v in sorted(before-after)]} + result[name] = changes + return result + + +def inspect(target): + report = {'schema': 1, 'target': target, 'graphs': {}} + for area in ('root',) + ADAPTERS: + manifest = ROOT / ('Cargo.toml' if area == 'root' else f'crates/{area}/Cargo.toml') + data = metadata(manifest, target) + members = {p['name']: p['id'] for p in data['packages'] if p['id'] in data['workspace_members']} + if set(members) != PRODUCTS[area]: + raise ValueError(f'{area}: missing or unexpected product roots: {sorted(members)}') + report['graphs'][area] = graph(data, [members[name] for name in sorted(PRODUCTS[area])]) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--target', required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--compare', type=Path) + args = parser.parse_args() + report = {'schema': 1, 'target': args.target, 'graphs': {}} + code = 1 + try: + if args.compare and args.compare.resolve() == args.output.resolve(): + raise ValueError('output must not overwrite baseline') + report = inspect(args.target) + if args.compare: + report['comparison'] = difference(json.loads(args.compare.read_text()), report) + code = int(any(g['violations'] for g in report['graphs'].values())) + except (ValueError, KeyError, TypeError, OSError, subprocess.SubprocessError) as error: + report['error'] = str(error) + if args.compare and args.compare.resolve() == args.output.resolve(): + print(report['error'], file=sys.stderr) + return 1 + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + '\n') + if code: + print(json.dumps(report, indent=2), file=sys.stderr) + return code + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/validate-upstream.py b/scripts/validate-upstream.py new file mode 100644 index 0000000..37f2bb0 --- /dev/null +++ b/scripts/validate-upstream.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Shared local/CI upstream gates. No source or lockfile updates.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +import time + +from upstream_dependencies import ADAPTERS, ROOT + + +def capture(*cmd): + return subprocess.check_output(cmd, cwd=ROOT).decode().strip() + + +def fingerprint(): + paths = subprocess.check_output(['git', 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], cwd=ROOT).split(b'\0') + hashes = {} + for raw in paths: + if not raw: + continue + name = os.fsdecode(raw) + path = ROOT / name + if path.is_file(): + hashes[name] = hashlib.sha256(path.read_bytes()).hexdigest() + return {'head': capture('git', 'rev-parse', 'HEAD'), + 'codex_sha': capture('git', '-C', 'third_party/codex', 'rev-parse', 'HEAD') if (ROOT / 'third_party/codex/.git').exists() else None, + 'codex_status': capture('git', '-C', 'third_party/codex', 'status', '--porcelain') if (ROOT / 'third_party/codex/.git').exists() else 'not initialized', + 'codex_diff': hashlib.sha256(subprocess.check_output(['git', '-C', 'third_party/codex', 'diff', '--binary', 'HEAD'], cwd=ROOT)).hexdigest() if (ROOT / 'third_party/codex/.git').exists() else None, + 'files': hashes} + + +def cargo(action, area='root', *args): + cmd = ['cargo', action] + if action != 'fmt': + cmd += ['--locked'] + if area != 'root': + cmd += ['--manifest-path', f'crates/{area}/Cargo.toml'] + return cmd + list(args) + + +def stages(): + result = { + 'pin': [['bash', 'scripts/check-upstream-pin.sh']], + 'policy': [['bash', 'scripts/check-no-model-deps.sh']], + 'python': [[sys.executable, '-m', 'unittest', 'discover', '-s', 'scripts/tests']], + 'format': [cargo('fmt', area, '--check') for area in ('root',) + ADAPTERS], + 'clippy-root': [cargo('clippy', 'root', '--all-targets', '--', '-D', 'warnings')], + 'clippy-adapters': [cargo('clippy', a, '--all-targets', '--', '-D', 'warnings') for a in ('patch', 'pty', 'file-system')], + 'clippy-codex': [cargo('clippy', a, '--all-targets', '--', '-D', 'warnings') for a in ('codex-runtime', 'linux-sandbox')], + 'macos-core': [cargo(action, a, *(['--all-targets', '--', '-D', 'warnings'] if action == 'clippy' else [])) for a in ('pty', 'file-system') for action in ('clippy', 'test')], + 'integration': [], + 'linux-isolation': [cargo('build', 'linux-sandbox', '--bin', 'codespace-linux-sandbox'), cargo('test', 'linux-sandbox', '--test', 'isolation')], + 'dependencies': [], + } + for area in ADAPTERS: + extra = ['--bins', '--test', 'cli'] if area == 'linux-sandbox' else [] + result['unit-' + area] = [cargo('test', area, *extra)] + result['unit-linux-sandbox-protocol'] = [cargo('test', 'root', '-p', 'codespace-linux-sandbox-protocol')] + return result + + +HELPERS = {'patch': ('codespace-patch', 'CODESPACE_PATCH_BIN'), + 'codex-runtime': ('codespace-codex-runtime', 'CODESPACE_RUNTIME_BIN'), + 'linux-sandbox': ('codespace-linux-sandbox', 'CODESPACE_LINUX_SANDBOX_BIN')} + + +def helper_env(target_dir): + return {variable: str(target_dir / 'debug' / binary) for binary, variable in HELPERS.values()} + + +def execute(commands, env, log): + for command in commands: + print('+ ' + ' '.join(command), flush=True) + log.write('+ ' + ' '.join(command) + '\n') + log.flush() + status = subprocess.run(command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT).returncode + if status: + raise RuntimeError(f'command exited {status}: {command}') + + +def run(selected, output): + output.mkdir(parents=True, exist_ok=True) + report = {'schema': 1, 'platform': platform.platform(), 'stages': [], 'status': 'failed'} + failed = False + try: + before = fingerprint() + report['inputs'] = before + report['rust'] = capture('rustc', '-vV') + report['build_environment'] = {k: os.environ.get(k) for k in ('DEVELOPER_DIR', 'SDKROOT', 'RUSTFLAGS', 'CARGO_ENCODED_RUSTFLAGS', 'RUSTUP_TOOLCHAIN')} + if before.get('codex_sha') is None and any(s not in ('policy', 'python') for s in selected): + raise RuntimeError('Codex submodule is not initialized') + target = next(line.split(': ', 1)[1] for line in report['rust'].splitlines() if line.startswith('host: ')) + env = os.environ.copy() + env['PIN_ONLY'] = '1' + # Full policy scan for reproducible standalone and CI qualification. + env.pop('SCAN_BASE', None) + target_dir = (ROOT / 'target' / 'upstream-validation').resolve() + env['CARGO_TARGET_DIR'] = str(target_dir) + env.update(helper_env(target_dir)) + for stage in selected: + entry = {'name': stage, 'status': 'failed'} + report['stages'].append(entry) + if stage == 'linux-isolation' and platform.system() != 'Linux': + entry.update(status='not_run', reason='requires Linux with bubblewrap and user namespaces') + continue + started = time.monotonic() + try: + commands = stages()[stage] + if stage == 'dependencies': + commands = [[sys.executable, 'scripts/upstream_dependencies.py', '--target', target, '--output', str(output / 'dependencies.json')]] + elif stage == 'integration': + commands = [cargo('build', area, '--bin', binary) for area, (binary, _) in HELPERS.items()] + commands += [cargo('test', 'root', '--workspace')] + stage_env = env.copy() + if stage == 'linux-isolation': + stage_env['CODESPACE_REQUIRE_LINUX_SANDBOX'] = '1' + with (output / (stage + '.log')).open('w') as log: + execute(commands, stage_env, log) + entry['status'] = 'passed' + except (OSError, RuntimeError) as error: + entry['error'] = str(error) + failed = True + entry['seconds'] = round(time.monotonic() - started, 2) + if fingerprint() != before: + raise RuntimeError('source or validation inputs changed during execution') + report['status'] = 'failed' if failed else ('incomplete' if any(s['status'] == 'not_run' for s in report['stages']) else 'passed') + report['scope'] = selected + report['full_linux_qualification'] = not failed and platform.system() == 'Linux' and set(all_stages()) <= set(selected) + except (OSError, ValueError, RuntimeError, subprocess.SubprocessError, StopIteration) as error: + report['error'] = str(error) + failed = True + finally: + (output / 'report.json').write_text(json.dumps(report, indent=2, sort_keys=True) + '\n') + print(json.dumps({k: report[k] for k in ('status', 'stages')}, indent=2)) + return int(failed) + + +def all_stages(): + return [s for s in stages() if s != 'macos-core'] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('stage', nargs='+', choices=['all'] + list(stages())) + parser.add_argument('--output', type=Path, default=ROOT / 'target/upstream-reports/local') + args = parser.parse_args() + selected = all_stages() if 'all' in args.stage else list(dict.fromkeys(args.stage)) + # Reports must be ignored; otherwise they would invalidate the source snapshot themselves. + output = args.output.resolve() + if subprocess.run(['git', 'check-ignore', '-q', str(output / 'report.json')], cwd=ROOT).returncode != 0: + parser.error('--output must be inside a git-ignored directory (e.g. target/upstream-reports)') + return run(selected, output) + + +if __name__ == '__main__': + sys.exit(main())