diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..bef3f1f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,8 @@ +* @jstar0 + +/docs/superpowers/specs/ @jstar0 +/internal/fold/ @jstar0 +/internal/pack/ @jstar0 +/internal/storage/ @jstar0 +/internal/vfs/ @jstar0 +/platform/darwin/fskit/ @jstar0 diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..2794e4d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,58 @@ +name: Bug report +description: Report a reproducible CodexFold defect using synthetic or redacted data. +title: "bug: " +labels: [bug] +body: + - type: markdown + attributes: + value: Do not attach real Codex rollouts, databases, credentials, private prompts, or unredacted logs. Use GitHub Security Advisories for vulnerabilities. + - type: input + id: version + attributes: + label: Version or commit + placeholder: v0.2.1 or commit SHA + validations: + required: true + - type: dropdown + id: platform + attributes: + label: Platform + options: + - macOS + - Linux + - Windows + - Other + validations: + required: true + - type: textarea + id: behavior + attributes: + label: Observed behavior + description: Include the command, sanitized error, and whether storage-only or filesystem-preview behavior was involved. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected behavior + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: Synthetic reproduction + description: Provide minimal steps using generated or redacted fixtures. + validations: + required: true + - type: textarea + id: verification + attributes: + label: Verification already attempted + description: List doctor, tests, hashes, restart checks, or rollback attempts without private data. + - type: checkboxes + id: hygiene + attributes: + label: Data hygiene + options: + - label: I removed real session content, credentials, private prompts, local paths, and unredacted logs. + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..7fb17d7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Security vulnerability + url: https://github.com/samekind/codexfold/security/advisories/new + about: Report vulnerabilities and sensitive findings privately. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..ad1ec8a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,29 @@ +name: Feature request +description: Propose a product behavior or engineering improvement. +title: "feat: " +labels: [enhancement] +body: + - type: textarea + id: problem + attributes: + label: Problem + description: Describe the concrete workflow or limitation, not only the proposed implementation. + validations: + required: true + - type: textarea + id: outcome + attributes: + label: Required outcome + description: State observable acceptance criteria. + validations: + required: true + - type: textarea + id: constraints + attributes: + label: Safety and compatibility constraints + description: Note byte identity, rollback, privacy, platform, performance, or release-gate implications. + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Include simpler options and why they are insufficient. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..25924dd --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,21 @@ +## Outcome + +Describe the user-visible or maintainer-visible result. + +## Safety Impact + +Describe effects on session bytes, routing, storage, filesystem behavior, service lifecycle, compatibility, and rollback. Write `None` only when none apply. + +## Verification + +List the exact commands and real environments used. Distinguish unit, synthetic, mounted-adapter, real-client, restart, and production evidence. + +## Checklist + +- [ ] The change matches the product contract and does not weaken a release gate. +- [ ] Tests cover the behavior or regression. +- [ ] `go test ./...`, race tests, vet, formatting, and required cross-builds pass. +- [ ] Platform-specific validation was run when platform code changed. +- [ ] Documentation and readiness language match the available evidence. +- [ ] No real rollout, credential, private prompt, local path, database, log, build artifact, or Xcode user state is included. +- [ ] Production Codex data and production service definitions were not used for development validation. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13fc7fa..dfae646 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,21 +4,108 @@ on: push: branches: [main] pull_request: + workflow_dispatch: permissions: contents: read +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: + quality: + name: quality + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.1 + - uses: actions/setup-go@v7.0.0 + with: + go-version-file: go.mod + cache: true + - name: Check formatting + shell: bash + run: | + files="$(gofmt -l .)" + if [ -n "$files" ]; then + printf '%s\n' "$files" + exit 1 + fi + - name: Check module files + run: | + go mod tidy + git diff --exit-code -- go.mod go.sum + - name: Check release metadata + run: ./scripts/check-release.sh + - run: go vet ./... + test: + name: test (${{ matrix.os }}) strategy: + fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v7.0.1 + - uses: actions/setup-go@v7.0.0 with: - go-version: "1.26.x" + go-version-file: go.mod cache: true - run: go test ./... -count=1 - run: go build ./cmd/codexfold + - name: Test native FSKit cache implementation + if: runner.os == 'macOS' + run: ./scripts/test-native-fskit-cache.sh + + release-config: + name: release-config + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.1 + - uses: goreleaser/goreleaser-action@v7.2.3 + with: + distribution: goreleaser + version: "~> v2" + args: check + + windows-compile: + name: windows-compile + runs-on: windows-latest + steps: + - uses: actions/checkout@v7.0.1 + - uses: actions/setup-go@v7.0.0 + with: + go-version-file: go.mod + cache: true + - name: Compile all test packages without claiming runtime validation + run: go test ./... -run '^$' -count=1 + - run: go build ./cmd/codexfold + + race: + name: race + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.1 + - uses: actions/setup-go@v7.0.0 + with: + go-version-file: go.mod + cache: true + - run: go test -race ./... -count=1 + + cross-build: + name: cross-build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.1 + - uses: actions/setup-go@v7.0.0 + with: + go-version-file: go.mod + cache: true + - name: Build Linux and Windows artifacts + shell: bash + run: | + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "$RUNNER_TEMP/codexfold-linux-amd64" ./cmd/codexfold + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o "$RUNNER_TEMP/codexfold-windows-amd64.exe" ./cmd/codexfold + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -c -o "$RUNNER_TEMP/codexfold-testfs-linux.test" ./internal/testfs + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go test -c -o "$RUNNER_TEMP/codexfold-testfs-windows.test.exe" ./internal/testfs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..d83dda9 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,43 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +jobs: + release: + name: release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 + - uses: actions/setup-go@v7.0.0 + with: + go-version-file: go.mod + cache: true + - name: Require a versioned main-branch commit + shell: bash + run: | + git fetch origin main:refs/remotes/origin/main + release_commit=$(git rev-list -n 1 "$GITHUB_REF_NAME") + git merge-base --is-ancestor "$release_commit" origin/main + ./scripts/check-release.sh "$GITHUB_REF_NAME" + - name: Verify release source + run: | + go test ./... -count=1 + go vet ./... + - name: Publish archives and checksums + uses: goreleaser/goreleaser-action@v7.2.3 + with: + distribution: goreleaser + version: "~> v2" + args: >- + release --clean + --release-notes=docs/releases/${{ github.ref_name }}.md + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 9235d5a..45e3a19 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /codexfold +/codexfold.exe +/dist/ /*.sqlite /*.sqlite-shm /*.sqlite-wal @@ -6,3 +8,5 @@ *.test .DS_Store .worktrees/ +xcuserdata/ +*.xcuserstate diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..d295d3a --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,53 @@ +version: 2 + +project_name: codexfold + +builds: + - id: codexfold + main: ./cmd/codexfold + binary: codexfold + env: + - CGO_ENABLED=0 + goos: + - darwin + - linux + - windows + goarch: + - amd64 + - arm64 + flags: + - -trimpath + ldflags: + - -s -w -X github.com/samekind/codexfold/internal/cli.Version={{ .Version }} + +archives: + - id: codexfold + formats: + - tar.gz + format_overrides: + - goos: windows + formats: + - zip + name_template: >- + {{ .ProjectName }}_{{ .Version }}_{{ .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else }}{{ .Arch }}{{ end }} + files: + - LICENSE + - NOTICE + - README.md + - CHANGELOG.md + +checksum: + name_template: checksums.txt + +snapshot: + version_template: "{{ incpatch .Version }}-next" + +changelog: + disable: true + +release: + prerelease: auto + +report_sizes: true diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c2469bd --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,62 @@ +# Changelog + +All notable changes to CodexFold are recorded here. The project follows +[Semantic Versioning](https://semver.org/spec/v2.0.0.html) while it remains +pre-1.0, so a minor version may contain compatibility changes. + +## [Unreleased] + +## [0.3.0-beta.1] - 2026-07-23 + +### Added + +- Apple-native macOS FSKit frontend backed by a versioned Unix-domain-socket + protocol and the Go CodexFold daemon. +- Transparent packed-session reads, append deltas, copy-on-write fallback, + generation journals, namespace refresh, crash recovery, and compatibility + quarantine. +- Transactional FSKit App, helper, service-definition, and rollback updates. +- Real isolated Codex CLI and Desktop validation for resume, append, tool use, + fork, archive/unarchive, daemon restart, and host restart. +- Linux FUSE3 service and mount lifecycle validation, plus Windows WinFsp and + Windows Service compile coverage. +- Release archives and checksums for macOS, Linux, and Windows CLI builds. + +### Changed + +- The canonical Go module moved from `github.com/jstar0/codexfold` to + `github.com/samekind/codexfold`. Existing imports must use the new path. +- The filesystem engine remains opt-in and reports `fs-engine-preview`; the + default CLI build remains safe for storage analysis and recovery workflows. + +### Validation Boundary + +- macOS build 102 passed the isolated native mount matrix, exact-byte checks, + five independently restarted performance rounds, bounded-RSS checks, and + real current-client acceptance. +- This release does not claim production readiness. Native-source retention, + actual in-flight power-loss testing, the incident-free observation period, + and remaining platform-specific client gates are still required. +- The FSKit App is source-distributed in this release. The validated local App + uses an Apple Development identity and is not a generally distributable, + notarized binary. + +## [0.2.1] - 2026-07-18 + +- Added proof-first removal of archived sessions that are exact contiguous + subsets of another session, with transactional Codex state updates and + retained recovery evidence. + +## [0.2.0] - 2026-07-18 + +- Added guarded session-state maintenance and Windows durability follow-up. + +## [0.1.0] - 2026-07-18 + +- Initial local-first scan, fold, exact restore, and object-store release. + +[Unreleased]: https://github.com/samekind/codexfold/compare/v0.3.0-beta.1...HEAD +[0.3.0-beta.1]: https://github.com/samekind/codexfold/compare/v0.2.1...v0.3.0-beta.1 +[0.2.1]: https://github.com/samekind/codexfold/releases/tag/v0.2.1 +[0.2.0]: https://github.com/samekind/codexfold/releases/tag/v0.2.0 +[0.1.0]: https://github.com/samekind/codexfold/releases/tag/v0.1.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..7cec415 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,22 @@ +# Code of Conduct + +CodexFold contributors are expected to collaborate professionally, focus review on technical behavior and evidence, and respect the privacy and safety constraints of a project that handles local conversation data. + +## Expected Behavior + +- Be direct, respectful, and specific about technical concerns. +- Critique code, design, tests, and claims rather than people. +- Disclose uncertainty and distinguish observed evidence from inference. +- Protect private data and report sensitive findings through a private channel. +- Accept maintainer decisions on safety gates, even when they delay a feature or release. + +## Unacceptable Behavior + +- Harassment, personal attacks, discrimination, threats, or sustained disruption. +- Publishing private session content, credentials, personal information, or security details without authorization. +- Misrepresenting test coverage, readiness, provenance, or compatibility evidence. +- Pressuring contributors to bypass review, rollback, privacy, or production-safety requirements. + +## Enforcement + +Maintainers may edit or remove contributions, comments, or access that violate this policy. Report sensitive conduct issues privately to the repository owner through GitHub; use a private Security Advisory when the report also involves confidential data or a vulnerability. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ddecea8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,78 @@ +# Contributing to CodexFold + +CodexFold handles private local session data and implements storage and filesystem behavior where a silent mismatch is unacceptable. Contributions should be small enough to review, explicit about safety boundaries, and backed by evidence that matches the claim. + +## Before You Start + +- Read the [product contract](docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md), the [implementation alignment](docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md), and the [maintainer guide](docs/maintainer-guide.md). +- Open an issue for product behavior, format, compatibility, or architecture changes before implementing a large change. +- Never use a real rollout, state database, credential, signing export, or private prompt as a fixture. +- Do not weaken a release gate to make an implementation pass. + +## Development Setup + +Requirements: + +- Go version declared in `go.mod`. +- Git. +- Xcode 27 and XcodeGen for native macOS FSKit work. +- Platform prerequisites only when testing the corresponding preview adapter. + +Run the common quality gate: + +```bash +./scripts/test-cross-platform.sh +./scripts/check-release.sh +git diff --check +``` + +For a focused change, run the smallest relevant package tests first, then the common gate before requesting review. + +Windows currently has compile and cross-build gates only. Do not describe those checks as real WinFsp or Windows Service runtime validation. + +## Branches and Commits + +- Create a branch from current `main`; use a descriptive prefix such as `feat/`, `fix/`, `docs/`, or `test/`. +- Use conventional, imperative commit subjects such as `fix: reject stale native writer state`. +- Keep generated files, binaries, user-specific Xcode state, local databases, and test artifacts out of commits. +- Update `platform/darwin/fskit/project.yml` first and regenerate the Xcode project; do not hand-maintain personal Xcode state. + +## Pull Requests + +- Explain the user-visible outcome, safety impact, and exact evidence. +- Distinguish unit, synthetic, mounted-adapter, real-client, host-restart, and production evidence. One category never substitutes for another. +- Add or update tests before changing a status or readiness claim. +- Keep production Codex homes and production service definitions untouched during development validation. +- Resolve review conversations and keep the branch current with `main` before merge. + +## Versions and Releases + +- Update `VERSION`, `CHANGELOG.md`, the matching file under `docs/releases`, and + the FSKit marketing/build versions in one release pull request. +- Run `scripts/check-release.sh` before tagging. A release tag must be exactly + `v$(cat VERSION)` and must point to a commit contained by `main`. +- Release CLI archives are generated from `.goreleaser.yaml`; do not commit + locally built archives, checksums, Apps, or provisioning profiles. +- Do not attach an Apple Development-signed FSKit App to a public release. A + public App asset requires Developer ID distribution signing, notarization, + and a separate installation test on a clean machine. + +## Filesystem Changes + +Native filesystem work must remain isolated until every relevant gate passes. Use disposable homes, stores, native roots, mount points, and service labels. A test may not route a production SQLite record or remove a production JSONL source. + +The macOS terminal architecture is: + +```text +Codex CLI / Desktop + -> Apple-native Swift FSKit extension + -> versioned binary UDS IPC + -> Go CodexFold daemon + -> packfile + index + manifest + append delta + COW backing +``` + +NFS and FUSE-T are development fallback or historical evidence only. They are not acceptable replacements for native FSKit release gates. + +## Security + +Report vulnerabilities through GitHub Security Advisories. Public issues and pull requests must contain only synthetic or fully redacted data; see [SECURITY.md](SECURITY.md). diff --git a/README.md b/README.md index 7fa8ba4..e56b578 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # CodexFold +[![CI](https://github.com/samekind/codexfold/actions/workflows/ci.yml/badge.svg)](https://github.com/samekind/codexfold/actions/workflows/ci.yml) +[![Release](https://img.shields.io/github/v/release/samekind/codexfold?include_prereleases)](https://github.com/samekind/codexfold/releases) +[![License](https://img.shields.io/github/license/samekind/codexfold)](LICENSE) + CodexFold is an unofficial, local-first tool for measuring, deduplicating, storing, and restoring Codex session rollouts. It finds exact duplicate raw JSON string tokens, complete JSONL records, and content-defined chunks. Folded rollouts use a shared SHA-256/zstd object store and a versioned manifest. Every restore must match the original byte count and SHA-256. @@ -8,16 +12,40 @@ It finds exact duplicate raw JSON string tokens, complete JSONL records, and con ## Current Status -`v0.2.1` is `storage-engine`: exact deduplicated storage, byte-identical recovery, incremental analysis, containment, and guarded removal are available. It is not a transparent virtual-filesystem release. Codex cannot directly open a folded manifest without materialization in this version. +`v0.3.0-beta.1` is the first versioned `fs-engine-preview`. It preserves the released storage-engine commands from `v0.2.1` and adds the transparent filesystem implementation, guarded service lifecycle, and platform validation described below. The requirements and release gates for normal JSONL paths backed transparently by shared storage are defined in [the transparent filesystem product contract](docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md). No release may claim `随点随开`, transparent session access, or production-ready virtual sessions before the platform-specific gates in that contract pass. +| Platform | Adapter | Evidence in this release | Readiness | +|---|---|---|---| +| macOS 27 | Apple-native Swift FSKit | Real isolated CLI/Desktop, native mount, restart, recovery, performance, and exact-byte Canary | `fs-engine-preview` | +| Linux | FUSE3 | Real unprivileged mount, mutation, remount, recovery, performance, and user-service lifecycle | Preview; no real Codex client gate yet | +| Windows | WinFsp | Cross-build and compile coverage | Not runtime-validated | + +The transparent filesystem preview has these explicit boundaries: + +- macOS now targets an Apple-native Swift FSKit extension connected over a versioned Unix-domain-socket protocol to the Go CodexFold daemon. The signed build 102 App/extension and current helper candidate pass the isolated mounted operation matrix, exact-byte and cache-coherency gates, independently restarted cold/warm `F_NOCACHE` performance rounds, bounded runtime RSS, crash and host-restart recovery, transactional app/binary rollback, exact current-client compatibility contracts, and real Codex CLI/Desktop acceptance. +- Release source metadata is `0.3.0 (103)`. Build 103 compiles and passes nested signature verification; the complete mounted and real-client evidence remains attached to behavior-identical build 102 rather than being silently relabeled. +- The earlier synchronous FUSE-T NFS route remains historical validation evidence and a development fallback only. FUSE-T's third-party FSKit backend remains rejected after deterministic byte-loss and cache-invalidation failures; it is not the Apple-native FSKit implementation in this repository. +- Linux FUSE3 has real unprivileged read, append, copy-on-write, truncate, archive rename, crash recovery, remount, performance, and `systemd --user` lifecycle evidence. +- Windows has a WinFsp adapter and native Windows Service host that cross-compile, but no real Windows/WinFsp host has validated them yet. +- The production service and production Codex home remain disabled. Retention, actual in-flight power loss, the incident-free observation gate, and the remaining platform-specific client gates still block promotion. + +See [the Linux FUSE3 validation](docs/validation-linux-fuse3.md) and [the macOS canary validation](docs/validation-macos-canary.md) for the evidence boundary. The default build remains storage-only; platform mounts require explicit build tags and installed host prerequisites. + ## Install +Install the versioned preview with Go: + ```bash -go install github.com/jstar0/codexfold/cmd/codexfold@latest +go install github.com/samekind/codexfold/cmd/codexfold@v0.3.0-beta.1 +codexfold --version ``` +The [GitHub Release](https://github.com/samekind/codexfold/releases/tag/v0.3.0-beta.1) provides checksum-covered default CLI archives for macOS, Linux, and Windows on `amd64` and `arm64`. These archives expose the local storage and recovery command surface; they do not contain a generally signed macOS FSKit App. + +The FSKit App under `platform/darwin/fskit` currently requires Xcode 27, XcodeGen, an eligible Apple development team, and source signing. The validated App uses a maintainer Apple Development identity and is neither Developer ID distributed nor notarized for general installation. Follow the [maintainer guide](docs/maintainer-guide.md) and use only an isolated Codex home until the product contract permits production promotion. + ## Analyze Scan selected sessions or the complete Codex home: @@ -81,6 +109,27 @@ codexfold remove-contained --apply The first command is proof-only. `--apply` additionally requires an existing verified fold, a current source SHA-256 match, and a successful temporary unfold. It then isolates the source file, removes the archived thread and associated local state in one SQLite transaction, cleans exact thread-ID references from Codex global state, and finally deletes the isolated source. A tombstone and fold manifest remain for byte-level recovery. Concurrent global-state changes abort the operation instead of being overwritten. +## Fork Families And Archival + +Inspect the explicit Codex spawn graph and compare two selected rollouts without mutation: + +```bash +codexfold fork-family show +codexfold fork-family compare +``` + +The report keeps graph ancestry separate from exact content evidence. It can identify identical applicable records, complete containment, shared prefixes with independent tails, other exact shared records, or an unknown relationship. It never labels a branch useless from ancestry, age, title, or size. + +Preview and explicitly archive one active session: + +```bash +codexfold archive +codexfold archive --apply +codexfold archive recover --apply +``` + +Archive is dry-run-first and preserves the rollout bytes. Apply requires the native writer probe, revalidates the selected SQLite route and complete source SHA-256, moves the rollout to Codex's flat `archived_sessions` path, and updates the official archive fields in one guarded transaction. A durable journal supports deterministic recovery if file and database commit acknowledgement are interrupted. Archive never deletes a session; exact-contained deletion remains the separate archived-only `remove-contained` operation. + ## Maintenance Verify every manifest and referenced object: @@ -107,14 +156,15 @@ codexfold gc --apply - Restore writes to a temporary file, verifies the complete SHA-256, then atomically replaces the target. - Existing indexes, manifests, and restore targets are never replaced without an explicit overwrite flag. - Contained-session removal is archived-only, proof-first, transaction-guarded, and retains recovery evidence. +- Fork-family reporting is evidence-only, archive is explicit and recoverable, and neither operation triggers deletion. ## Development ```bash -go test ./... -count=1 -go test -race ./... -count=1 -go vet ./... -go build ./cmd/codexfold +./scripts/test-cross-platform.sh +./scripts/test-native-fskit-cache.sh # macOS FSKit changes +./scripts/check-release.sh +git diff --check ``` -See [the architecture](docs/design.md), [Fold V1 format](docs/fold-v1.md), [v0.2 validation](docs/validation-v0.2.md), and [the transparent filesystem product contract](docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md). +See the [changelog](CHANGELOG.md), [v0.3.0-beta.1 release notes](docs/releases/v0.3.0-beta.1.md), [architecture](docs/design.md), [Fold V1 format](docs/fold-v1.md), [v0.2 validation](docs/validation-v0.2.md), [transparent filesystem product contract](docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md), and [maintainer guide](docs/maintainer-guide.md). diff --git a/SECURITY.md b/SECURITY.md index 415bb02..696b35b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,4 +2,16 @@ CodexFold processes local conversation rollouts that may contain secrets or private data. The project does not transmit scan inputs or report field contents. -Please report vulnerabilities privately through GitHub Security Advisories. Do not include real session files, credentials, or private prompts in public issues. +## Supported Code + +Security fixes target the latest release and the current `main` branch. Preview filesystem branches may change quickly and must not be treated as production-safe unless the repository explicitly publishes a platform readiness claim. + +## Reporting + +Please report vulnerabilities privately through GitHub Security Advisories. Do not include real session files, credentials, private prompts, local filesystem paths, service tokens, or unredacted logs in public issues. + +Include the affected version or commit, operating system, impact, and a minimal synthetic reproducer when possible. Maintainers will acknowledge the report, assess whether private coordination is required, and publish remediation details after a safe fix is available. + +## Data Handling + +Tests and bug reports must use generated or redacted fixtures. A contributor must never commit a real Codex rollout, Codex state database, credential, signing identity export, provisioning profile, or production service definition. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..222c5f8 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.3.0-beta.1 diff --git a/cmd/codexfold/main.go b/cmd/codexfold/main.go index 9115fb3..689eb27 100644 --- a/cmd/codexfold/main.go +++ b/cmd/codexfold/main.go @@ -5,13 +5,21 @@ import ( "fmt" "os" "os/signal" + "syscall" - "github.com/jstar0/codexfold/internal/cli" + "github.com/samekind/codexfold/internal/cli" + "github.com/samekind/codexfold/internal/launcher" ) func main() { - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() + ctx, stopLauncher, err := launcher.MonitorContext(ctx) + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } + defer stopLauncher() command := cli.NewRootCommand() command.SetContext(ctx) if err := command.Execute(); err != nil { diff --git a/docs/maintainer-guide.md b/docs/maintainer-guide.md new file mode 100644 index 0000000..a6e2e32 --- /dev/null +++ b/docs/maintainer-guide.md @@ -0,0 +1,94 @@ +# Maintainer Guide + +## Source of Truth + +Use this order when documents or code appear to disagree: + +1. The transparent filesystem product contract. +2. The implementation alignment document. +3. Current platform validation reports. +4. Current code and tests. +5. Historical plans and validation evidence. + +Changing architecture, safety guarantees, or readiness language requires updating the contract and alignment in the same pull request. + +## Current Architecture and Status + +The storage engine is released separately from the transparent filesystem preview. The macOS terminal candidate is Apple-native Swift FSKit -> versioned UDS -> Go daemon. Linux uses FUSE3 and Windows targets WinFsp. Production Codex routing remains disabled until the named platform gates pass. + +FUSE-T NFS evidence is retained to preserve regression knowledge. FUSE-T's own FSKit backend is rejected and must not be confused with the native Swift extension in `platform/darwin/fskit`. + +## Required Pull Request Gates + +Every pull request must pass: + +```bash +gofmt -l . +go mod tidy +git diff --exit-code -- go.mod go.sum +go test ./... -count=1 +go test -race ./... -count=1 +go vet ./... +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /tmp/codexfold-linux-amd64 ./cmd/codexfold +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o /tmp/codexfold-windows-amd64.exe ./cmd/codexfold +git diff --check +``` + +Native macOS FSKit changes additionally require an XcodeGen consistency check and Release build with the Xcode version declared by the project. Commit `project.yml` and the regenerated project together. + +Run `scripts/test-native-fskit-cache.sh` with the repository's Xcode toolchain. It compiles and executes the descriptor/read-ahead/cache lifecycle tests; a successful app build alone does not count as cache evidence. + +FSKit wire-protocol changes must remain capability-negotiated. A native descriptor may be sent only for a read-only handle, every received descriptor must be closed on all success and rejection paths, and unsupported peers must continue through the bounded byte-stream path. Run the descriptor-lifecycle, cache-invalidation, and mounted random-read suites together; none is a substitute for the others. + +Never use `pluginkit -r` for an installed FSKit module during an update. Apple keeps module-election state in the login session, and deregistration can leave an otherwise enabled module unavailable until the next login. Preserve the installed App bundle root, atomically swap `Contents`, register the target with LaunchServices, and use `lsregister -u` only to remove disposable candidate App registrations. + +Windows CI compiles every package and test binary but does not execute the full runtime suite. That is intentional until a real Windows/WinFsp host validates directory durability, locking, service, mount, and file-sharing semantics. A green Windows compile check is not Windows readiness evidence. + +## Isolated Native FSKit Validation + +Never point development tests at `~/.codex`. Create a disposable Codex home, store, native root, mount point, service label, and service definition. Production `com.codexfold.fs` must remain disabled during preview work. + +Mounted behavior tests use explicit paths: + +```bash +CODEXFOLD_NATIVE_FSKIT_MOUNT=/absolute/disposable/mount \ +CODEXFOLD_NATIVE_FSKIT_NATIVE_ROOT=/absolute/disposable/native \ +go test ./internal/mountfs -run '^TestNativeFSKitMounted' -count=1 -v +``` + +An app or binary update must use the transactional service command. The updater must stop both launchd jobs, wait for daemon and supervisor process locks to release, install staged definitions/app/binary, verify Host-child ancestry, mount health, and running build SHA, and restore the previous generation on failure. Do not replace an active extension bundle manually. + +Current-client approval requires a sanitized operation trace for the exact CLI or Desktop version. Normalize Darwin syscall spellings before contract evaluation, import the resulting operation set into the disposable store, and require both `fs compatibility` approval and a zero-issue `fs doctor` result before a real-client canary. A parser fixture or a contract from a different version is not current-client evidence. + +An isolated Codex Desktop process requires both `CODEX_ELECTRON_USER_DATA_PATH` and an explicit `--user-data-dir` argument. The environment variable isolates Codex state, while the Chromium argument prevents the disposable process from joining the production singleton. When a production Desktop is already running, launch the copy through LaunchServices with `open -n`; a direct executable launch may exit at the application-level singleton before Chromium applies its data-directory argument. Pass the isolated environment through launchd only for the launch window, clear it immediately afterward, and verify the child app-server's `CODEX_HOME`, Electron data path, and process ancestry before treating any Desktop action as canary evidence. + +## Evidence Levels + +- Unit or fixture tests prove only the code path they exercise. +- Mounted tests prove adapter behavior against disposable data. +- Real CLI/Desktop tests prove current-client behavior only for the exact tested versions. +- Restart and crash matrices prove process recovery, not power-loss durability. +- A successful canary does not satisfy retention or production readiness. + +Readiness claims must use only the capability names defined in the product contract. + +## Data and Artifact Hygiene + +- Keep all test rollouts synthetic and valid JSONL when they use a `.jsonl` suffix. +- Use `.bin` for arbitrary filesystem mutation fixtures so native writer preflight cannot mistake them for Codex rollouts. +- Clean disposable mount and native-backing paths even when the mount disappears during a test. +- Do not commit DerivedData, built apps, binaries, databases, logs, `xcuserdata`, or provisioning profiles. +- Do not print inherited launchd environments or credentials in public logs. + +## Merge and Release Procedure + +1. Update `VERSION`, `CHANGELOG.md`, `docs/releases/v.md`, and the FSKit marketing/build versions together. +2. Run `scripts/check-release.sh`, `scripts/test-cross-platform.sh`, the Swift cache tests, XcodeGen consistency, and the applicable mounted adapter suites. +3. Obtain an approving review and green required checks. +4. Merge into `main`; a release tag may not point to a branch-only commit. +5. Create the annotated `v` tag from the verified `main` commit. The release workflow builds default CLI archives, injects the tag into `codexfold --version`, produces `checksums.txt`, and uses the checked-in release notes. +6. Verify every uploaded archive against `checksums.txt` and execute at least one native release binary before publishing the release as non-draft. +7. Release notes must distinguish implemented, tested, preview, canary, and production-ready behavior. Never attach a maintainer Apple Development-signed FSKit App as a generally installable asset. +8. Never delete retained native sources or enable bulk enrollment before the contract permits it. + +If a release or service update fails, preserve the failing evidence, restore the last verified app/binary/definition generation, verify exact bytes and build identity, and keep automatic enrollment disabled until the incident is understood. diff --git a/docs/releases/v0.3.0-beta.1.md b/docs/releases/v0.3.0-beta.1.md new file mode 100644 index 0000000..a6fc246 --- /dev/null +++ b/docs/releases/v0.3.0-beta.1.md @@ -0,0 +1,82 @@ +# CodexFold v0.3.0-beta.1 + +This release turns the transparent session filesystem work into a versioned, +reviewable preview. The storage engine remains local-first and byte-exact. On +macOS, the terminal architecture is an Apple-native Swift FSKit extension +connected to the Go daemon over a capability-negotiated Unix-domain-socket +protocol. + +## Highlights + +- Transparent JSONL paths backed by packfiles, indexes, manifests, and append + deltas. +- Copy-on-write backing for non-append mutation and unknown safe write paths. +- Transactional App, helper, launchd-definition, mount, and rollback updates. +- Current-client compatibility contracts and fail-closed quarantine. +- Native namespace refresh, descriptor streaming, bounded concurrent + read-ahead, generation invalidation, and stale-size repair. +- Recoverable archive/unarchive and fork behavior without sharing writable + tails between parent and child sessions. + +## macOS Canary Evidence + +The signed local build 102 candidate passed an isolated Apple-native FSKit +Canary without routing the production Codex home or production SQLite state. +Release source metadata is `0.3.0 (103)`: build 103 completed a signed Release +build and nested Host/extension signature verification. Its FSKit behavior is +unchanged from build 102, so the mounted and real-client evidence below remains +attributed to the exact candidate that produced it. + +- Real bundled Codex CLI `0.145.0-alpha.30` resume, history use, tool execution, + and append passed. +- Real Codex Desktop `26.715.72359+5718` resume, send, fork, child continuation, + archive/unarchive, and restart passed. +- Parent and child histories remained valid JSONL and byte-identical to their + expected native or reconstructed forms. +- All 11 `fs doctor` components were healthy after the final host restart. +- Five independently restarted 256 MiB managed-file matrices passed the fixed + cold-read ratio gate of `0.70` and warm-read ratio gate of `0.80`. +- Accepted cold ratios were `0.762`, `2.042`, `0.800`, `1.761`, and `0.941`; + warm ratios were `0.954`, `1.011`, `0.963`, `0.993`, and `0.981`. +- Post-matrix aggregate RSS was `164,688 KiB`, below the `256 MiB` bound. +- A post-restart sample measured managed cold and warm reads at `5.96 GiB/s` + and `17.73 GiB/s`; FSKit append plus `fsync` measured `5.21 ms` p50, + `9.51 ms` p95, and `19.71 ms` p99. These are observations, not guaranteed + throughput on other machines. + +## Install the CLI + +This prerelease changes the canonical module path. Install it explicitly: + +```bash +go install github.com/samekind/codexfold/cmd/codexfold@v0.3.0-beta.1 +codexfold --version +``` + +GitHub Release archives contain the default storage-oriented CLI for macOS, +Linux, and Windows on `amd64` and `arm64`, plus `checksums.txt`. + +## FSKit Source Build + +The FSKit App is intentionally not attached as a public binary. The validated +candidate is signed with a maintainer Apple Development identity, not a +Developer ID distribution identity, and is not notarized for general use. +Build the App from `platform/darwin/fskit` with Xcode 27, XcodeGen, an eligible +Apple development team, and isolated test paths. Follow the maintainer guide +and macOS Canary report before enabling any route. + +## Readiness Boundary + +This is a preview release, not permission to delete native session sources or +bulk-enroll a production Codex home. Production promotion still requires: + +- retained native sources and verified rollback throughout the Canary period; +- actual in-flight power-loss testing; +- the incident-free observation gate; +- current-client compatibility approval after each Codex upgrade; +- real runtime validation before Linux or Windows receives a production-ready + capability claim. + +The complete evidence and limitations are documented in +`docs/validation-macos-canary.md`, `docs/validation-linux-fuse3.md`, and the +transparent filesystem product contract. diff --git a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md index 5dab6d7..fc0fcff 100644 --- a/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md +++ b/docs/superpowers/plans/2026-07-11-transparent-session-filesystem-implementation.md @@ -1,19 +1,43 @@ # Transparent Codex Session Filesystem Implementation Plan -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> Execute this plan task-by-task. Checkboxes describe current repository completion, not intended future work; mark a step complete only when code and fresh evidence support it. **Goal:** Make unmodified Codex Desktop and Codex CLI open, resume, fork, and continue managed JSONL sessions directly while exact duplicate bytes are stored once and current session writes remain durable and independently recoverable. **Architecture:** Extend Fold V1 with a block-addressable packed resolver, then place a platform-neutral exact-byte session engine above it. The engine composes an immutable manifest base with an append delta or verified writable backing; platform adapters only translate native file operations. Migration, compatibility quarantine, fallback, and promotion remain explicit journaled transactions, with real Codex routing disabled until shadow and platform gates pass. -**Tech Stack:** Go 1.26, zstd, Cobra, modernc SQLite, `cgofuse` v1.6.0 behind platform/build tags, macFUSE as the initial macOS adapter candidate. +**Tech Stack:** Go 1.26, zstd, Cobra, modernc SQLite, an Apple-native Swift FSKit extension with versioned UDS IPC on macOS, FUSE3 on Linux, and WinFsp plus Windows SCM on Windows. FUSE-T remains historical validation evidence and a development fallback, not the terminal macOS architecture. + +> Architecture update, 2026-07-18: the terminal macOS route is the Apple-native Swift FSKit extension -> versioned binary UDS -> Go daemon. Earlier FUSE-T task evidence remains useful regression history but does not authorize reverting the product architecture or claiming native FSKit readiness. + +## Alignment Snapshot + +Current public status remains `fs-engine-preview`. Tasks 1 through 10 and 12 through 14 are implemented. Task 11 has substantial isolated and bounded real-home macOS evidence, including current-client compatibility, sleep/wake, process-interruption recovery, and one idle retained-source managed CLI session surviving an actual host reboot. Linux FUSE3 now has real unprivileged operation, crash/restart, performance, mount-policy, and `systemd --user` lifecycle evidence. Windows WinFsp and SCM support are implemented and cross-compile, but lack a real Windows host. Actual in-flight power loss, the dedicated retention window, seven incident-free days, Linux client/upgrade/rollback/retention gates, and all real Windows gates remain open. + +| Task | Status | Current evidence | Remaining work | +| --- | --- | --- | --- | +| 1 | Complete | Commit `17564e9`; `internal/pack` tests | None in this task | +| 2 | Complete | Commit `039e6b9`; exact and 10,000-range view tests | None in this task | +| 3 | Complete | Commit `9a7f1e8`; append and COW tests | None in this task | +| 4 | Complete | Commit `076d772`; journal, compaction, and fallback tests | None in this task | +| 5 | Complete | Commit `35a53fc`; status, shadow, doctor, and benchmark tests | Platform evidence remains outside this task | +| 6 | Complete | Commit `5be10d2`; exact-version compatibility and optimistic route tests | New installed client versions still require fresh contracts | +| 7 | Complete | Commit `3f51aa5`; neutral operations, real macOS FUSE-T, and real Linux FUSE3 adapter tests | Windows real-adapter evidence remains a separate product gate | +| 8 | Complete | Standalone CLI, guarded lifecycle, bounded planner/apply loop, and isolated automatic-enrollment evidence | Production enablement remains gated by platform readiness | +| 9 | Complete | Commit `4589ffa`; launchd, real `systemd --user`, Windows SCM compile, and update preflight tests | Windows service runtime and production update promotion remain platform-gated | +| 10 | Complete | Commit `a1ac76e`; synthetic, crash, race, cross-compile, and 758 MiB evidence | This task proves only the shared engine preview | +| 11 | Partial | Real macOS CLI/Desktop, FUSE-T, rollback, daemon restart, idle managed-session host reboot, and quarantine evidence | Complete the remaining disruptive and retention gates | +| 12 | Complete | Bounded planner/apply/service tests plus isolated canonical automatic enrollment, native-writer probing, restart, append, quarantine, and failed-cutover evidence | Real-home automatic apply remains promotion-gated | +| 13 | Complete | Fork graph reports, exact content comparison, guarded official-compatible archive transactions, recovery, static content-change boundaries, and isolated native plus managed FUSE-T round trips | None in this task | +| 14 | Complete | Physical inventory, hard mutation budgets, lease-aware bounded GC, and truthful projected/actual accounting | Destructive retention remains promotion-gated | +| 15 | Partial | Current macOS client contracts plus restart gates; real Linux FUSE3 operation, crash, performance, and systemd lifecycle; Windows WinFsp/SCM cross-compile | Actual in-flight power loss, retention windows, Linux client/upgrade/rollback gates, and all real Windows gates remain | ## Global Constraints - `TF-001`: normal open and resume require no manual `materialize` or preparation command. - `TF-002`: Codex Desktop and CLI remain unmodified and access normal regular-file JSONL paths. - `TF-003`: exact bytes and every operation observed in native Codex traces must have native-equivalent behavior. -- `TF-004`: identical bytes across sessions and forks are stored once while histories remain independently writable. +- `TF-004`: exact repeated fields, records, and content-defined chunks at arbitrary positions are stored once across sessions and forks while histories remain independently writable; strict prefix sharing is not required. - `TF-005`: append writes persist to a delta without complete base hydration. - `TF-006`: truncate, random write, and other representable mutations transition to complete copy-on-write before success. - `TF-007`: packed reads perform neither per-part loose-object opens nor per-object persistent-index queries. @@ -25,7 +49,13 @@ - `TF-013`: status output uses only `storage-engine`, `fs-engine-preview`, `platform-canary`, `production-ready:`, and `cross-platform-ready`. - `TF-014`: a migration snapshot is not deleted before `production-ready:` and the per-session retention gate. - `TF-015`: unknown client versions enter compatibility quarantine and cannot write a virtual route before current bytes are automatically routed to verified native backing. -- `TF-016`: macFUSE or another privileged prerequisite is not installed without explicit user authorization. +- `TF-016`: FUSE-T or another privileged prerequisite is not installed without explicit user authorization. +- `TF-017`: canonical namespace activation requires a verified CodexFold mount identity, write-sealed unmounted backing, route normalization, and a watcher that tolerates canonical and mount-alias spellings. +- `TF-018`: branch classification and archival are conservative, proof-first, explicitly selected, and recoverable. +- `TF-019`: fully contained archived-session deletion requires exact containment and recovery proof. +- `TF-020`: byte-preserving optimization never invokes content-changing repair, reconciliation, or prompt cleanup implicitly. +- `TF-021`: full-size copies, retained generations, temporary artifacts, and savings claims obey hard physical-space budgets and accounting. +- `TF-022`: the public product has no private control-plane runtime or documentation dependency. - Mock and fixture evidence never satisfies a gate that names real Codex, a real adapter, client upgrade, host restart, or canary retention. - Public code and documentation contain no private paths, domains, credentials, real session IDs, or private control-plane dependency. @@ -49,6 +79,12 @@ | `TF-014` | 4, 6 | 10, 11 | | `TF-015` | 4, 6 | 10, 11 | | `TF-016` | 7, 9 | 11 | +| `TF-017` | 7, 9 | 10, 11 | +| `TF-018` | 13 | 13 | +| `TF-019` | Storage-engine baseline, 13 | `internal/contain`, `internal/prune`, and Task 13 boundary tests | +| `TF-020` | Storage-engine baseline, 13 | `internal/reconcile` and CLI boundary tests | +| `TF-021` | 12, 14 | 14, 15 | +| `TF-022` | All public tasks | 10 and public sanitization checks | --- @@ -67,17 +103,17 @@ - Consumes: Fold V1 `fold.Manifest`, `fold.ObjectRef`, and loose zstd objects. - Produces: `pack.Build(ctx, storeDir, BuildOptions) (BuildResult, error)`, `pack.Open(storeDir, OpenOptions) (*Resolver, error)`, and `(*Resolver).ReadAt(ctx, ref, dst, offset) (int, error)`. -- [ ] **Step 1: Write failing pack round-trip, corruption, transaction, and random-read tests** +- [x] **Step 1: Write failing pack round-trip, corruption, transaction, and random-read tests** Tests construct repeated small objects and one object larger than two 256 KiB blocks, build a generation with a 1 MiB pack limit, remove a copied loose-object fixture, and assert all offset/length reads match source bytes. They also corrupt a block, interrupt before `CURRENT` publication, and assert the previous generation remains readable. -- [ ] **Step 2: Run the focused tests and confirm missing package/API failures** +- [x] **Step 2: Run the focused tests and confirm missing package/API failures** Run: `go test ./internal/pack -count=1` Expected: FAIL because `internal/pack` and its exported API do not exist. -- [ ] **Step 3: Implement the candidate packed format and transactional builder** +- [x] **Step 3: Implement the candidate packed format and transactional builder** Use this public shape: @@ -106,15 +142,15 @@ type Object struct { Decode each loose object as a stream, split decoded bytes into independently compressed 256 KiB blocks, hash both object and blocks, and write bounded immutable pack files. Publish a verified generation directory and atomically replace `packs/CURRENT`; never encode physical locations into Fold V1 manifests. -- [ ] **Step 4: Implement in-memory resolver lookup and bounded block cache** +- [x] **Step 4: Implement in-memory resolver lookup and bounded block cache** Load the candidate JSON index once at `Open`, keep `map[string]Object`, use `ReadAt` on pack files, verify block checksum and decoded size, and cache decompressed blocks under a byte budget. A runtime read must not open a loose object or query SQLite. -- [ ] **Step 5: Implement pack doctor and loose-object streaming helper** +- [x] **Step 5: Implement pack doctor and loose-object streaming helper** Export a streaming loose-object reader from `internal/fold` without changing Fold V1 paths. Doctor verifies `CURRENT`, every indexed extent, block digest, raw length, object digest, and every manifest reference. -- [ ] **Step 6: Run focused, race, and complete tests** +- [x] **Step 6: Run focused, race, and complete tests** Run: @@ -126,7 +162,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 7: Commit** +- [x] **Step 7: Commit** ```bash git add internal/pack internal/fold/store.go @@ -144,17 +180,17 @@ git commit -m "feat: add transactional packed object resolver" - Consumes: `fold.Manifest` and an `ObjectReader` implemented by `pack.Resolver`. - Produces: `vfs.NewView(manifest, reader) (*View, error)`, `(*View).Size() int64`, and `(*View).ReadAt(ctx, dst, offset) (int, error)`. -- [ ] **Step 1: Write failing exact-read tests** +- [x] **Step 1: Write failing exact-read tests** Cover empty reads, EOF, final partial reads, random offsets, cross-part boundaries, a read spanning more than two parts, and 10,000 deterministic random comparisons against a native byte slice. -- [ ] **Step 2: Run the focused test and verify the API is absent** +- [x] **Step 2: Run the focused test and verify the API is absent** Run: `go test ./internal/vfs -run 'TestView' -count=1` Expected: FAIL because `View` is undefined. -- [ ] **Step 3: Implement cumulative offsets and binary-search reads** +- [x] **Step 3: Implement cumulative offsets and binary-search reads** ```go type ObjectReader interface { @@ -170,7 +206,7 @@ type View struct { Validate total part bytes against `manifest.Source.Bytes`, locate the first part with `sort.Search`, and fill the destination across parts without materializing the session. Match `io.ReaderAt` EOF semantics exactly. -- [ ] **Step 4: Run focused, race, and complete tests** +- [x] **Step 4: Run focused, race, and complete tests** Run: @@ -182,7 +218,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 5: Commit** +- [x] **Step 5: Commit** ```bash git add internal/vfs @@ -201,17 +237,17 @@ git commit -m "feat: add exact virtual rollout byte view" - Consumes: immutable `View`, session state directory, and native snapshot metadata. - Produces: `vfs.OpenSession(ctx, Options) (*Session, error)`, reader/writer handles, `Append`, `WriteAt`, `Truncate`, `Sync`, and `MaterializeCurrent`. -- [ ] **Step 1: Write failing append and COW state-machine tests** +- [x] **Step 1: Write failing append and COW state-machine tests** Assert append immediately extends visible bytes, `fsync` persists after reopen, one writer lease is enforced, readers retain generation snapshots, random write and truncate create byte-verified native backing before mutation, and interrupted COW leaves the old generation readable. -- [ ] **Step 2: Run focused tests and verify expected missing API failures** +- [x] **Step 2: Run focused tests and verify expected missing API failures** Run: `go test ./internal/vfs -run 'TestSession|TestAppend|TestCopyOnWrite' -count=1` Expected: FAIL because writable session APIs are undefined. -- [ ] **Step 3: Implement atomic session state and generation leases** +- [x] **Step 3: Implement atomic session state and generation leases** ```go type SessionState struct { @@ -228,15 +264,15 @@ type SessionState struct { Commit state through a synchronized temporary file and atomic replacement. Readers pin a generation. Writers acquire a process-local and on-disk lease and never trigger compaction from `Release`. -- [ ] **Step 4: Implement append fast path** +- [x] **Step 4: Implement append fast path** Append accepted bytes to a normal delta opened with `O_APPEND`, return success only for written bytes, synchronize on `Sync`, and compose reads as `base || delta`. -- [ ] **Step 5: Implement safe COW transition** +- [x] **Step 5: Implement safe COW transition** Freeze new writers, stream current visible bytes to a temporary native backing, verify byte count and SHA-256, atomically publish backing state, then apply `WriteAt` or `Truncate`. Never mutate packs or manifests in place. -- [ ] **Step 6: Run focused, crash-reopen, race, and complete tests** +- [x] **Step 6: Run focused, crash-reopen, race, and complete tests** Run: @@ -248,7 +284,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 7: Commit** +- [x] **Step 7: Commit** ```bash git add internal/vfs @@ -268,29 +304,29 @@ git commit -m "feat: add durable append and copy-on-write sessions" - Consumes: writable `Session`, Fold V1 writer, and atomic state commits. - Produces: `Recover`, `Compact`, `CreateCurrentNativeBacking`, and deterministic journal phase records. -- [ ] **Step 1: Write failing phase-interruption tests** +- [x] **Step 1: Write failing phase-interruption tests** Inject termination-equivalent errors before and after every prepare, data sync, state publish, route-ready, and cleanup phase. Reopen and assert exact bytes, one active generation, retained old readers, and idempotent recovery. -- [ ] **Step 2: Verify the recovery tests fail for missing APIs** +- [x] **Step 2: Verify the recovery tests fail for missing APIs** Run: `go test ./internal/vfs -run 'TestRecover|TestCompact|TestFallback' -count=1` Expected: FAIL because journal operations do not exist. -- [ ] **Step 3: Implement append-only journal and recovery dispatcher** +- [x] **Step 3: Implement append-only journal and recovery dispatcher** Journal records contain operation ID, session ID, operation kind, phase, source generation, candidate generation, paths, byte counts, and digests, but never session content. Synchronize phase records before the represented state change. -- [ ] **Step 4: Implement idle compaction with optimistic revalidation** +- [x] **Step 4: Implement idle compaction with optimistic revalidation** Require zero writers, no writer lease, elapsed idle window, and unchanged delta size, mtime, and digest. Fold a current native stream into a new immutable manifest generation, verify the complete virtual SHA-256, atomically switch state, and retain old files until generation leases close. -- [ ] **Step 5: Implement latest-byte native fallback** +- [x] **Step 5: Implement latest-byte native fallback** `CreateCurrentNativeBacking` freezes writers, streams the latest committed virtual bytes, verifies exact digest, publishes a normal JSONL, and records it separately from the stale migration snapshot. It may reuse the snapshot only when digest and size still match. -- [ ] **Step 6: Run phase, race, and complete tests** +- [x] **Step 6: Run phase, race, and complete tests** Run: @@ -302,7 +338,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 7: Commit** +- [x] **Step 7: Commit** ```bash git add internal/vfs @@ -322,17 +358,17 @@ git commit -m "feat: add journaled recovery compaction and fallback" - Consumes: pack doctor, session engine, route metadata, and native source. - Produces: typed status, shadow evidence, doctor report, and JSON benchmark report. -- [ ] **Step 1: Write failing status, shadow, and doctor tests** +- [x] **Step 1: Write failing status, shadow, and doctor tests** Assert only canonical status terms are emitted; block-by-block and random shadow comparison catches a one-byte mismatch; doctor distinguishes daemon and mount health and checks pack, manifest, delta, backing, route, fallback, journal, and client compatibility independently. -- [ ] **Step 2: Run focused tests and verify missing package failures** +- [x] **Step 2: Run focused tests and verify missing package failures** Run: `go test ./internal/fsctl -count=1` Expected: FAIL because `internal/fsctl` does not exist. -- [ ] **Step 3: Implement canonical status and complete doctor aggregation** +- [x] **Step 3: Implement canonical status and complete doctor aggregation** ```go type Capability string @@ -345,11 +381,11 @@ const ( Return structured issues with component, severity, session ID, generation, and remediation; never include rollout contents. -- [ ] **Step 4: Implement shadow and benchmark runners** +- [x] **Step 4: Implement shadow and benchmark runners** Shadow compares full SHA-256 plus deterministic random offset/length reads. Benchmark measures native and virtual cold/warm sequential reads, 4 KiB random p50/p95/p99, stat/open, append, append+fsync, CPU time, RSS, and bytes read under the 128 MiB test budget. -- [ ] **Step 5: Run focused and complete tests** +- [x] **Step 5: Run focused and complete tests** Run: @@ -361,7 +397,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 6: Commit** +- [x] **Step 6: Commit** ```bash git add internal/fsctl @@ -383,17 +419,17 @@ git commit -m "feat: add shadow doctor benchmark and fs status" - Consumes: sanitized native operation traces, installed client versions, Codex SQLite state, and current-byte fallback. - Produces: compatibility results and optimistic `RouteSession`/`RestoreSession` transactions. -- [ ] **Step 1: Write failing trace, version, and SQLite transaction tests** +- [x] **Step 1: Write failing trace, version, and SQLite transaction tests** Cover sanitized `fs_usage` parsing, unknown-version quarantine, exact approved-version matching, optimistic route update, concurrent route change rejection, and rollback to a current backing rather than a stale migration snapshot. -- [ ] **Step 2: Run focused tests and verify missing API failures** +- [x] **Step 2: Run focused tests and verify missing API failures** Run: `go test ./internal/compat ./internal/codex -count=1` Expected: FAIL because compatibility and route APIs are absent. -- [ ] **Step 3: Implement machine-readable compatibility contracts** +- [x] **Step 3: Implement machine-readable compatibility contracts** ```go type Contract struct { @@ -408,15 +444,15 @@ type Contract struct { Store operation names, flags, and observed semantics without paths or contents. A version mismatch returns quarantine, never inferred compatibility. -- [ ] **Step 4: Implement optimistic Codex state routing** +- [x] **Step 4: Implement optimistic Codex state routing** Use `BEGIN IMMEDIATE`, `busy_timeout`, and `update threads set rollout_path=? where id=? and rollout_path=?`. Verify exactly one row changes, commit, then re-read. Route only after shadow succeeds and current-byte fallback metadata is durable. -- [ ] **Step 5: Implement upgrade quarantine transaction** +- [x] **Step 5: Implement upgrade quarantine transaction** When a new client version is detected, pause enrollment and destructive operations, create and verify current native backing for routed sessions, then atomically route those sessions to native files before marking the version safe to launch writes. -- [ ] **Step 6: Run focused, race, and complete tests** +- [x] **Step 6: Run focused, race, and complete tests** Run: @@ -428,7 +464,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 7: Commit** +- [x] **Step 7: Commit** ```bash git add internal/compat internal/codex @@ -451,39 +487,39 @@ git commit -m "feat: add Codex compatibility and route transactions" - Consumes: session manager and the native-operation contract. - Produces: a testable platform-neutral filesystem and `Mount(ctx, Options) error` adapter boundary. -- [ ] **Step 1: Write failing filesystem operation tests** +- [x] **Step 1: Write failing filesystem operation tests** Exercise root listing, regular-file stat, open flags, sequential/random read, append, fsync, flush/release, truncate, random write, rename/unlink policy, stable handles, concurrent readers, and single writer. -- [ ] **Step 2: Run focused tests and verify missing API failures** +- [x] **Step 2: Run focused tests and verify missing API failures** Run: `go test ./internal/mountfs -count=1` Expected: FAIL because the package is absent. -- [ ] **Step 3: Implement platform-neutral operation methods** +- [x] **Step 3: Implement platform-neutral operation methods** Keep all path validation, handle ownership, session lookups, and error mapping independent of FUSE. Expose operations through Go methods returning `syscall.Errno`; do not put storage logic in the adapter. -- [ ] **Step 4: Add `cgofuse` v1.6.0 behind explicit build constraints** +- [x] **Step 4: Add `cgofuse` v1.6.0 behind explicit build constraints** -`host_cgofuse.go` uses `//go:build fuse && cgo` and translates cgofuse callbacks to the neutral filesystem. `host_stub.go` uses `//go:build !fuse || !cgo` and returns a typed prerequisite error. Default `go test ./...` and cross-compilation must not require macFUSE headers. +`host_cgofuse.go` uses `//go:build fuse && cgo` and translates cgofuse callbacks to the neutral filesystem. `host_stub.go` uses `//go:build !fuse || !cgo` and returns a typed prerequisite error. Default `go test ./...` and cross-compilation must not require an installed FUSE host. -- [ ] **Step 5: Run default, race, and cross-platform compile tests** +- [x] **Step 5: Run default, race, and cross-platform compile tests** Run: ```bash go test ./internal/mountfs -count=1 go test -race ./internal/mountfs -count=1 -CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test ./internal/mountfs -count=1 -CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go test ./internal/mountfs -count=1 +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -c -o /tmp/codexfold-mountfs-linux.test ./internal/mountfs +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go test -c -o /tmp/codexfold-mountfs-windows.test.exe ./internal/mountfs go test ./... -count=1 ``` -Expected: all PASS without installed macFUSE. +Expected: all PASS without an installed FUSE host. -- [ ] **Step 6: Commit** +- [x] **Step 6: Commit** ```bash git add internal/mountfs go.mod go.sum @@ -503,30 +539,34 @@ git commit -m "feat: add platform filesystem and tagged fuse host" - Consumes: pack, fsctl, compat, codex route, vfs, and mountfs packages. - Produces: the public `codexfold pack` and `codexfold fs` command contracts. -- [ ] **Step 1: Write failing command-surface and dry-run tests** +- [x] **Step 1: Write failing command-surface and dry-run tests** Assert the complete public command tree exists, status/doctor/compatibility/benchmark are read-only, migrate/rollback/compact require `--apply`, and bulk enrollment excludes active or changing sessions until their promotion policy allows them. -- [ ] **Step 2: Run CLI tests and verify command failures** +- [x] **Step 2: Run CLI tests and verify command failures** Run: `go test ./internal/cli -run 'TestPack|TestFS|TestRoot' -count=1` Expected: FAIL because `pack` and `fs` commands are missing. -- [ ] **Step 3: Implement `pack` and read-only `fs` commands** +- [x] **Step 3: Implement `pack` and read-only `fs` commands** Expose `pack build`, `pack doctor`, `fs status`, `fs doctor`, `fs compatibility`, and `fs benchmark` with JSON output. Status remains `storage-engine` until preview gates are actually met. -- [ ] **Step 4: Implement guarded lifecycle commands** +- [x] **Step 4: Implement guarded lifecycle commands** Expose `fs serve`, `fs migrate`, `fs rollback`, `fs compact`, and `fs recover`. Mutation commands are dry-run by default and require `--apply`. Migration requires clean doctor, passing shadow evidence, approved client version, and eligible session state. - [ ] **Step 5: Implement bounded automatic discovery** +Current state: missing. Session discovery primitives exist, but no stable-session policy loop or bounded automatic enrollment transaction exists. + Discover existing, new, and forked Codex sessions from state; keep active native paths directly openable; enqueue only stable eligible sessions for shadow. Never require per-session production setup. - [ ] **Step 6: Run CLI, race, and complete tests** +Current state: the existing CLI, race, build, and complete tests pass, but this step remains open until automatic-enrollment tests are present and passing. + Run: ```bash @@ -539,7 +579,7 @@ go build ./cmd/codexfold Expected: all PASS. -- [ ] **Step 7: Commit** +- [x] **Step 7: Commit the implemented standalone CLI surface** ```bash git add internal/cli @@ -550,34 +590,36 @@ git commit -m "feat: expose transparent filesystem command surface" **Files:** - Create: `internal/service/service.go` -- Create: `internal/service/launchd_darwin.go` -- Create: `internal/service/service_other.go` +- Create: `internal/service/systemd.go` +- Create: `internal/service/windows.go` +- Create: `internal/service/mount_probe_*.go` - Create: `internal/service/service_test.go` -- Modify: `internal/cli/fs.go` +- Modify: `internal/cli/fs_service.go` +- Create: `internal/cli/fs_service_runtime_windows.go` **Interfaces:** - Consumes: built tagged binary, mount path, installed-client compatibility result, and doctor status. - Produces: deterministic service definition, start/stop/status, and update preflight. -- [ ] **Step 1: Write failing service-render and update-guard tests** +- [x] **Step 1: Write failing service-render and update-guard tests** -Assert launchd arguments are absolute, logs contain no session content, daemon and mount health are separate, preview auto-update is rejected, and a client version change enters quarantine before restart. +Assert launchd, systemd, and Windows SCM definitions use the same absolute `fs serve` arguments, logs contain no session content, daemon and mount health are separate, preview auto-update is rejected, and a client version change enters quarantine before restart. -- [ ] **Step 2: Run focused tests and verify missing service API** +- [x] **Step 2: Run focused tests and verify missing service API** Run: `go test ./internal/service ./internal/cli -run 'TestService|TestFSService' -count=1` Expected: FAIL because service APIs are absent. -- [ ] **Step 3: Implement service lifecycle without self-elevation** +- [x] **Step 3: Implement service lifecycle without self-elevation** -Render a per-user launchd plist and use `launchctl bootstrap/bootout/kickstart` only after an explicit apply command. Detect prerequisites but never install macFUSE or request elevation from library code. +Render and manage launchd on macOS, `systemd --user` on Linux, and the native SCM host on Windows only after an explicit apply command. Detect FUSE-T, FUSE3, or WinFsp prerequisites but never install them, enable Linux linger, or request elevation from library code. -- [ ] **Step 4: Implement update compatibility guard** +- [x] **Step 4: Implement update compatibility guard** Before service binary promotion, run doctor and compatibility against installed clients. Preview/canary versions require explicit promotion. Unknown client versions trigger Task 6 quarantine and current-byte native routing. -- [ ] **Step 5: Run focused and complete tests** +- [x] **Step 5: Run focused and complete tests** Run: @@ -588,7 +630,7 @@ go test ./... -count=1 Expected: all PASS. -- [ ] **Step 6: Commit** +- [x] **Step 6: Commit** ```bash git add internal/service internal/cli/fs.go @@ -608,15 +650,15 @@ git commit -m "feat: add guarded filesystem service lifecycle" - Consumes: all platform-neutral packages and tagged stubs. - Produces: reproducible gate report for `fs-engine-preview`; does not claim real adapter or Codex readiness. -- [ ] **Step 1: Add a deterministic fork/session corpus generator** +- [x] **Step 1: Add a deterministic fork/session corpus generator** Generate exact repeated fields at arbitrary positions, repeated JSONL records, forked histories, large multi-block fields, independent append tails, random writes, truncation, invalid JSONL, and empty sessions without using real user content. -- [ ] **Step 2: Add fault injection and stress tests** +- [x] **Step 2: Add fault injection and stress tests** Run 10,000 random reads, 100,000 appends, concurrent reader/single-writer race tests, every journal phase interruption, resolver corruption, daemon-equivalent reopen, route transaction races, and current-byte fallback validation. -- [ ] **Step 3: Run complete correctness and race gates** +- [x] **Step 3: Run complete correctness and race gates** Run: @@ -625,21 +667,21 @@ go test ./... -count=1 go test -race ./... -count=1 go vet ./... go build ./cmd/codexfold -CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test ./... -count=1 -CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go test ./... -count=1 +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ./cmd/codexfold +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build ./cmd/codexfold ``` Expected: all PASS. -- [ ] **Step 4: Run packed and virtual benchmarks** +- [x] **Step 4: Run packed and virtual benchmarks** Compare the same generated 758 MiB rollout through native and virtual reads, record warm/cold throughput, p50/p95/p99, CPU, RSS, cache budget, and loose-object open count. Failure leaves status below `fs-engine-preview`. -- [ ] **Step 5: Document exact evidence and limitations** +- [x] **Step 5: Document exact evidence and limitations** Record commands, hardware, versions, results, and unmet real-adapter gates. Do not call fixture results transparent, canary, or production-ready. -- [ ] **Step 6: Commit** +- [x] **Step 6: Commit** ```bash git add internal/testfs scripts/test-cross-platform.sh docs/validation-fs-preview.md @@ -650,38 +692,42 @@ git commit -m "test: validate transparent filesystem engine preview" **Files:** - Create: `docs/validation-macos-canary.md` -- Modify only after approval: local macFUSE prerequisite and user launch service outside the public repository. +- Modify only after approval: local FUSE-T prerequisite and the standalone per-user launch service state. - Modify only after all gates pass: selected Codex state rows through `codexfold fs migrate --apply`. **Interfaces:** - Consumes: completed Tasks 1 through 10, explicit system-extension authorization, real Codex versions, selected archived sessions, and retained native snapshots. - Produces: actual `platform-canary` evidence or an explicit blocked result; no stronger status without seven-day retention. -- [ ] **Step 1: Capture native Codex Desktop and CLI operations** +- [x] **Step 1: Capture native Codex Desktop and CLI operations** With explicit elevation approval, run sanitized `fs_usage` tracing for list/open/read/pread/append/fsync/truncate/rename/unlink/lock/watcher behavior during history display, resume, message send, tool use, fork, archive, unarchive, repair, restart, and upgrade. Import the trace into a machine-readable compatibility contract. -- [ ] **Step 2: Reconcile the adapter with every observed operation** +- [x] **Step 2: Reconcile the adapter with every observed operation** Add or correct platform operation tests before changing adapter code. Any unsupported observed operation blocks installation and migration. -- [ ] **Step 3: Request and apply macFUSE authorization** +- [x] **Step 3: Request and apply the selected FUSE host authorization** Install the selected prerequisite only after explicit approval, build with `-tags fuse`, mount a temporary fixture namespace, and run fstest/fsx-equivalent plus the project operation suite. A mount alone is not success. -- [ ] **Step 4: Run real-session shadow without changing Codex routes** +- [x] **Step 4: Run real-session shadow without changing Codex routes** Select 5–10 archived sessions, fold and pack without source removal, compare every byte and 10,000 random ranges, run doctor and benchmark, then keep Codex on native routes. Any mismatch stops the task. - [ ] **Step 5: Route retained-source canaries** +Current state: isolated retained-source CLI and Desktop canaries passed direct open, resume, append, tool use, fork, archive/unarchive, daemon restart, rollback, re-migration, and quarantine. One idle retained-source managed CLI session also passed an actual host reboot, post-boot managed resume, exact rollback, and native resume. This step remains open because managed-session sleep/wake, host interruption during append/compaction/migration/rollback, current Desktop compatibility, and real-home retained-source canaries are not complete. + After clean shadow and compatibility, migrate only the selected archived sessions. Verify Desktop direct click, CLI resume, history, message send, tool use, fork, archive, unarchive, daemon termination, mount restart, sleep/wake, host restart, rollback, and compatibility quarantine. Never delete native snapshots. - [ ] **Step 6: Start seven-day canary retention** +Current state: not started because the project has not reached `platform-canary`. + Record daemon/mount health, exact-byte doctor, recovery incidents, client versions, and performance. Status remains `platform-canary` during retention; `production-ready:macos` requires the full period with zero unresolved incidents. -- [ ] **Step 7: Commit only public sanitized evidence** +- [x] **Step 7: Commit only public sanitized evidence** ```bash git add docs/validation-macos-canary.md @@ -690,11 +736,72 @@ git commit -m "docs: record macOS transparent filesystem canary" No private path, session ID, trace content, credential, or control-plane name may enter the public evidence. +### Task 12: Bounded Automatic Discovery And Enrollment + +**Status:** Complete in the current implementation. Production enablement remains gated by platform readiness and retention. + +**Requirements:** `TF-001`, `TF-010`, `TF-011`, `TF-014`, `TF-015`, `TF-021`. + +**Exact next work:** + +- [x] Add policy tests for existing sessions, newly created sessions, forks, active writers, changing files, archived eligibility, unknown client versions, failed doctor state, insufficient disk budget, bounded batches, restart idempotency, and failed cutover. +- [x] Implement a read-only enrollment planner that consumes Codex state, rollout stability evidence, compatibility, doctor, writer state, promotion stage, and storage-budget preflight, and emits explicit eligible/ineligible reasons without changing routes. +- [x] Implement bounded apply transactions that fold, pack, shadow, stage at most one retained native snapshot, wait for exact mount acknowledgement, and only then update routing. A failure leaves the original native route and source unchanged. +- [x] Integrate the planner into the standalone service with a bounded interval and batch size. Newly created sessions and forks remain native while active and need no per-session command when they later become eligible. +- [x] Validate in an isolated Codex home across daemon restart, real CLI append, client-version quarantine, and failed canonical cutover before any real-home automatic enrollment is allowed. + +### Task 13: Conservative Branch Lifecycle And Content-Change Boundary + +**Status:** Complete. Conservative family classification, guarded archive execution, exact-contained deletion, and explicit content-changing repair/reconciliation boundaries are implemented and verified. + +**Requirements:** `TF-018`, `TF-019`, `TF-020`. + +**Exact next work:** + +- [x] Add read-only fork-family reports that distinguish shared exact content, independent tails, complete containment, active/archived state, and unknown relationships. Never label a branch useless from ancestry, age, title, or size alone. +- [x] Trace and test the current official Codex archive operation, then add a dry-run-first archive mutation that requires an explicit session selection, revalidates database route and source digest, preserves the rollout, and updates file and state atomically. +- [x] Keep `remove-contained` as a separate archived-only operation and add integration coverage proving that family classification or archive never triggers deletion automatically. +- [x] Add CLI and static boundary regression tests proving `repair-rollout` and `reconcile-rollout` remain the only content-changing reconciliation entrypoints and cannot be called by fold, migrate, compact, enrollment, rollback, GC, archive, or family paths. + +### Task 14: Hard Storage Budgets, Retention, Cleanup, And Reclamation Accounting + +**Status:** Complete in the current implementation. Destructive retention promotion remains disabled before platform readiness. + +**Requirements:** `TF-009`, `TF-014`, `TF-021`. + +**Exact next work:** + +- [x] Add a platform-neutral storage inventory that accounts separately for logical session bytes, unique loose objects, packs, native sources, retained snapshots, current fallbacks, active deltas, writable backings, old generations, retirement state, journal-owned recovery files, unowned temporary files, and metadata. +- [x] Add preflight APIs that calculate projected peak bytes and reject fold, pack, migrate, rollback, compact, enrollment, copy-on-write, materialization, repair, and reconciliation before writing when the hard temporary budget or free-space reserve would be exceeded. +- [x] Enforce one immutable migration snapshot and one current writable fallback per managed session, one full-session scratch file per transaction, and current-plus-previous pack-generation retention until leases close. +- [x] Add startup and explicit GC for abandoned temporary files, expired unleased generations, and bounded retired state. Journal-owned recovery files, active leases, and the sole recoverable generation are retained. +- [x] Extend status, doctor, mutation, and GC results with physical inventory, projected peak, projected final, projected reclaimable, and actual reclaimed bytes. Low-space, interrupted-cleanup, retained-fallback, hard-link, lease, and repeated-GC tests pass. + +### Task 15: Remaining Platform And Retention Gates + +**Status:** Partial as release evidence; Linux adapter and service execution are now real, but the remaining platform and retention gates still block stronger capability claims. + +**Requirements:** `TF-003`, `TF-008`, `TF-009`, `TF-011`, `TF-012`, `TF-014`, `TF-015`, `TF-017`, `TF-021`. + +**Exact next work:** + +- [x] Import exact compatibility contracts for the currently installed Codex Desktop and CLI versions and return the isolated canary doctor to a clean client state. +- [x] Run retained-source managed macOS canaries through sleep/wake and real host restart, including process-interrupted append, compaction, migration, and rollback cases required by the contract. +- [x] Implement the Linux FUSE3 adapter with explicit `fuse fuse3` build tags and execute real unprivileged read, append, copy-on-write, truncate, canonical rename, native fallback, `SIGKILL` stale-mount recovery, remount, performance, backing-seal, and `systemd --user` install/start/status/stop gates. +- [x] Implement the Windows WinFsp adapter and native SCM service host, mount probe, configuration, start/stop/status, and restart policy; default and WinFsp binaries and tests cross-compile. +- [ ] Continue only bounded real-home canary observation and complete seven incident-free days before any `platform-canary` promotion decision; general automatic apply remains disabled. +- [ ] Execute Linux real-client compatibility, upgrade quarantine, rollback, and retention gates. +- [ ] Execute Windows WinFsp operation, crash/restart, performance, real-client compatibility, upgrade quarantine, rollback, and retention gates on a real Windows host. + ## Plan Self-Review -- `TF-001` through `TF-016` each map to implementation and verification tasks. -- Real-client, real-adapter, restart, upgrade, and retention gates remain in Task 11 and cannot be satisfied by Task 10 fixtures. -- macFUSE is a candidate and explicit authorization boundary, not a baked-in product promise. +- `TF-001` through `TF-022` each map to implementation and verification tasks or an explicitly identified storage-engine baseline. +- Real-client, real-adapter, restart, upgrade, and retention gates remain in Tasks 11 and 15 and cannot be satisfied by Task 10 fixtures. +- FUSE-T is the validated macOS host and FUSE3 is the native-gated Linux host; Windows remains implementation and cross-compile only until independent WinFsp execution passes. - The stale migration snapshot is never used as current fallback after virtual writes diverge. - The default build remains portable and does not require installed FUSE headers. - No task changes a real Codex route before shadow, compatibility, doctor, and explicit apply gates pass. +- Automatic enrollment remains blocked until Task 12 and Task 14 are complete. +- Branch archival, exact-contained deletion, and content-changing repair remain separate operations. +- No logical deduplication result is presented as physical reclamation without storage accounting. +- Public product behavior remains independent of any private control plane. diff --git a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md index 5d21667..89f2639 100644 --- a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md +++ b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-design.md @@ -17,7 +17,7 @@ Every implementation plan, task, test report, release note, and control-plane st | `TF-001` | Opening or resuming a managed session requires no manual materialization or preparation command. | | `TF-002` | Unmodified Codex Desktop and Codex CLI access a normal regular-file JSONL path and do not know that storage is virtual. | | `TF-003` | Every byte and every file operation Codex actually uses has native-equivalent observable behavior. Platform readiness is blocked by any unsupported operation used by Codex. | -| `TF-004` | Identical content across sessions and forks is stored once in the shared object store; session histories remain independently writable. | +| `TF-004` | Identical byte content across sessions and forks is stored once in the shared object store; session histories remain independently writable. Reuse applies to exact repeated fields, records, and content-defined chunks at arbitrary positions and is not limited to a shared file prefix. | | `TF-005` | Normal append writes go to a durable delta without complete base materialization. | | `TF-006` | Truncate, random write, and every non-append mutation that can be represented safely must transition to copy-on-write before success. A mutating operation used by Codex may not be silently rejected in a production-ready adapter. | | `TF-007` | Packed runtime reads do not open loose object files per manifest part and do not perform a persistent-index lookup per object. The concrete durable and runtime index formats are selected by implementation evidence and must satisfy the performance and recovery gates. | @@ -30,6 +30,12 @@ Every implementation plan, task, test report, release note, and control-plane st | `TF-014` | Native fallback deletion is disabled until platform production readiness and per-session retention gates pass. | | `TF-015` | A Codex client version without passing compatibility evidence enters compatibility quarantine: enrollment and destructive automation pause, and an already-routed session is automatically switched to a byte-verified current native writable backing before that client version may write. Routine client upgrades require no manual session preparation. | | `TF-016` | Platform filesystem prerequisites requiring elevated or system-extension approval are installed only after explicit user authorization. | +| `TF-017` | A canonical mount may never degrade into a writable ordinary directory or expose stale session files. The unmounted backing directory is empty and write-sealed, activation requires a live CodexFold mount identity, and service start succeeds only after the daemon, required platform mount policy, and operational mount probe are healthy. Desktop realpath rewrites from `CODEX_HOME/sessions` or `archived_sessions` into the mount alias are synchronously normalized in the Codex state database, and the route watcher accepts either spelling without exiting. | +| `TF-018` | Fork-family classification and branch archival are conservative, evidence-driven, and dry-run-first. Fork ancestry, age, size, or title alone never proves that a branch is useless. An archive mutation requires explicit user selection or an explicit policy, revalidates current Codex state and rollout bytes, and preserves a recoverable session. | +| `TF-019` | Deleting a fully duplicated branch is limited to an archived session whose applicable JSONL record sequence is proven exactly and completely contained in another retained session. Apply additionally requires current-source and temporary-unfold recovery proof, transactional Codex state cleanup, and a retained tombstone and manifest. Similarity and fork ancestry are not deletion evidence. | +| `TF-020` | Byte-preserving storage optimization never changes rollout bytes. Repair, reconciliation, prompt cleanup, message removal, or any other content-changing operation is a separate explicit workflow that writes a separately verified output and is never run implicitly by fold, migration, compaction, enrollment, GC, or rollback. | +| `TF-021` | Full-size transaction files, retained native snapshots, writable fallbacks, old pack generations, recovery artifacts, and temporary files are governed by hard preflight and retention budgets. Successful and abandoned temporary artifacts are cleaned automatically. Status and completion reports separate logical bytes, physical store bytes, retained source/fallback bytes, temporary/recovery bytes, projected reclamation, and actual reclaimed bytes; no physical-saving claim is allowed while equivalent full copies still occupy disk. | +| `TF-022` | CodexFold is a standalone public product. Its public code, CLI, daemon, service definitions, configuration, storage formats, doctor, GC, rollback, and enrollment contain no dependency on or reference to a private control plane. External package managers or operators may install and supervise CodexFold only from outside the product boundary. | ### Decision Hierarchy @@ -46,7 +52,7 @@ An implementation change is therefore acceptable only when its requirement cover ### Drift Control - Each implementation-plan task lists the requirement IDs it implements or verifies. -- Each plan starts with a complete requirement-to-task coverage table for `TF-001` through `TF-016`. +- Each plan starts with a complete requirement-to-task coverage table for `TF-001` through `TF-022`. - A requirement with no implementation or verification task blocks plan approval. - Completion reports list fresh evidence by requirement ID and state any unmet ID explicitly. - Mock, fixture, or synthetic evidence cannot satisfy a requirement that names real Codex, a real platform adapter, a client upgrade, a host restart, or a canary period. @@ -91,6 +97,11 @@ Passing unit tests, successful materialization, a mounted filesystem, one succes - Claiming identical APFS, ext4, or NTFS metadata that Codex does not observe. - Migrating real user sessions during engine development. - Deleting native fallbacks before canary and rollback gates pass. +- Treating strict byte-prefix ancestry as the only reusable-content shape. +- Automatically deciding that a branch is useless from age, title, size, or fork ancestry. +- Running repair, reconciliation, prompt cleanup, or other content-changing transforms as part of byte-preserving optimization. +- Claiming physical disk savings from logical deduplication while retained sources, fallbacks, recovery copies, or temporary files still occupy the same bytes. +- Requiring a private deployment or control-plane product at runtime. ## Architecture @@ -200,7 +211,7 @@ The engine must implement and test: | `flush/release` | Release handle state without triggering unsafe inline compaction | | `truncate` | Transition to writable backing before changing visible size | | random `write/pwrite` | Transition to writable backing before mutation | -| `rename/unlink` | Implement native-equivalent behavior when Codex tracing shows that the client uses it; otherwise management operations use explicit APIs | +| `rename/unlink` | Implement native-equivalent behavior when Codex tracing shows that the client uses it; Codex archive/unarchive currently requires canonical active/archive directories inside one virtual namespace | | `mmap` | Supported when the platform adapter can provide coherent read pages; otherwise platform readiness is blocked | | file locks | Preserve the lock behavior observed in the native Codex trace | @@ -259,10 +270,14 @@ The trace suite covers listing, opening, scrolling old history, resume, sending ### macOS -- Current reference adapter: macFUSE. -- Service: user launch service with keep-alive and mount health monitoring. -- Required tests: APFS native baseline, Apple Silicon, Codex Desktop, Codex CLI, sleep/wake, network changes, user logout/login, daemon kill, mount restart, and Codex upgrade. -- If macFUSE remains the selected adapter, its installation and system-extension approval are separate user-authorized deployment steps. Development before that approval uses the platform-neutral engine and adapter mocks. +- Selected production candidate: an Apple-native Swift FSKit extension using versioned binary Unix-domain-socket IPC to the Go CodexFold daemon. +- Service: two user launch services, one for the Host/Go daemon chain and one for mount supervision, with child-process lock ownership, build identity, mount health, and atomic app/binary rollback checks. +- Required tests: APFS native baseline, Apple Silicon, Codex Desktop, Codex CLI, canonical `sessions` and `archived_sessions` namespace moves, sleep/wake, network changes, user logout/login, daemon kill, mount restart, and Codex upgrade. +- The Swift extension exposes regular JSONL paths and filesystem metadata while the Go daemon owns packed reads, append delta, copy-on-write backing, generation recovery, and canonical routing. The production implementation may not depend on NFS or a third-party FUSE compatibility layer. +- FUSE-T `1.2.7` with synchronous NFS remains historical canary evidence and a development-only fallback. It is not the terminal macOS architecture and cannot satisfy native FSKit production readiness on its own. +- FUSE-T `1.2.7`'s FSKit backend is rejected for production. An isolated real mount lost the first of two complete JSONL records written at the same stale EOF, and managed-to-native route changes remained cached past the five-second correctness gate. Basic read/write, `F_FULLFSYNC`, truncate, remount, and throughput results do not override a byte-loss failure. FUSE-T also documents that notifications are unavailable for its FSKit backend. +- The Apple-native FSKit implementation is distinct from FUSE-T's rejected FSKit backend. It remains `fs-engine-preview` until its complete performance, cache coherency, real-client, crash, upgrade, rollback, retention, and power-loss gates pass independently. +- Platform readiness requires a directory-level canonical namespace or an equivalent mechanism that keeps Codex archive and unarchive moves native-compatible. ### Linux @@ -286,7 +301,7 @@ Platform readiness is independent. Passing macOS gates does not imply Linux or W The platform-neutral core defines byte layout and transaction behavior, not a lowest-common-denominator filesystem API. Each adapter must implement the strongest native semantics Codex uses on that platform; macOS behavior may not be weakened to match Windows or Linux limitations. -macFUSE, FUSE3, and WinFsp are initial candidates rather than product promises. If native-operation traces or platform gates disqualify a candidate, it must be replaced without weakening `TF-001` through `TF-016`. +Apple-native FSKit, FUSE3, and WinFsp are current candidates rather than product promises. If native-operation traces or platform gates disqualify a candidate, it must be replaced without weakening `TF-001` through `TF-022`. ## Migration And Rollback @@ -324,6 +339,43 @@ After platform production readiness, enrollment is policy-driven rather than man - No user action is required to enroll, open, resume, fork, compact, or re-enroll a normal session. - Enrollment failure leaves the native database route and source file unchanged. +## Branch Cleanup And Content-Change Boundary + +Storage sharing, branch archival, exact-contained deletion, and content-changing repair are four separate operations: + +1. **Storage sharing** preserves every byte and may reuse exact fields, records, or content-defined chunks found anywhere in any session. +2. **Branch classification and archival** reports evidence first. It may recommend an archive candidate, but it never mutates from ancestry, age, title, or size alone and never removes recovery ability. +3. **Exact-contained deletion** applies only to an already archived session after complete direct containment and recovery proof. It is not a side effect of folding, packing, enrollment, compaction, or GC. +4. **Repair, reconciliation, and prompt cleanup** change content and therefore write a separate verified output. They never replace either source implicitly and never participate in byte-identical savings claims. + +## Storage Budget And Reclamation Accounting + +Before any operation that can create a full-size session copy or a new store generation, CodexFold calculates its projected peak physical bytes and checks the configured hard budget and required free-space reserve. Automatic enrollment is disabled until this preflight is implemented and passes. + +The default retention model is cardinality-bounded: + +- A managed session has at most one immutable migration snapshot and at most one current native writable fallback. A current fallback must replace or reuse stale current-fallback state rather than accumulate another full copy. +- One transaction may create at most one full-session scratch file for the affected session. Named historical copies such as `native-before`, `fold-before`, `merged`, and `repaired` are not implicit recovery generations. +- Pack publication retains the current generation and only the immediately previous verified generation while a lease or rollback window requires it. Older unleased generations are GC candidates. +- Startup recovery removes abandoned temporary artifacts only after journal analysis proves that they are not the sole committed or recoverable generation. + +Every mutating command reports, before and after apply: + +```text +logical session bytes +unique object bytes +pack bytes +native source bytes +retained snapshot bytes +current fallback bytes +temporary and recovery bytes +projected peak bytes +projected reclaimable bytes +actual reclaimed bytes +``` + +Logical duplicate savings and physical disk reclamation are distinct metrics. A fold or migration may report logical reuse while reporting zero actual reclamation when source or fallback copies are still retained. + ## Failure Semantics - Pack, manifest, delta, backing, and journal commits use temporary files, synchronization, and atomic replacement. @@ -331,8 +383,14 @@ After platform production readiness, enrollment is policy-driven rather than man - Startup recovery resolves every pending journal entry before accepting mounts. - A corrupt object, pack frame, manifest, or index blocks the affected session and preserves its native fallback. - Mount health failure blocks new migration and compaction. +- The ordinary directory underneath a canonical mount is empty, is never used as session storage, and remains non-writable whenever the mount is absent. +- Every mount instance exposes a process-generated identity through an operational read; provider type or `statfs` alone is not mount-health evidence. +- Namespace activation requires the live mount identity and may not accept a plain directory containing look-alike `sessions` trees. +- A store has one filesystem-host process lock. Service installation and restart return success only after launchd reports a running process and the mount identity is readable. - Database and global-state changes use optimistic revalidation and rollback. - A session with an active writer is never folded, removed, migrated, or rolled back. +- A branch is never archived or removed solely from inferred fork ancestry, age, title, or size. +- Budget preflight failure blocks the mutating operation before any full-size temporary file is created. - A detected Codex Desktop or CLI version change immediately enters compatibility quarantine and schedules the native-operation compatibility suite. - Quarantine pauses enrollment, migration, compaction that removes fallback state, fallback deletion, and GC. - Before an unapproved client version may write an already-routed session, the service automatically switches it to a verified current native writable backing. It never routes the stale migration snapshot as current data. @@ -402,6 +460,7 @@ A single SHA-256 mismatch, unexpected Codex file operation, unresolved crash-rec - Daemon process health and mount health as separate states. - Mount path ownership, adapter version, and mounted-generation identity. +- Empty and write-sealed ordinary mount backing state whenever the adapter is not mounted. - Active pack generation, every resolver entry, object boundaries, stored checksum, raw length, and object SHA-256. - Manifest generation validity and complete virtual reconstruction. - Delta path, size, mtime, digest, synchronization state, and writer lease. @@ -439,6 +498,7 @@ Platform adapters translate native filesystem calls only. They do not fork or re - Pack and delta files use user-only permissions. - Mount access is restricted to the owning user. - Management operations require explicit apply flags for migration, rollback, fallback deletion, and GC. +- Public runtime behavior and configuration have no private control-plane dependency. - Crash reports and benchmark output must not include raw rollout data. ## CLI Contract diff --git a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-traceability-review.md b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-traceability-review.md index 8c8ec54..271c728 100644 --- a/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-traceability-review.md +++ b/docs/superpowers/specs/2026-07-11-transparent-session-filesystem-traceability-review.md @@ -13,6 +13,7 @@ The purpose is to prevent implementation plans from replacing the requested outc | Codex opens a normal JSONL path without manual materialization | `TF-001`, `TF-002`, Product Promise | Aligned | | The Codex client is not patched and does not know storage is virtual | `TF-002`, Scope | Aligned | | Duplicate content across sessions and forks is stored once | `TF-004`, Reference Packfile Design | Aligned | +| Exact repeated fields, records, and chunks can be shared even when they are not a strict file prefix | `TF-004`, Branch Cleanup And Content-Change Boundary | Corrected: the contract now states explicitly that strict prefix ancestry is not the only reusable shape | | A session can be read at arbitrary offsets without complete materialization | `TF-003`, Virtual File Model, File Operation Contract | Aligned | | Append writes use a durable delta and do not hydrate the complete base | `TF-005`, Append And Copy-On-Write | Aligned | | Truncate and random writes automatically move to a complete writable backing | `TF-006`, Append And Copy-On-Write | Corrected: the first spec allowed fail-closed as an equal outcome; production behavior now requires copy-on-write for safely representable mutations | @@ -29,20 +30,25 @@ The purpose is to prevent implementation plans from replacing the requested outc | The stable workflow is automatic rather than one manual migrate per session | `TF-011`, Migration And Rollback | Corrected: automatic discovery and enrollment of existing sessions, new sessions, and forks was missing from the first spec | | Codex upgrades must not silently change supported file behavior | `TF-015`, Native Behavior Discovery, Behavioral Gates | Aligned | | Every Codex Desktop or CLI version change quarantines virtual writes until current bytes are automatically routed to verified native backing or compatibility passes | `TF-015`, Failure Semantics | Corrected: routine upgrades remain automatic without exposing unknown write behavior to virtual storage | -| macOS, Linux, and Windows use independently validated platform adapters | `TF-012`, Platform Adapters | Aligned; macFUSE, FUSE3, and WinFsp are current candidates, not product promises | +| macOS, Linux, and Windows use independently validated platform adapters | `TF-012`, Platform Adapters | Aligned; FUSE-T, FUSE3, and WinFsp are current candidates, not product promises | | Shared storage, read, write, generation, doctor, and recovery behavior stays in the platform-neutral core | `TF-012`, Platform-Neutral Core Contract | Aligned as a responsibility boundary; internal formats and algorithms remain replaceable | | Each platform is certified independently | `TF-012`, Canonical Status Terms, Platform Adapters | Aligned | | Windows handles share mode, oplock, replace, Defender, case-insensitive paths, service restart, and mount naming | `TF-003`, `TF-012`, Windows adapter | Corrected: mount namespace or drive-letter behavior was added | | iOS and Android access a desktop host and do not host this local filesystem | Scope, Mobile | Corrected: the mobile boundary was missing | -| macFUSE or other elevated prerequisites require explicit user approval | `TF-016`, Platform Adapters | Aligned | +| FUSE-T or other privileged prerequisites require explicit user approval | `TF-016`, Platform Adapters | Aligned | | Status language must not call the storage engine transparent or production-ready | `TF-013`, Canonical Status Terms | Aligned | +| Useless or closed fork branches may be classified and archived conservatively, but ancestry or age cannot decide the mutation | `TF-018`, Branch Cleanup And Content-Change Boundary | Added: the original cleanup goal was not represented as a non-negotiable requirement | +| An archived branch that is exactly and completely contained in another retained session can be removed only after recovery proof | `TF-019`, Branch Cleanup And Content-Change Boundary | Added: the existing containment implementation is now protected by the product contract | +| Prompt cleanup, repair, and reconciliation are content-changing workflows and must not be confused with byte-preserving folding | `TF-020`, Branch Cleanup And Content-Change Boundary | Added: the implementation already separates outputs, but the contract did not prevent future drift | +| Temporary copies, recovery generations, retained snapshots, and claimed savings require bounded physical-space accounting | `TF-021`, Storage Budget And Reclamation Accounting | Added: logical deduplication was previously specified without a hard physical-space contract | +| CodexFold remains an independent public product even when an external operator installs or supervises it | `TF-022`, Security And Privacy | Added: private deployment policy cannot enter the public runtime architecture | ## Open Engineering Questions That Do Not Change The Goal These questions require evidence during implementation. They are not permission to weaken a requirement: - The exact native Codex operation trace on each client version. -- Whether the current macFUSE candidate can satisfy the observed `mmap`, lock, watcher, and cache behavior. +- Whether the current FUSE-T adapter continues to satisfy every operation introduced by future Codex versions. - The optimal immutable pack size and decompressed-cache admission policy within `TF-008` limits. - The idle window that prevents compaction from racing a resumed writer. - The maximum mount-recovery time that remains acceptable during canary. @@ -65,4 +71,4 @@ The corrected contract matches the approved outcome. The intent-level review fou 9. Client upgrades enter compatibility quarantine and route current bytes to verified native writable backing before unknown writes. 10. Adapter products and cache/index algorithms are reference choices rather than product promises. -After these corrections, no known goal-level drift remains in the design contract. Implementation details may still improve, but they must preserve the fixed outcome, invariants, and gates. This is a contract conclusion, not a claim that transparent filesystem implementation or production validation is complete. +The 2026-07-14 alignment added the cleanup, physical-space, and standalone-product commitments that were present in the original product discussion but absent from the first review. After those additions, no known goal-level drift remains in the design contract. Implementation gaps remain, especially automatic enrollment, conservative branch classification and archive execution, hard disk-budget enforcement, real Linux and Windows adapters, managed-session host-restart and sleep/wake validation, and canary retention. This is a contract conclusion, not a claim that transparent filesystem implementation or production validation is complete. diff --git a/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md b/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md new file mode 100644 index 0000000..337f110 --- /dev/null +++ b/docs/superpowers/specs/2026-07-14-transparent-session-filesystem-implementation-alignment.md @@ -0,0 +1,99 @@ +# Transparent Session Filesystem Implementation Alignment + +## Purpose + +This document aligns the original product commitments, the canonical transparent-filesystem contract, the implementation plan, the current repository, and the available validation evidence. + +It does not redesign CodexFold, authorize real-session enrollment, promote the capability above `fs-engine-preview`, or treat fixtures as production evidence. CodexFold remains a standalone public product. Native launchd, `systemd --user`, and Windows SCM supervision are part of the standalone runtime; private or external control-plane coupling remains outside it. + +Baseline refreshed against the native FSKit development checkpoint and current Task 12 through Task 15 implementation on 2026-07-23. + +The validation references in this document describe the completed signed build 102 App/extension and current helper isolated Canary. It passed installed mount-health, byte-identity, restart, host-reboot, rollback, and exact current-client gates. None of the status rows below authorize production activation or transfer that evidence to a later candidate. + +## Original Commitment To Requirement Mapping + +| Original product commitment | Canonical requirement | Alignment result | +| --- | --- | --- | +| Codex Desktop and CLI open and resume a normal JSONL path without a materialization step | `TF-001`, `TF-002`, `TF-003` | Preserved | +| The client remains unmodified and cannot distinguish managed storage from a supported native file | `TF-002`, `TF-003` | Preserved | +| Exact duplicate content is stored once across sessions and forks | `TF-004` | Preserved | +| Reuse is not limited to a strict shared prefix; repeated fields, records, and chunks at arbitrary positions are shareable | `TF-004` | Clarified in the contract | +| Forks remain independently writable after sharing common content | `TF-004`, `TF-005`, `TF-006` | Preserved | +| Normal writes append to a durable delta; non-append mutations use safe copy-on-write | `TF-005`, `TF-006` | Preserved | +| Runtime reads use packed storage rather than tens of thousands of loose-object opens | `TF-007`, `TF-008` | Preserved | +| Correctness includes performance, bounded memory, crash recovery, restart recovery, and exact rollback | `TF-008`, `TF-009`, `TF-010` | Preserved | +| Stable production operation discovers existing sessions, new sessions, and forks automatically | `TF-011` | Preserved and implemented behind preview, compatibility, health, stability, and storage gates | +| macOS, Linux, and Windows share one storage engine but have independent adapters and readiness gates | `TF-012`, `TF-016`, `TF-017` | Preserved | +| Capability language cannot overstate a storage engine, preview, or one successful canary | `TF-013` | Preserved | +| Native sources and current recoverable bytes remain available until the relevant gates pass | `TF-010`, `TF-014`, `TF-015` | Preserved | +| Useless or closed branches can be identified and archived, but the tool must not guess destructively | `TF-018` | Implemented with evidence-only family reports and explicit recoverable archive transactions | +| A branch that is exactly 100% contained in another retained session can be deleted only after exact recovery proof | `TF-019` | Added to the contract; implementation exists | +| Prompt cleanup, repair, and reconciliation are separate content-changing workflows, not storage folding | `TF-020` | Added to the contract; implementation and static regression boundaries exist | +| Temporary files, recovery generations, retained snapshots, and repeated operations must not consume unbounded disk | `TF-021` | Added to the contract; hard budgets, bounded retention, leases, and GC are implemented | +| Reported savings distinguish logical reuse from actual physical bytes reclaimed | `TF-021` | Added to the contract; projected and actual physical accounting is implemented | +| CodexFold is an independent open-source product with no private control-plane dependency | `TF-022` | Added to the contract; current repository is aligned | + +## Requirement To Implementation And Evidence + +| Requirement | Current implementation | Tests or evidence | Status | +| --- | --- | --- | --- | +| `TF-001` | `internal/cli/fs.go`, `internal/mountfs`, canonical migration, automatic enrollment, and routing | Isolated CLI/Desktop direct-open, resume, and automatic-enrollment canaries | Partial only at release level: implemented and verified on macOS canaries; real-home automatic apply remains preview-gated | +| `TF-002` | `internal/mountfs`, `internal/fskitproto`, the Swift FSKit extension, canonical routing, and mount identity | Native FSKit mounted behavior plus real current CLI/Desktop resume, append, fork, archive, restart, and post-crash continuation | Implemented and verified in an isolated macOS canary; production promotion remains gated by power-loss and retention evidence | +| `TF-003` | Neutral operation layer plus exact compatibility contracts in `internal/compat` | Real macOS traces and adapter canaries plus real Linux FUSE3 operations | Partial: current installed macOS clients and Linux adapter operations are covered; real Linux Codex clients and Windows are not validated | +| `TF-004` | `internal/scan`, `internal/cdc`, `internal/fold`, `internal/pack` | Repeated field, record, CDC, fork, and non-prefix corpus tests | Implemented | +| `TF-005` | `internal/vfs` append delta and writer leases | Append-without-hydration tests and real CLI/Desktop append evidence | Implemented | +| `TF-006` | `internal/vfs` copy-on-write backing and neutral write operations | Random-write, truncate, interruption, native FSKit mounted mutation, and historical FUSE-T tests | Implemented | +| `TF-007` | Immutable packs, in-memory index, bounded cache, random-read resolver | Pack round-trip/corruption tests and 758 MiB packed-read benchmark | Implemented | +| `TF-008` | Packed-read benchmark, mounted performance tests, and `internal/testfs` stress harness | 758 MiB core benchmark, three native FSKit cold/warm `F_NOCACHE` rounds, cache-coherency matrix, bounded RSS, historical synchronous FUSE-T measurements, and Linux FUSE3 race performance | Implemented and verified for the shared core and macOS adapter; Windows runtime metrics remain open | +| `TF-009` | Journal recovery, generation recovery, service keep-alive, restart-safe retirement | Recovery tests, daemon restart canaries, managed Deep Idle sleep/wake, and actual retained-source host reboot | Partial: no actual power loss during an in-flight transaction | +| `TF-010` | Shadow compare, optimistic routes, retained snapshots, current-byte fallback | 90,000 real random-range comparisons, rollback and failure-containment canaries, and one bounded retained-source user-home canary | Implemented for macOS canaries; retention remains open | +| `TF-011` | `internal/enroll`, `fs enroll`, and the bounded standalone-service loop discover existing, new, and forked sessions, persist stability observations, take a fail-closed native-writer snapshot, and reuse fold/pack/migrate transactions | Policy tests plus a real writable-descriptor probe and isolated canonical FUSE enrollment, daemon restart, real CLI append, quarantine, and failed-cutover evidence | Implemented; real-home automatic apply remains disabled until platform promotion | +| `TF-012` | Shared Go core, Apple-native Swift FSKit on macOS, Linux FUSE3, and Windows WinFsp adapters | Native FSKit mounted behavior, current-client, crash, and rollback tests; historical FUSE-T evidence; real Linux adapter tests; and Windows cross-compiles | Partial at the cross-platform product level: macOS real-client acceptance passes, while all real Windows gates remain open | +| `TF-013` | Canonical capability type in `internal/fsctl/status.go` | Status rejection tests and CLI status tests | Implemented; current status is `fs-engine-preview` | +| `TF-014` | Snapshot retention and destructive-action guards | Migration, rollback, and quarantine tests | Implemented as a safety rule; retention promotion gates remain open | +| `TF-015` | Exact-version compatibility and update preflight quarantine | Unknown-version fallback and isolated canary tests | Implemented; the currently installed macOS CLI and Desktop are covered | +| `TF-016` | Native FSKit Host/extension packaging, non-elevating launchd, `systemd --user`, and Windows SCM lifecycle | Signed native FSKit app installation/rollback, launchd crash matrix, real Linux service, and Windows cross-compile evidence | Implemented; native release packaging and Windows runtime execution remain unverified | +| `TF-017` | Canonical namespace, write-sealed backing, mount identity, route normalization, daemon/supervisor locks, and build identity | Native FSKit mounted namespace and current-client tests, four-process crash matrix, app/binary rollback, real Linux FUSE3, and historical FUSE-T evidence | Partial at the cross-platform product level: isolated macOS current-client gates pass; power-loss, retention, and Windows gates remain open | +| `TF-018` | `internal/codex` spawn edges, `internal/family` graph/content evidence, `internal/archive` guarded transactions, and public `fork-family` plus `archive` commands | Diverse relationship fixtures, repeated-record performance regression, source-change rejection, official archive trace, native apply/recovery, and isolated managed FUSE-T archive/unarchive plus daemon restart | Implemented | +| `TF-019` | `internal/contain` and `internal/prune`; public `contains` and `remove-contained` commands | Exact containment, archived-only apply, transaction rollback, and recovery-manifest tests | Implemented | +| `TF-020` | Exact fold/migrate paths are byte-preserving; `repair-rollout` and `reconcile-rollout` write separate explicit outputs; a static production-import boundary prevents other workflows from invoking reconciliation | `internal/reconcile`, CLI behavior, and AST boundary tests | Implemented | +| `TF-021` | `internal/storage` provides physical inventory, configurable hard budgets, mutation preflight, generation and retired-state retention, lease-aware startup/explicit GC, and projected versus actual reclamation | Hard-link accounting, low-space refusal, lease retention, interrupted cleanup, repeated GC, cross-platform compile, and live read-only inventory evidence | Implemented; destructive retention remains promotion-gated | +| `TF-022` | Standalone CLI, daemon, launchd/systemd/SCM service management, configuration, storage, doctor, GC, rollback, and enrollment code | Public coupling scan and sanitization test | Implemented | + +## Implementation Plan Task Status + +| Task | Status | Evidence | Exact remaining scope | +| --- | --- | --- | --- | +| Task 1: packed object generation and resolver | Complete | Commit `17564e9`; `internal/pack` tests pass | None in Task 1 | +| Task 2: exact immutable virtual byte view | Complete | Commit `039e6b9`; exact and 10,000 random-read tests pass | None in Task 2 | +| Task 3: append and copy-on-write engine | Complete | Commit `9a7f1e8`; append, COW, writer, reopen, and interruption tests pass | None in Task 3 | +| Task 4: journal, compaction, and fallback | Complete | Commit `076d772`; recovery, compaction, and latest-byte fallback tests pass | None in Task 4 | +| Task 5: shadow, doctor, benchmark, and status | Complete | Commit `35a53fc`; focused and shared-core evidence exists | Real-platform promotion remains outside Task 5 | +| Task 6: compatibility and route transactions | Complete | Commit `5be10d2`; route race and exact-version tests pass | New client versions require new contracts, not a redesign | +| Task 7: neutral filesystem and tagged FUSE host | Complete | Commit `3f51aa5`; neutral, real macOS FUSE-T, and real Linux FUSE3 tests pass; Windows WinFsp cross-compiles | Windows real-adapter execution remains platform work | +| Task 8: standalone CLI and automatic enrollment | Complete | Command surface, guarded lifecycle, bounded planner/apply loop, native service arguments, and isolated real FUSE enrollment evidence | Production enablement remains outside Task 8 | +| Task 9: service lifecycle and update guard | Complete | Commit `4589ffa`; launchd, real `systemd --user`, Windows SCM compile, and preflight tests pass | Windows runtime and stronger automatic update claims remain release-gated | +| Task 10: synthetic, crash, performance, and compile gates | Complete for the shared engine | Commit `a1ac76e`; preview validation report | It cannot satisfy real-adapter or retention gates | +| Task 11: real macOS trace, adapter, shadow, and canary | Partial | Native FSKit build 102 mounted behavior, performance/coherency, bounded RSS, four-process and host-reboot recovery, atomic update rollback, and complete real current CLI/Desktop acceptance | Actual in-flight power loss and the incident-free retention window remain | +| Task 12: bounded automatic discovery and enrollment | Complete | Planner/apply/service tests plus isolated canonical FUSE enrollment, daemon restart, real CLI append, quarantine, and failed-cutover evidence | Real-home automatic apply remains platform-gated | +| Task 13: conservative branch lifecycle and content-change boundary | Complete | Spawn-edge family reports, exact relationship comparison, official-compatible guarded archive and recovery, separate exact-contained deletion, and static content-change boundaries pass unit, race, native, and managed FUSE-T validation | None in this task | +| Task 14: hard storage budgets, retention, cleanup, and accounting | Complete | Platform-neutral inventory, hard preflight, lease-aware bounded GC, truthful accounting, low-space and repeated-GC tests | Destructive retention remains platform-gated | +| Task 15: remaining platform and retention gates | Partial | Current macOS contracts and real-client acceptance; real Linux FUSE3 operation, crash, performance, backing policy, and systemd lifecycle; Windows WinFsp/SCM cross-compile | Actual in-flight power loss, retention period, Linux client/upgrade/rollback gates, and all real Windows gates remain | + +## Remaining Product Behavior And Exact Next Work + +| Missing behavior | Exact implementation work | Required verification | +| --- | --- | --- | +| Remaining macOS promotion gates | Perform an actual in-flight power-loss canary and complete the incident-free retention window only on disposable or explicitly approved data | Exact full-file SHA, deterministic journal recovery, no route loss, and the required incident-free window | +| Linux remaining readiness | Keep the implemented FUSE3 adapter and systemd lifecycle behind platform gates | Real Linux Codex traces, client upgrade quarantine, rollback, and retention | +| Windows readiness | Execute the implemented WinFsp adapter and SCM host without moving shared behavior out of the core | Native operations, crash/restart, performance, real Codex traces, upgrade quarantine, rollback, and retention on a real Windows host | + +## Current Capability Decision + +The shared storage, virtual-file, bounded automatic-enrollment, and physical-space governance engines are implemented and validated strongly enough for `fs-engine-preview`. Linux now has real adapter and native service evidence, while Windows remains implementation and cross-compile only. The repository still lacks the remaining macOS retention and disruptive gates, Linux real-client and lifecycle promotion gates, and all real Windows gates. Therefore: + +- Keep the capability at `fs-engine-preview`. +- Keep real user sessions native unless they are explicitly selected for a retained-source canary. +- Do not claim physical disk reclamation from logical deduplication alone. +- Keep real-home automatic enrollment disabled until the macOS promotion and retention gates pass, even though Tasks 12 and 14 are implemented. +- Do not introduce a private control-plane dependency into any public surface. diff --git a/docs/validation-fs-preview.md b/docs/validation-fs-preview.md new file mode 100644 index 0000000..010af44 --- /dev/null +++ b/docs/validation-fs-preview.md @@ -0,0 +1,62 @@ +# Filesystem Engine Preview Validation + +This document records platform-neutral storage and session-engine evidence. It does not claim a real FUSE adapter, real Codex compatibility, a macOS canary, or production readiness. + +## Required Commands + +```sh +./scripts/test-cross-platform.sh +CODEXFOLD_RUN_LARGE_TEST=1 go test ./internal/testfs -run TestLargePreviewBenchmark -count=1 -v -timeout 30m +``` + +## Covered Behavior + +- Deterministic synthetic forks with shared history, independently different tails, repeated fields, repeated records, non-prefix duplicate content, a multi-block field, invalid JSONL, and an empty session. +- Complete SHA comparison and 10,000 deterministic random reads for every synthetic session. +- 100,000 append operations followed by `fsync`, reopen, and exact current-byte verification. +- Concurrent readers with one writer, random write copy-on-write, truncate, and generation-safe reopen. +- Journal interruption and recovery tests in `internal/vfs`, packed corruption tests in `internal/pack`, and optimistic route-race tests in `internal/codex`. +- Linux and Windows non-CGO compile-only checks. Cross-compiled test binaries are not executed on macOS. +- A generated 758 MiB rollout read entirely from packs after the loose-object directory is taken offline. + +## Status Boundary + +Passing these checks can justify only `fs-engine-preview`. The following remain separate authorization-gated evidence: + +- Root `fs_usage` traces from real Codex Desktop and CLI versions. +- A compiled and mounted `fuse && cgo` adapter with macFUSE authorized by the user. +- Real archived-session shadow and retained-source canaries. +- Desktop click, resume, send, tool, fork, archive, restart, sleep/wake, rollback, and upgrade quarantine behavior. +- Seven incident-free retention days before `production-ready:macos`. + +## Latest Result + +Run on 2026-07-12 using an Apple M4 Pro MacBook Pro with 12 CPU cores and 48 GiB RAM, macOS 26.5.1, and Go 1.26.4. + +The 758 MiB source was deliberately highly repetitive. The cold pass requested and successfully applied macOS `F_NOCACHE` to both the native rollout file and every opened pack file, with an empty process-level decompressed block cache. It is a cache-bypass gate, not a disk-power-cycle or root-level system-cache purge. These values are deterministic engine evidence, not a claim that every real Codex workload has the same compression or reuse ratio. + +| Metric | Cold cache-bypass pass | Warm pass | +| --- | ---: | ---: | +| Native sequential throughput | 14.58 GB/s | 16.46 GB/s | +| Virtual sequential throughput | 34.56 GB/s | 54.69 GB/s | +| Virtual/native ratio | 2.37x | 3.32x | +| Random read p50 | 0.708 us | 0.625 us | +| Random read p95 | 1.041 us | 0.958 us | +| Random read p99 | 1.292 us | 1.250 us | + +Additional results: + +- Fold: 44.69 s. +- Pack build: 0.065 s. +- Complete SHA plus 10,000 random-range shadow: 3.51 s. +- Go system memory: 135.45 MiB. +- Maximum RSS: 135.70 MiB. +- User CPU for the complete heavy gate: 69.72 s. +- System CPU for the complete heavy gate: 15.46 s. +- Configured decompressed block cache: 128 MiB. +- Native `F_NOCACHE` applied during the cold pass: yes. +- Pack-file `F_NOCACHE` applied during the cold pass: yes. +- Loose-object directory offline during cold benchmark, shadow, and warm benchmark: yes. +- 100,000 one-byte append calls followed by `fsync`: 2.18 s in the normal test build. + +The platform-neutral gates pass and justify `fs-engine-preview`. This result does not satisfy any Task 11 real-adapter or real-Codex gate. diff --git a/docs/validation-linux-fuse3.md b/docs/validation-linux-fuse3.md new file mode 100644 index 0000000..2eeaf2f --- /dev/null +++ b/docs/validation-linux-fuse3.md @@ -0,0 +1,51 @@ +# Linux FUSE3 Validation + +## Status + +The Linux adapter has passed a real FUSE3 gate on Debian 12 as an unprivileged user. The same validation run used a race-enabled binary built with `CGO_ENABLED=1` and `-tags "fuse fuse3"`. The default non-CGo Linux build still compiles to the explicit prerequisite stub and does not silently select FUSE2. + +This evidence validates the Linux adapter and the `systemd --user` service lifecycle. It does not validate a real Linux Codex client, a client upgrade, the retention window, or a real Windows host. The project therefore remains `fs-engine-preview`. + +## Real Adapter Gate + +The race-enabled FUSE3 suite passed all of the following against disposable roots: + +- Exact managed reads, append plus `fsync`, random writes through complete copy-on-write, truncate, clean unmount, and remount. +- Canonical archive and unarchive renames, managed-over-native preference, native fallback, and managed-state removal. +- A separately hosted filesystem process killed with `SIGKILL`, strict detection of the resulting disconnected mount, `fusermount3 -uz` recovery, backing-directory resealing, and a successful replacement mount. +- A `0500` unmounted backing directory before activation and after every normal or crash-recovery shutdown. +- A dependency boundary that keeps `internal/fold` independent of Codex SQLite discovery and keeps `internal/mountfs` independent of `internal/codex`, `internal/service`, and all `modernc.org` packages. + +The latest race run measured a 16 MiB sequential managed read at `65.65 MiB/s` and 50 append-plus-`fsync` operations at `1.131011 ms` p95. These exceed the current Linux safety floors of `25 MiB/s` sequential read and `250 ms` append-plus-`fsync` p95. They are adapter gates, not universal hardware claims. + +## Real Systemd User Gate + +The generated unit passed `systemd-analyze --user verify`. A FUSE3-enabled CodexFold binary then completed this isolated lifecycle through the public CLI: + +1. `fs service install --apply` wrote the user unit, enabled it, started it, and returned only after both the process and CodexFold mount identity were healthy. +2. `fs service status` reported `daemon_running=true` and `mount_healthy=true`; the kernel exposed a `fuse` mount with source `codexfold`. +3. A direct `SIGKILL` of the running service process triggered the unit's restart policy. The replacement used a new main PID, recovered the disconnected FUSE mount, reset `ExecMainStatus` to zero, and returned `daemon_running=true` plus `mount_healthy=true` with exactly one restart. +4. `fs service start --apply` also stopped and restarted the unit explicitly, produced a new main PID, and restored a healthy mount. +5. `fs service stop --apply` removed the mount and left its ordinary backing directory at mode `0500`. +6. A CLI-level real FUSE regression now starts `fs serve`, kills it with `SIGKILL`, starts the same command against the stale mount, verifies recovery, then uses `SIGTERM` for a clean `0500` shutdown. +7. The validation unit, enable symlink, process, mount, build toolchain, and temporary data were removed after the gate. + +`systemd --user` must already be available to the invoking user. A headless machine that must start the user service before login may require an administrator to enable user lingering. CodexFold does not self-elevate or change linger policy. + +## Windows Boundary + +The Windows path currently includes a WinFsp-tagged host, Windows mount identity probe, SCM configuration renderer, native service installation and status commands, restart policy, and an SCM handler that runs the same in-process `fs serve` implementation and cancels it on stop or shutdown. Default and `winfsp` CLI, adapter, and test binaries cross-compile as PE32+ x86-64 executables. + +No real Windows plus WinFsp machine has executed mount, append, copy-on-write, rename, crash, service restart, performance, upgrade quarantine, or rollback tests. Windows remains implementation and compile evidence only. + +## Reproduction Gates + +```bash +go test ./... -count=1 +go test -race ./... -count=1 +go vet ./... +CGO_ENABLED=1 go test -tags fuse ./... -count=1 -timeout 5m +CODEXFOLD_RUN_FUSE3_TEST=1 CGO_ENABLED=1 go test -race -tags "fuse fuse3" ./internal/mountfs -count=1 -timeout 5m +CODEXFOLD_RUN_SYSTEMD_USER_TEST=1 go test ./internal/service -count=1 -timeout 2m +GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -tags winfsp ./cmd/codexfold +``` diff --git a/docs/validation-macos-canary.md b/docs/validation-macos-canary.md new file mode 100644 index 0000000..4a6c984 --- /dev/null +++ b/docs/validation-macos-canary.md @@ -0,0 +1,232 @@ +# macOS Adapter And Canary Validation + +## Native FSKit Development Checkpoint + +As of 2026-07-23, the terminal macOS validation candidate is the signed build 102 Apple-native Swift FSKit App/extension running the current Go helper through versioned binary UDS IPC. The transactionally installed isolated candidate passes mounted metadata and xattr behavior, append, `fsync`, `F_FULLFSYNC`, overlapping open lifetimes, random write, truncate, namespace and archive moves, mmap, open-unlink, external file and directory refresh, read-ahead coherency, eight-way virtual concurrent reads, exact-byte comparison, and managed service restart. Build-identity checks, environment sanitization, transactional app/definition/helper installation, and rollback also pass. + +The evidence below belongs to that completed build 102 App/extension and helper combination. A later App, extension, helper, registration, or upgrade-transaction change is a new candidate: it must repeat the installed-canary mount-health, byte-identity, current-client, restart, and rollback gates before this document can be used as evidence for it. In particular, an installed module must never be removed with `pluginkit -r`; candidate-registration cleanup uses LaunchServices only. + +The prior 8 KiB round-trip bottleneck is closed. Read-only native files use a capability-negotiated descriptor fast path, stable native fallback reads can stream without materializing a second Go payload, and virtual files use bounded concurrent read-ahead with generation-based invalidation. Cross-block requests avoid over-fetching beyond the negotiated frame limit, and old-generation prefetch completion cannot clear a new generation's in-flight state. Build 102 drains each closed handle's prefetch queue and uses eight concurrent 12 MiB blocks for a bounded 96 MiB horizon, so a following open does not compete with stale work or outrun the cold pipeline. The same 256 MiB virtual and retained-reference bytes matched by SHA-256 in every run. Earlier build 99/100 full-matrix cold samples at `0.695`, `0.659`, and `0.623` were rejected against the unchanged `0.70` gate, and a later build 101 sample at `0.703` was treated as insufficient margin. Five build 102 full matrices, each starting after an independent managed-service restart, passed at `0.762`, `2.042`, `0.800`, `1.761`, and `0.941`, with warm ratios of `0.954`, `1.011`, `0.963`, `0.993`, and `0.981` above the unchanged `0.80` gate. `F_NOCACHE` succeeded on both paths in every accepted cold round. These are observed same-run ratios rather than fixed throughput guarantees. + +The post-matrix aggregate RSS was 164,688 KiB: 91,472 KiB for the Go daemon, 43,248 KiB across two extension processes, 16,192 KiB across the two App wrappers, and 13,776 KiB for the supervisor. It remained below the 256 MiB acceptance bound after repeated full-file reads and service restarts. Killing and recovering the Go daemon, Host, FSKit extension, and supervisor independently preserved the complete 256 MiB SHA-256. Managed service restarts and one actual host reboot preserved the managed and native session branches. Every accepted recovery ended with all 11 `fs doctor` components healthy and zero issues. + +Sanitized operation traces from real isolated Codex CLI `0.144.3`, bundled CLI `0.145.0-alpha.30`, and Desktop `26.715.72359+5718` were imported as exact-version contracts. The current Desktop trace includes real history reads, parent writes and sync, UI fork creation, child writes and sync, namespace enumeration, metadata access, and release behavior. Four exact contracts are present, compatibility evaluation for the current bundled CLI and Desktop is approved without quarantine, and `fs doctor` reports all storage, route, client, daemon, mount, and recovery components healthy. + +The current candidate passed real managed CLI and Desktop resume, durable append, a real repository fix with `go test ./...`, the official CLI fork flow, the Desktop `Continue in new task from here` flow, parent/child isolation, official archive/unarchive, managed service restart, host reboot, full-history recovery, and post-restart continuation. Build 102 then resumed the same managed parent through the current bundled CLI, recovered the prior fix rationale, inspected the real source and tests, and ran `go test ./...` successfully. The final managed parent contained 919,038 bytes and 636 valid JSONL records. Its original 393,640-byte folded base and every previously recorded full-file prefix remained byte-identical while all later writes stayed in the append delta; no writable backing was created. The build 102 Desktop app-server used the isolated Codex home and Electron data directory, and twice read the complete 919,038-byte managed parent without changing its SHA-256. The independently writable native Desktop child contained 882,688 bytes and 607 valid records, matched its native backing exactly, and had no managed-session state. The parent's database update preceded the child creation, the child turn did not update the parent, and branch-specific markers never crossed back into the parent. + +One intentionally interrupted slow-provider CLI turn emitted a client-local rollout-writer `EIO` before Codex reopened the file. The daemon trace contains no failed write for that event; all retry appends and the final `fsync` succeeded, the pre-interruption full-file SHA-256 remained an exact prefix, and all resulting records parsed. A subsequent normal real CLI turn completed without the warning. This is retained as interruption evidence, not counted as a clean client pass. Production activation, production service loading, and real-home migration remain disabled; actual in-flight power loss and the incident-free retention window remain open macOS promotion gates. + +The FUSE-T evidence below is retained as historical compatibility and regression evidence. It is not the terminal architecture and must not be used to claim Apple-native FSKit readiness. + +## Historical FUSE-T Status + +The FUSE-T macOS adapter and isolated real Codex CLI and Desktop canaries have passed for read, append, resume, fork, child-session enrollment, canonical archive/unarchive moves, launchd restart, rollback, namespace deactivation, and unknown-version quarantine. A retained-source CLI canary survived an actual host reboot while managed, then resumed through the recovered mount and rolled back to an exact native JSONL. The currently installed CLI and Desktop versions also passed exact compatibility and isolated retained-source canaries. Process-level interruption recovery now covers append, compaction, migration, and rollback, and a managed session passed an actual Deep Idle sleep/wake cycle followed by a real model turn. The user Codex home now uses the canonical namespace with ordinary sessions remaining native passthrough and one explicitly selected retained-source canary managed for observation. The project remains at `fs-engine-preview` because that canary has not completed retention, in-flight transaction evidence does not claim an actual power-loss test, and the seven-day incident-free gate has not completed. + +Backend selection evidence on 2026-07-17: + +- The installed FUSE-T `1.2.7` FSKit extension was enabled through the official macOS settings UI and mounted only disposable paths under `/private/tmp` and Go test temporary directories. `statfs` reported filesystem `fuse-t`, source `file:///private/tmp/fuset-session-*/session.json`, and the `fskit` mount flag, proving that the test did not fall back to NFS. +- The FSKit backend passed exact reads, ordinary append, `fsync`, `F_FULLFSYNC`, random write, truncate, copy-on-write, unmount, remount, and ten consecutive performance mounts. Those ten runs read about 2.0-2.7 GiB/s, or 25-38% of their same-run APFS baselines, with append-plus-`fsync` p95 between about 5.7 and 12.0 ms. +- The FSKit backend failed the deterministic stale-offset correctness gate. Two complete records written through one descriptor at the same previous EOF produced only the second record; the visible result was `record 0, record 2` instead of `record 0, record 1, record 2`. The transport delivered a coalesced write and offers no equivalent of the NFS synchronous-mount correction. +- Three managed-to-native route transition tests also failed to expose the new bytes within five seconds. This matches FUSE-T's published limitation that its FSKit backend does not support FUSE notifications. Requiring an unmount/remount for rollback or route invalidation is incompatible with transparent active-session behavior. +- FUSE-T FSKit is therefore rejected, despite passing basic operations and acceptable throughput. The shipped macOS path remains explicitly selected FUSE-T NFS with `MNT_SYNCHRONOUS`; FSKit is not an automatic fallback. +- After removing the rejected backend and retaining `F_FULLFSYNC`, pure-managed `statfs`, and nested-mount scan protections, the final complete real NFS adapter suite passed in 9.168 seconds. In that run, mounted read throughput was 3.61 GiB/s versus 10.31 GiB/s on the same-run APFS baseline, and append-plus-`fsync` p95 was 5.006 ms versus 3.990 ms. These cache-sensitive figures establish ample headroom, not a fixed APFS percentage guarantee. + +Additional conservative branch-lifecycle evidence on 2026-07-16: + +- The current official CLI archive transaction was traced in a disposable Codex home. It moves the rollout byte-for-byte into the flat `archived_sessions` directory, sets `archived` and `archived_at`, advances only `updated_at_ms` using the database maximum plus one, preserves `updated_at`, updates `rollout_path`, and leaves global UI state unchanged. +- `fork-family show` reports the explicit spawn-edge component and active/archive state. `fork-family compare` keeps graph ancestry separate from exact content evidence and distinguishes identical applicable records, complete left/right containment, shared-prefix independent tails, other exact shared records, and unknown relationships. It never infers usefulness from age, title, size, or ancestry. +- Family comparison opens each rollout once, uses cursor-based duplicate matching, and rejects size, modification-time, or file-identity changes before returning evidence. A 1,000-identical-record regression completes within its bounded test context rather than performing quadratic file reopens. +- `archive` is dry-run-first. Apply requires the native writer probe, revalidates SQLite route and complete source SHA-256 inside an immediate transaction, writes a durable prepared/renamed journal, preserves official archive field behavior, and rolls back file and database state on failure. If commit acknowledgement is ambiguous, it leaves the exact target and journal intact instead of guessing; explicit recovery then either rolls back or finalizes from observed state. +- A disposable native rollout archived with an unchanged complete SHA-256 and no residual journal. A second disposable valid Codex session was folded, packed, migrated through the synchronous canonical FUSE-T namespace, officially unarchived, archived with CodexFold, and read back with the same complete SHA-256. After a complete filesystem-daemon stop and restart, the official CLI unarchived the same managed bytes again. Final rollback, service stop, and namespace deactivation left no validation mount or process behind. +- A static production-import regression allows the content-changing reconciliation package only in the explicit `fs_reconcile.go` CLI boundary. Fold, migration, compaction, enrollment, rollback, GC, archive, family reporting, and exact-contained deletion cannot invoke repair or reconciliation implicitly. +- Default, race, vet, FUSE-tagged, complete real FUSE-T, real Linux FUSE3 race, real Linux `systemd --user`, Linux/Windows default build, Windows WinFsp/SCM cross-compile, public-sanitization, and diff checks pass with Task 13 complete. The validation used only disposable homes and did not update the installed binary, the production daemon, or any real user session. Linux evidence and its remaining boundary are recorded separately in [the Linux FUSE3 validation](validation-linux-fuse3.md). + +Additional bounded automatic-enrollment evidence on 2026-07-16: + +- The standalone FUSE service ran a bounded periodic enrollment loop against a fresh isolated canonical Codex home. The first cycle persisted a stability observation; a later unchanged cycle selected one archived session and reused the public `fold`, `pack build`, and canonical `fs migrate` transactions rather than a separate migration implementation. +- The first empty-store run exposed and fixed a bootstrap gate: the absence of a committed pack generation is valid before the first enrollment only while no managed session state exists. A missing or broken committed generation still blocks enrollment. +- The service now takes one batched native-writer snapshot per planning cycle. A real isolated rollout held open through a writable descriptor was reported as `writer-active`; probe failure blocks planning instead of assuming that no writer exists. +- Automatic enrollment completed exact shadow verification, 10,000 random-range comparisons, retained one immutable native snapshot, received the exact mount acknowledgement, and preserved the canonical SQLite route. The visible file and retained snapshot matched the original SHA-256; generation remained 1 with an empty delta and no writable backing. +- After a complete daemon stop and restart, the synchronous FUSE-T mount restored the same visible bytes. Repeated policy cycles reported the session as already managed and created neither a second snapshot nor another managed state. +- The current unmodified Codex CLI unarchived and resumed the automatically enrolled session. The original 68,707-byte base prefix remained exact, the new 5,819 bytes were durable only in `append.delta`, the complete 74,526-byte JSONL parsed, and no copy-on-write backing appeared. +- An unknown-client compatibility canary materialized the latest 74,526 visible bytes, verified their SHA-256, routed SQLite to a normal native quarantine fallback, and retired managed state. It did not fall back to the stale migration snapshot. +- A separate isolated failure canary used a healthy noncanonical mount so canonical acknowledgement could never arrive. Migration timed out, left the SQLite route unchanged, restored the exact native source, removed the candidate snapshot, and left no active managed state. +- The lease regression discovered by the full real FUSE-T suite was fixed at the source: reader and resolver leases can no longer recursively recreate a retired session or missing pack generation. The failing same-path republish case then passed three consecutive real FUSE-T runs, followed by the complete real adapter suite. +- Default tests, FUSE-tagged tests, full race tests, `go vet`, real FUSE-T tests, real Linux FUSE3 and `systemd --user` gates, and Linux/Windows cross-platform build gates passed after these changes. Windows remains compile-only. Automatic apply remains disabled for the real Codex home while capability is `fs-engine-preview`. + +Additional synchronous-write and canonical-activation evidence on 2026-07-16: + +- Canonical activation preserved all 2,313 native rollouts. The mounted and native path/size/mtime inventories matched exactly, and the pre/post underlying native inventory also matched path, size, mtime, inode, mode, owner, and group. Explicit critical canaries retained full SHA-256 checks. Ordinary sessions remained native passthrough and the managed-session count stayed zero. +- The one-shot background activation job completed the namespace switch but failed before reopening Desktop because macOS denied that LaunchAgent a recursive traversal of the network-volume-backed mount. Foreground Codex processes could traverse the same mount. The activation script now inventories the underlying native tree instead of recursively hashing or traversing the mounted tree, rechecks Desktop, CLI, and app-server quiescence immediately before activation, and retries preflight if Codex reopens. The user reopened Desktop manually; no rollout content was changed by the failed reopen step. +- The first dedicated real-home migration passed exact shadow verification and 10,000 random reads, but a real CLI turn exposed a corruption bug: the original 97,388-byte prefix remained exact while the final JSONL contained a partially overwritten record. The canary was rolled back and restored byte-for-byte before further work. +- The real write trace showed that Codex opens rollout JSONL with `O_RDWR` and explicit offsets. A same-handle JSONL append guard was added, but the failing FUSE-T regression proved that the macOS NFS client could merge two same-offset `pwrite` calls before either reached CodexFold. Per-open and global libfuse `direct_io`, plus disabled NFS attribute caching, did not change that behavior. +- Updating only the mounted localhost NFS volume with `mount -u -o sync` made the previously deterministic stale-offset regression pass. The Darwin adapter now withholds its health identity until that update succeeds and `MNT_SYNCHRONOUS` is visible through `statfs`; a failure unmounts the host instead of advertising readiness. No global NFS configuration, patched FUSE-T binary, privileged helper, or system-wide mount change is used. +- One historical warm-cache 64 MiB run measured 7,045 MiB/s versus 7,435 MiB/s from APFS, or 95%. That ratio is not a general performance guarantee: later same-run comparisons varied materially with APFS cache speed. The enforced gate is mounted throughput of at least 1 GiB/s and at least 25% of the same-run APFS baseline, plus bounded append-and-sync latency. +- A fresh isolated real CLI canary used the official unarchive flow and then resumed through the synchronous canonical mount. The complete view grew from 97,388 to 120,859 bytes and 33 valid JSONL records. The complete original prefix retained SHA-256 `4cd4bcc1807d875b70e04b3028441f330f9c7ee0cd41cbcff08c18c9ec44d416`, the 23,471-byte delta parsed independently, the expected historical and new markers were recalled, generation remained 1, and no writable backing appeared. +- The same dedicated canary was then enrolled in the canonical user home while every ordinary rollout remained native passthrough. A real current CLI unarchive and resume produced a 120,864-byte, 33-record valid JSONL with the same exact 97,388-byte prefix SHA-256, a separately valid 23,476-byte delta, generation 1, and no writable backing. The model recalled the historical marker and emitted the new acceptance marker. Official archive moved the managed route back to `archived_sessions`; exactly one managed session remains, the mounted volume reports synchronous I/O, and the complete filesystem doctor is healthy. +- Default, FUSE-tagged, race, vet, shell, cross-platform compile, and complete real FUSE-T suites passed after the fix. The real FUSE-T suite explicitly requires synchronous mount readiness before exercising the stale-offset regression. + +Additional current-client, interruption, and sleep/wake evidence on 2026-07-15: + +- Exact contracts were imported and approved for PATH `codex-cli 0.144.3`, Desktop `26.707.72221+5307`, and the Desktop-bundled app server `0.144.2`. The current Desktop opened an isolated managed task, displayed the complete native and managed history, completed a real model turn, survived forced termination and restart, rolled back exactly, and resumed natively. +- A canonical migration process and the FUSE daemon were both terminated immediately after managed state became durable but before cutover. Launchd started a fresh daemon, startup recovery retired the incomplete state, and both the retained native file and mounted native view retained the same complete SHA-256. A separate regression proves that a noncanonical migration failure also retires state created by that failed attempt. +- During a real CLI append, the FUSE daemon was terminated after a 9,291-byte delta prefix was durable. Codex observed one `EIO`, reopened its rollout writer, retried, and completed the turn. The durable prefix remained byte-identical, the complete 86-record JSONL parsed, no writable backing appeared, and a later real resume recalled the interrupted turn. +- Compaction now acquires the cross-process writer lease in addition to the in-process writer state. A termination before state publication recovered by rolling back the candidate generation and removing the journal-owned candidate delta, scratch file, and state temporary. A second termination after atomic state publication recovered by completing the candidate generation. Both sides preserved the same exact 132,510-byte visible SHA-256, and a later real resume recalled pre-compaction history and appended through the new delta. +- Canonical rollback was paused after the daemon acknowledged a verified native target, then both rollback and daemon processes were terminated. A fresh daemon preserved a complete readable route and the pending request. Re-running the same rollback reused the exact token only because generation, route, byte count, and SHA-256 still matched; it then retired managed state and cleared the control files. The resulting 136,514-byte, 104-record native JSONL resumed successfully. +- macOS power logs recorded entry into Software Sleep and wake from Deep Idle. Across that cycle, the same daemon PID and FUSE mount remained healthy, and the managed view stayed exactly 144,580 bytes, 122 valid records, a 4,018-byte delta, and the same SHA-256. A real post-wake resume recalled the pre-sleep turn and produced a 148,547-byte, 131-record view with a 7,985-byte delta and no writable backing. +- These interruption runs used real FUSE-T, real launchd restarts, and unmodified Codex clients. They validate deterministic recovery from simultaneous client/control-process and daemon termination. They do not represent an actual power loss during an in-flight transaction. + +Additional host-restart evidence on 2026-07-15: + +- The isolated canary used PATH `codex-cli 0.144.3` with a real Responses model turn. Its current version contract was exact and approved before migration. The installed Desktop `26.707.72221+5307` was not used in this run and remains outside this evidence. +- A native parent was created and resumed before folding. Its 85,776-byte, 49-record baseline was folded, packed, and reconstructed with an exact SHA-256 plus 10,000 successful random-range comparisons while the source was retained. +- The managed parent resumed and appended through `append.delta` without a writable backing. After a standalone daemon restart, another real turn recalled the prior managed turn and appended again. The complete base prefix remained byte-identical. +- A real CLI fork created an ordinary native child while the parent remained managed. The child and parent then resumed independently; marker checks and complete-file hashes showed no cross-branch writes. +- The first rollback restored the exact 107,091-byte managed view to an ordinary JSONL. A native resume recalled the managed history and appended successfully, producing 111,271 bytes and 97 valid JSONL records. +- The updated parent was folded and migrated again before an actual macOS reboot. After login, launchd recreated both the standalone process and FUSE-T mount. Before any new turn, the recovered managed view was exactly 111,271 bytes, 97 records, and SHA-256 `98369563df4766c50d4d8886c8dff8163471e19d670325a2ef1190537064cbba`, with an empty delta and no writable backing. +- A real post-reboot resume recalled the latest native turn and appended 4,255 bytes through the managed delta. The 111,271-byte base prefix kept the same SHA-256, the complete visible file became 115,526 bytes and 106 valid records, and no writable backing appeared. +- Post-reboot rollback materialized the exact 115,526-byte visible view. A final native resume recalled the managed post-reboot turn and appended successfully. The parent ended at 119,678 bytes and 115 valid records with SHA-256 `e891263cbdfcbe9f149138eb94bfb4371afee0424b20d4afa1d261954dde5144`; the child remained unchanged at SHA-256 `76b19d4c5b0d1b41b312dadfee3b9b871165da2dac58ba47b1e60c1e1cb102bd`. +- Namespace deactivation restored ordinary `sessions` and `archived_sessions` directories. Both SQLite routes point to native JSONL files, every record parses, all expected parent markers remain in order, and parent/child marker isolation still passes. +- This proves idle retained-source managed-session recovery across one actual host reboot. It does not cover a reboot or power loss during an active append, compaction, migration, or rollback transaction, and it does not authorize enrollment of the real Codex home. + +Additional failure-containment evidence on 2026-07-14: + +- After an actual host reboot, launchd started a fresh CodexFold process and the FUSE-T mount identity was healthy. The store contained zero managed session states, ordinary user sessions remained native, and status reported `fs-engine-preview`. This proves service and mount boot recovery only; it does not satisfy managed-session host-restart recovery. +- The post-reboot `fs doctor` check found daemon, mount, backing, delta, fallback, journal, manifest, pack, and route components healthy. It remained unhealthy overall because the currently installed Codex clients do not yet have exact compatibility contracts. +- Canonical rollback now uses a two-stage retirement request and acknowledgement. The daemon keeps the managed session loaded while preferring a verified native target, so removing or changing that target falls back to managed bytes instead of creating an `ENOENT` window. +- A live pending-retirement restart loaded the managed fallback into a fresh daemon, acknowledged the exact generation and route, and preserved the complete SHA-256. Toggling the native target 100 times while opening the mounted route 2,000 times produced zero read failures. +- A second live restart began with an earlier successful acknowledgement after the native target had disappeared. The fresh daemon replaced it with `native rollback target is unavailable or changed`, remained running, and exposed the complete managed JSONL with the same SHA-256. +- A normal rollback completed with zero route-read failures, preserved the exact visible SHA-256, then passed archive, overwrite-fold, pack rebuild, 10,000-range shadow verification, canonical re-migration, unarchive, and a complete FUSE service restart. +- The isolated real Codex task resumed after that restart and performed a repository review, added and mutation-tested a restart regression, ran Go and race tests, and wrote a detailed verdict. Its mounted rollout grew from 1,226,174 to 1,579,063 bytes; the complete 1,226,174-byte prefix retained SHA-256 `eff00f0583833b1d9cb03b12ed5b19cb68240c37e953c086f028e8bc6a4de2f6`, and all 746 JSONL records parsed. +- Recovery before retirement is covered explicitly: the rollback request uses the recovered `managed.State().Generation`. A fresh-daemon regression also verifies that a stale successful acknowledgement is replaced with a rejection when its native target is no longer valid. +- Exact contracts were imported from real FUSE traces for PATH CLI `0.144.3` and Desktop `26.707.71524+5263`. +- Canonical migration now verifies the mounted managed target before removing the native directory entry. A clean first migration passed without a retry. +- A real Desktop canary preserved the exact 79,067-byte, 16-record source prefix, appended an 8,414-byte, 12-record managed delta, and rolled back to their exact 87,481-byte concatenation. A subsequent native Desktop turn appended 5,590 bytes and 9 records. The final 93,071-byte, 37-record JSONL parsed completely and preserved byte order. +- Rollback now holds an exclusive writer lease across materialization and state retirement. A live FUSE writer and a real Desktop app-server both caused rollback to fail closed; after every writer drained, rollback preserved the exact visible SHA-256 and a new Desktop turn persisted to the native JSONL. +- The canonical activation gate compares every native rollout's path, size, modification time, inode, mode, owner, and group before and after the directory rename. Explicit critical sessions retain full SHA-256 comparisons. Mounted visibility is checked separately after activation because a background LaunchAgent can be denied recursive traversal of the FUSE-T mount even while the foreground Codex client can use it normally. This avoids holding Codex closed while reading the complete session corpus without falling back to file-size-only evidence. + +Additional failure-containment evidence on 2026-07-13: + +- The mount backing directory rejects symlinks and any ordinary files before the host starts. +- The ordinary backing directory is mode `0500` while unmounted, contains no namespace entries, and rejects attempts to create a session branch. +- A live mount exposes a per-process random `.codexfold-health` generation. The mount probe requires both the supported FUSE provider and a successful read of that identity. +- Canonical namespace activation rejects a plain directory even when it contains `sessions` and `archived_sessions` look-alikes. +- A store-wide advisory process lock rejects a second filesystem host. +- Service install and start wait for both a running launchd process and a readable mount identity; failed readiness is booted out rather than reported as success. +- An isolated launchd canary passed canonical activation, transparent native passthrough, clean stop, write-sealed downtime, restart, `SIGKILL`, automatic PID replacement, post-restart append, final stop, and namespace deactivation. +- Codex Desktop `26.707.61608+5200` was observed resolving the canonical `sessions` symlink to `fold-fs/sessions` and writing that real path back to SQLite. Without a guard, the route watcher exited and Desktop later removed the stale thread row. Canonical activation now installs synchronous SQLite insert/update triggers that normalize both active and archived mount aliases back to `CODEX_HOME`, including Unicode paths. +- The route watcher independently accepts exact mount aliases and maps them to the same virtual route instead of terminating the filesystem service. +- A repaired isolated Desktop canary opened the complete history, appended `CODEXFOLD_ROUTE_GUARD_RESTART_OK`, survived a forced Desktop `SIGKILL`, reopened the same task, displayed the marker, retained the canonical SQLite path, and kept the FUSE service running. +- The sanitized Desktop trace was imported as the exact `26.707.61608+5200` contract with `statfs`, `getattr`, `read`, `open`, `rename`, `utimens`, `readdir`, `write`, `fsync`, `flush`, and `release`. Together with PATH `codex-cli 0.144.1`, compatibility evaluation is approved and not quarantined. +- The configured third-party Responses provider rejected model turns because its hosted image tool conflicted with the local `image_gen.imagegen` tool. The user append, filesystem durability, restart, and route normalization checks completed before that provider-level rejection; no model-reply claim is made for this focused canary. + +Observed on 2026-07-12: + +- Codex desktop bundle: `26.707.51957` build `5175`. +- Desktop-bundled CLI: `codex-cli 0.144.0-alpha.4`. +- CLI resolved from `PATH`: `codex-cli 0.142.5`. +- Exact-version contracts passed together for the desktop-bundled CLI, the PATH CLI, and Desktop. +- FUSE adapter: FUSE-T `1.2.7`; macFUSE is not required or supported by this validation route. +- `CGO_ENABLED=1 go test -tags fuse ./...`: passed. +- The real FUSE-T fixture tests passed mount, list, stat, exact reads, reopen, EOF append, fsync, random-write copy-on-write, truncate, unmount, remount, and canonical managed rename in both directions. +- Managed extended attributes use hidden native carrier files, and AppleDouble sidecars follow managed archive/unarchive moves in both directions. +- Canonical mode exposes `sessions/YYYY/MM/DD/...` and `archived_sessions/...` in one FUSE-T namespace while passing unmanaged and newly created rollouts through a native backing tree. +- Canonical mode rejects a missing or relative native backing root instead of interpreting an empty root as the current working directory. +- The route watcher reads Codex state once per polling cycle and only updates a mounted session when its generation or canonical route changes. +- Canonical migration uses a two-phase cutover: it stages a hidden hard-link or cross-volume copy, waits for a daemon acknowledgement for the exact generation and route, then removes the native directory entry. A failed cutover retires the managed state and discards only the staged copy. +- The canonical migrate and rollback commands wait up to 15 seconds by default, but return immediately after the exact target bytes are verified. +- A session added to the store after mount became readable through on-demand loading without remounting. +- An equal-length truncate from real Codex remained a no-op and did not hydrate a writable backing file. +- A real fork from a virtual parent created a native child session; parent and child then resumed independently without cross-contamination. +- The child was archived while native, folded and packed, then enrolled through the mounted filesystem after remount. +- After an isolated database-only archive-flag reset, the enrolled child resumed through its virtual path and appended successfully. That reset remains historical evidence only; canonical mode no longer needs it for archive/unarchive. + +## Real Shadow Evidence + +Nine archived real sessions were copied into an isolated validation store without changing their Codex routes or deleting their source files. The set covered small and medium sessions, one session around 23 MiB, and one real fork parent/child pair. + +Results: + +- 9 of 9 complete-file SHA-256 comparisons passed. +- 90,000 of 90,000 random-range comparisons passed. +- The generated pack contained 1,182 objects. +- Pack doctor reported zero issues. +- A 2026-07-13 direct read-only scan of one current archived rollout processed 538,542 bytes and 192 records with zero invalid JSON records, zero missing sessions, and no file change during the scan. + +These results validate exact reconstruction and random reads. They do not validate Desktop behavior or long-running service reliability. + +## Isolated Real Codex Canary + +The canary used an isolated Codex home and state database. It did not modify the user's real Codex routes. The final full-flow run used the desktop-bundled `codex-cli 0.144.0-alpha.4` and a clean isolated root; the later focused route-guard run used Desktop `26.707.61608+5200` and PATH `codex-cli 0.144.1`; the host-restart run used PATH `codex-cli 0.144.3`. `scripts/prepare-isolated-codex-home.sh` copies the current `config.toml`, `auth.json`, and optional `models_cache.json` byte-for-byte and uses APFS clones for static plugin assets, so the canary uses the current provider configuration without allowing canary writes to modify the source home. + +The validated sequence was: + +1. Start the real FUSE-T filesystem service. +2. Migrate one retained-source session through the product command. +3. Verify the complete mounted file and 10,000 random ranges. +4. Route only the isolated SQLite record to the mounted JSONL. +5. Resume with the unmodified desktop-bundled Codex CLI and append through `append.delta` without creating a complete backing file. +6. Stop, remount, resume again, run a shell tool, and append again without creating a complete backing file. +7. Roll back to a verified ordinary JSONL containing the latest visible bytes. +8. Resume and append successfully from that native fallback. +9. Repeat canonical migration with the two-phase daemon acknowledgement, then verify the source entry is removed only after the matching mounted route is live. + +The additional fork sequence was: + +1. Route the parent to the mounted virtual JSONL. +2. Run the unmodified CLI `fork` command with a real prompt. +3. Confirm the child was created as an ordinary native rollout while the parent stayed virtual. +4. Resume the native child and the virtual parent separately. +5. Archive the native child, fold and pack it, remount, and migrate the child through the real CLI route. +6. Resume the migrated child through the virtual path and verify parent/child append isolation by file size and marker content. + +The Desktop sequence was: + +1. Prepare the isolated Codex home from the current `config.toml` and `auth.json`, verify both SHA-256 values match before launch, and clone static plugin assets with copy-on-write isolation. +2. Start a separate Desktop process with an isolated Electron data directory and verify its child app-server has the isolated `CODEX_HOME`. +3. Open the managed session through `codex://threads/` and verify the virtual history is displayed. +4. Send a real message through the Desktop composer and verify the reply appears in the UI and only the append delta grows; no complete writable backing file is created. +5. Use Desktop's `Continue in new task from here` action, continue the child, and verify the child is native while the virtual parent remains unchanged by the child marker. +6. Import the sanitized Desktop operation trace as the exact `26.707.51957+5175` compatibility contract. + +A separate provider check started the unmodified desktop-bundled CLI with the prepared isolated home. The provider's model endpoint returned its non-OpenAI model catalog, the rollout recorded the configured provider and model, and the Responses request completed with `CODEXFOLD_FIXED_PROVIDER_OK`. Codex may rewrite home-relative plugin paths inside the isolated copy after startup; it did not replace the configured model provider. + +The canonical namespace sequence was: + +1. Expose the isolated Codex home's `sessions` and `archived_sessions` directories through one FUSE-T mount, with a separate native backing tree for unmanaged files. +2. Start with a managed parent at its canonical archived route. +3. Run the official `codex unarchive ` command and verify the SQLite route, archived flag, and mounted path moved to `sessions/YYYY/MM/DD/...`. +4. Resume the unarchived session with the unmodified desktop-bundled CLI and receive the requested canary marker. +5. Run the official `codex archive ` command and verify the route returned to `archived_sessions/...` with no active managed JSONL and no native JSONL duplicate. +6. Stop and restart the filesystem service, then repeat unarchive, resume, and archive successfully while the destination date directories exist only in the native backing tree. +7. Roll back the managed parent to a native JSONL, resume it through the unmodified CLI, stop the service, deactivate the namespace, and resume again from ordinary directories. + +The adapter exposes `setxattr`, `getxattr`, `listxattr`, and `removexattr` for managed files through hidden native carrier files. macOS AppleDouble metadata sidecars are moved with their managed rollout during archive and unarchive. The real FUSE-T integration test verifies that the xattr survives both moves and that stale sidecars do not remain at the old route. + +The rollback safety regression also covers a native fallback that becomes newer than managed state. Unknown-version quarantine must preserve that current native route and must not overwrite it with stale managed bytes. + +The unknown-version canary used a fourth clean isolated home. A fake `codex-cli 9.9.9` triggered quarantine, materialized current bytes into `fs/fallbacks//quarantine-current.jsonl`, updated SQLite to that ordinary JSONL, retired the managed state, kept the daemon and mount healthy, and allowed namespace deactivation without changing the fallback route. + +The per-user launchd service was installed against an isolated home. It recovered a healthy FUSE-T mount after both `SIGTERM` and `SIGKILL`. Service installation initially blocked because `launchctl kickstart -k` waited for the old FUSE process; the lifecycle now uses non-destructive `kickstart`, returns promptly, and reports daemon and mount health separately. + +A direct `SIGTERM` stopped the foreground service and removed the mount cleanly. A PTY `Ctrl-C` experiment left a reparented process once; this was a test-harness behavior and is not used as lifecycle evidence. + +## Remaining Gates + +The following gates are still open: + +- Completion of the dedicated retained-source user-home canary retention window. +- An actual power-loss or host-restart interruption while a transaction is in flight; simultaneous process termination and a separate idle managed-session host reboot have passed, but they are recorded as distinct evidence. +- Seven incident-free days after reaching `platform-canary`. + +Until every applicable gate passes, the project must keep the capability at `fs-engine-preview`, retain original JSONL files, and avoid changing real Codex routes. + +## Reproducible Test Commands + +```sh +go test ./... +CGO_ENABLED=1 go test -tags fuse ./... -count=1 -timeout 5m +go test -race ./internal/mountfs ./internal/vfs ./internal/cli ./internal/service +CODEXFOLD_RUN_FUSE_TEST=1 CGO_ENABLED=1 \ + go test -tags fuse ./internal/mountfs -count=1 -v +``` diff --git a/go.mod b/go.mod index d2cada0..9ecbd8b 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,12 @@ -module github.com/jstar0/codexfold +module github.com/samekind/codexfold go 1.26 require ( github.com/klauspost/compress v1.19.0 github.com/spf13/cobra v1.10.2 + github.com/winfsp/cgofuse v1.6.0 + golang.org/x/sys v0.36.0 modernc.org/sqlite v1.40.1 ) @@ -17,7 +19,6 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/spf13/pflag v1.0.9 // indirect golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect - golang.org/x/sys v0.36.0 // indirect modernc.org/libc v1.66.10 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go.sum b/go.sum index 1db7c0c..f642994 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/winfsp/cgofuse v1.6.0 h1:re3W+HTd0hj4fISPBqfsrwyvPFpzqhDu8doJ9nOPDB0= +github.com/winfsp/cgofuse v1.6.0/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= diff --git a/internal/archive/archive.go b/internal/archive/archive.go new file mode 100644 index 0000000..37c1184 --- /dev/null +++ b/internal/archive/archive.go @@ -0,0 +1,638 @@ +package archive + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/samekind/codexfold/internal/codex" + _ "modernc.org/sqlite" +) + +type Options struct { + Apply bool + Now time.Time + WriterActive func(context.Context, codex.Session) (bool, error) + BeforeRename func() error + AfterRename func() error +} + +type Result struct { + SessionID string `json:"session_id"` + SourcePath string `json:"source_path"` + TargetPath string `json:"target_path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + DryRun bool `json:"dry_run"` + Archived bool `json:"archived"` +} + +type RecoveryResult struct { + SessionID string `json:"session_id"` + SourcePath string `json:"source_path"` + TargetPath string `json:"target_path"` + RolledBack bool `json:"rolled_back"` + Finalized bool `json:"finalized"` +} + +type phase string + +const ( + phasePrepared phase = "prepared" + phaseRenamed phase = "renamed" +) + +type journal struct { + Version int `json:"version"` + SessionID string `json:"session_id"` + SourcePath string `json:"source_path"` + TargetPath string `json:"target_path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + Phase phase `json:"phase"` +} + +type snapshot struct { + Bytes int64 + SHA256 string +} + +type threadRow struct { + RolloutPath string + Archived bool +} + +var commitArchiveTransaction = func(ctx context.Context, conn *sql.Conn) error { + _, err := conn.ExecContext(ctx, `commit`) + return err +} + +func JournalPath(store string, sessionID string) string { + return filepath.Join(filepath.Clean(store), "archive", "journals", sessionID+".json") +} + +func Archive(ctx context.Context, home string, store string, session codex.Session, options Options) (Result, error) { + if !validSessionID(session.ID) || !filepath.IsAbs(home) || !filepath.IsAbs(store) || !filepath.IsAbs(session.RolloutPath) { + return Result{}, errors.New("absolute home, store, rollout path, and safe session ID are required") + } + home = filepath.Clean(home) + store = filepath.Clean(store) + sourcePath := filepath.Clean(session.RolloutPath) + if session.Archived { + return Result{}, errors.New("session is already archived") + } + if _, err := relativeWithin(filepath.Join(home, "sessions"), sourcePath); err != nil { + return Result{}, errors.New("active rollout is outside the Codex sessions directory") + } + targetPath := filepath.Join(home, "archived_sessions", filepath.Base(sourcePath)) + if sourcePath == targetPath { + return Result{}, errors.New("archive source and target must differ") + } + current, err := hashPath(sourcePath) + if err != nil { + return Result{}, fmt.Errorf("hash active rollout: %w", err) + } + result := Result{ + SessionID: session.ID, SourcePath: sourcePath, TargetPath: targetPath, + Bytes: current.Bytes, SHA256: current.SHA256, DryRun: !options.Apply, + } + if _, err := os.Lstat(targetPath); err == nil { + return Result{}, errors.New("archive target already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return Result{}, err + } + db, err := openStateDB(home) + if err != nil { + return Result{}, err + } + defer func() { _ = db.Close() }() + row, err := readThread(ctx, db, session.ID) + if err != nil { + return Result{}, err + } + if row.Archived || filepath.Clean(row.RolloutPath) != sourcePath { + return Result{}, errors.New("selected Codex thread is no longer active at the expected rollout") + } + if options.Apply && options.WriterActive == nil { + return Result{}, errors.New("archive apply requires a native writer probe") + } + if active, err := writerActive(ctx, session, options.WriterActive); err != nil { + return Result{}, err + } else if active { + return Result{}, errors.New("cannot archive a session with an active writer") + } + if !options.Apply { + return result, nil + } + journalPath := JournalPath(store, session.ID) + if _, err := os.Lstat(journalPath); err == nil { + return Result{}, errors.New("pending archive journal already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return Result{}, err + } + if err := os.MkdirAll(filepath.Dir(targetPath), 0o700); err != nil { + return Result{}, err + } + pending := journal{ + Version: 1, SessionID: session.ID, SourcePath: sourcePath, TargetPath: targetPath, + Bytes: current.Bytes, SHA256: current.SHA256, Phase: phasePrepared, + } + if err := writeJournal(journalPath, pending); err != nil { + return Result{}, err + } + journalOwned := true + removeJournal := func() error { + if !journalOwned { + return nil + } + if err := os.Remove(journalPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + journalOwned = false + return syncArchiveDirectory(filepath.Dir(journalPath)) + } + if options.BeforeRename != nil { + if err := options.BeforeRename(); err != nil { + return Result{}, errors.Join(err, removeJournal()) + } + } + conn, err := db.Conn(ctx) + if err != nil { + return Result{}, errors.Join(err, removeJournal()) + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, `begin immediate`); err != nil { + return Result{}, errors.Join(fmt.Errorf("begin Codex archive transaction: %w", err), removeJournal()) + } + transactionClosed := false + defer func() { + if !transactionClosed { + _, _ = conn.ExecContext(context.Background(), `rollback`) + } + }() + row, err = readThreadConn(ctx, conn, session.ID) + if err != nil { + return Result{}, errors.Join(err, removeJournal()) + } + if row.Archived || filepath.Clean(row.RolloutPath) != sourcePath { + return Result{}, errors.Join(errors.New("Codex route or archive state changed before rename"), removeJournal()) + } + verified, err := hashPath(sourcePath) + if err != nil || verified != current { + if err == nil { + err = errors.New("active rollout changed before archive rename") + } + return Result{}, errors.Join(err, removeJournal()) + } + if active, err := writerActive(ctx, session, options.WriterActive); err != nil { + return Result{}, errors.Join(err, removeJournal()) + } else if active { + return Result{}, errors.Join(errors.New("cannot archive a session with an active writer"), removeJournal()) + } + if _, err := os.Lstat(targetPath); err == nil { + return Result{}, errors.Join(errors.New("archive target appeared before rename"), removeJournal()) + } else if !errors.Is(err, os.ErrNotExist) { + return Result{}, errors.Join(err, removeJournal()) + } + if err := os.Rename(sourcePath, targetPath); err != nil { + return Result{}, errors.Join(fmt.Errorf("rename rollout into archive: %w", err), removeJournal()) + } + renamed := true + rollbackFile := func() error { + if !renamed { + return nil + } + if _, err := os.Lstat(sourcePath); err == nil { + return errors.New("cannot roll back archive while source path exists") + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + matches, err := pathMatches(targetPath, current) + if err != nil { + return err + } + if !matches { + return errors.New("cannot roll back archive because target bytes changed") + } + if err := os.MkdirAll(filepath.Dir(sourcePath), 0o700); err != nil { + return err + } + if err := os.Rename(targetPath, sourcePath); err != nil { + return err + } + renamed = false + return syncArchiveRename(targetPath, sourcePath) + } + if err := syncArchiveRename(sourcePath, targetPath); err != nil { + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + pending.Phase = phaseRenamed + if err := writeJournal(journalPath, pending); err != nil { + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + if options.AfterRename != nil { + if err := options.AfterRename(); err != nil { + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + } + columns, err := threadColumns(ctx, conn) + if err != nil { + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + now := options.Now + if now.IsZero() { + now = time.Now() + } + query := `update threads set rollout_path = ?, archived = 1` + args := []any{targetPath} + if columns["archived_at"] { + query += `, archived_at = ?` + args = append(args, now.Unix()) + } + if columns["updated_at_ms"] { + var maximum sql.NullInt64 + if err := conn.QueryRowContext(ctx, `select max(updated_at_ms) from threads`).Scan(&maximum); err != nil { + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + if maximum.Valid && maximum.Int64 == math.MaxInt64 { + return Result{}, errors.Join(errors.New("thread update clock overflow"), rollbackFile(), removeJournal()) + } + next := int64(1) + if maximum.Valid { + next = maximum.Int64 + 1 + } + query += `, updated_at_ms = ?` + args = append(args, next) + } + query += ` where id = ? and rollout_path = ? and archived = 0` + args = append(args, session.ID, sourcePath) + update, err := conn.ExecContext(ctx, query, args...) + if err != nil { + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + rows, err := update.RowsAffected() + if err != nil || rows != 1 { + if err == nil { + err = fmt.Errorf("archive update affected %d rows", rows) + } + return Result{}, errors.Join(err, rollbackFile(), removeJournal()) + } + if err := commitArchiveTransaction(ctx, conn); err != nil { + _, _ = conn.ExecContext(context.Background(), `rollback`) + transactionClosed = true + verifyCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + after, stateErr := readThread(verifyCtx, db, session.ID) + if stateErr != nil { + return result, errors.Join(fmt.Errorf("archive commit outcome is unknown: %w", err), stateErr) + } + switch { + case after.Archived && filepath.Clean(after.RolloutPath) == targetPath: + renamed = false + result.DryRun = false + result.Archived = true + return result, fmt.Errorf("archive committed but commit acknowledgement failed; run archive recover --apply: %w", err) + case !after.Archived && filepath.Clean(after.RolloutPath) == sourcePath: + fileErr := rollbackFile() + var journalErr error + if fileErr == nil { + journalErr = removeJournal() + } + return result, errors.Join(fmt.Errorf("commit Codex archive transaction: %w", err), fileErr, journalErr) + default: + return result, errors.Join(fmt.Errorf("archive commit outcome is ambiguous: %w", err), errors.New("Codex thread route no longer matches either archive state")) + } + } + transactionClosed = true + renamed = false + result.DryRun = false + result.Archived = true + if err := removeJournal(); err != nil { + return result, err + } + return result, nil +} + +func Recover(ctx context.Context, home string, store string, sessionID string) (RecoveryResult, error) { + if !validSessionID(sessionID) || !filepath.IsAbs(home) || !filepath.IsAbs(store) { + return RecoveryResult{}, errors.New("absolute home and store paths and a safe session ID are required") + } + home = filepath.Clean(home) + store = filepath.Clean(store) + path := JournalPath(store, sessionID) + pending, err := readJournal(path) + if err != nil { + return RecoveryResult{}, err + } + if pending.SessionID != sessionID || !validSessionID(sessionID) { + return RecoveryResult{}, errors.New("archive journal session does not match recovery request") + } + if _, err := relativeWithin(filepath.Join(home, "sessions"), pending.SourcePath); err != nil { + return RecoveryResult{}, errors.New("archive journal source is outside the Codex sessions directory") + } + expectedTarget := filepath.Join(home, "archived_sessions", filepath.Base(pending.SourcePath)) + if filepath.Clean(pending.TargetPath) != expectedTarget { + return RecoveryResult{}, errors.New("archive journal target does not match the official flat archive path") + } + result := RecoveryResult{SessionID: sessionID, SourcePath: pending.SourcePath, TargetPath: pending.TargetPath} + db, err := openStateDB(home) + if err != nil { + return RecoveryResult{}, err + } + defer func() { _ = db.Close() }() + conn, err := db.Conn(ctx) + if err != nil { + return RecoveryResult{}, err + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, `begin immediate`); err != nil { + return RecoveryResult{}, fmt.Errorf("begin Codex archive recovery transaction: %w", err) + } + transactionClosed := false + defer func() { + if !transactionClosed { + _, _ = conn.ExecContext(context.Background(), `rollback`) + } + }() + row, err := readThreadConn(ctx, conn, sessionID) + if err != nil { + return RecoveryResult{}, err + } + want := snapshot{Bytes: pending.Bytes, SHA256: pending.SHA256} + sourceExists, sourceMatches, err := inspectPath(pending.SourcePath, want) + if err != nil { + return RecoveryResult{}, err + } + targetExists, targetMatches, err := inspectPath(pending.TargetPath, want) + if err != nil { + return RecoveryResult{}, err + } + switch { + case !row.Archived && filepath.Clean(row.RolloutPath) == filepath.Clean(pending.SourcePath): + switch { + case sourceExists && sourceMatches && !targetExists: + case !sourceExists && targetExists && targetMatches: + if err := os.MkdirAll(filepath.Dir(pending.SourcePath), 0o700); err != nil { + return RecoveryResult{}, err + } + if err := os.Rename(pending.TargetPath, pending.SourcePath); err != nil { + return RecoveryResult{}, err + } + if err := syncArchiveRename(pending.TargetPath, pending.SourcePath); err != nil { + return RecoveryResult{}, err + } + default: + return RecoveryResult{}, errors.New("archive rollback state is ambiguous or changed") + } + result.RolledBack = true + case row.Archived && filepath.Clean(row.RolloutPath) == filepath.Clean(pending.TargetPath): + if sourceExists || !targetExists || !targetMatches { + return RecoveryResult{}, errors.New("committed archive files are ambiguous or changed") + } + result.Finalized = true + default: + return RecoveryResult{}, errors.New("Codex thread state no longer matches the archive journal") + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return RecoveryResult{}, err + } + if err := syncArchiveDirectory(filepath.Dir(path)); err != nil { + return RecoveryResult{}, err + } + if _, err := conn.ExecContext(ctx, `commit`); err != nil { + return RecoveryResult{}, fmt.Errorf("commit Codex archive recovery transaction: %w", err) + } + transactionClosed = true + return result, nil +} + +func openStateDB(home string) (*sql.DB, error) { + dbPath := filepath.Join(filepath.Clean(home), "state_5.sqlite") + info, err := os.Stat(dbPath) + if err != nil { + return nil, fmt.Errorf("locate Codex archive database: %w", err) + } + if !info.Mode().IsRegular() { + return nil, errors.New("Codex archive database is not a regular file") + } + db, err := sql.Open("sqlite", sqliteReadWriteDSN(dbPath)) + if err != nil { + return nil, fmt.Errorf("open Codex archive database: %w", err) + } + if _, err := db.Exec(`pragma busy_timeout = 10000`); err != nil { + _ = db.Close() + return nil, err + } + return db, nil +} + +func sqliteReadWriteDSN(path string) string { + slashPath := filepath.ToSlash(path) + if runtime.GOOS == "windows" { + slashPath = strings.ReplaceAll(path, "\\", "/") + if !strings.HasPrefix(slashPath, "/") { + slashPath = "/" + slashPath + } + } + uri := &url.URL{Scheme: "file", Path: slashPath} + query := uri.Query() + query.Set("mode", "rw") + uri.RawQuery = query.Encode() + return uri.String() +} + +func readThread(ctx context.Context, db *sql.DB, sessionID string) (threadRow, error) { + var row threadRow + var archived int + if err := db.QueryRowContext(ctx, `select rollout_path, archived from threads where id = ?`, sessionID).Scan(&row.RolloutPath, &archived); err != nil { + return threadRow{}, fmt.Errorf("read Codex archive thread: %w", err) + } + row.Archived = archived != 0 + return row, nil +} + +func readThreadConn(ctx context.Context, conn *sql.Conn, sessionID string) (threadRow, error) { + var row threadRow + var archived int + if err := conn.QueryRowContext(ctx, `select rollout_path, archived from threads where id = ?`, sessionID).Scan(&row.RolloutPath, &archived); err != nil { + return threadRow{}, fmt.Errorf("revalidate Codex archive thread: %w", err) + } + row.Archived = archived != 0 + return row, nil +} + +func threadColumns(ctx context.Context, conn *sql.Conn) (map[string]bool, error) { + rows, err := conn.QueryContext(ctx, `pragma table_info(threads)`) + if err != nil { + return nil, err + } + defer rows.Close() + columns := make(map[string]bool) + for rows.Next() { + var cid int + var name, dataType string + var notNull, primaryKey int + var defaultValue any + if err := rows.Scan(&cid, &name, &dataType, ¬Null, &defaultValue, &primaryKey); err != nil { + return nil, err + } + columns[name] = true + } + return columns, rows.Err() +} + +func writerActive(ctx context.Context, session codex.Session, probe func(context.Context, codex.Session) (bool, error)) (bool, error) { + if probe == nil { + return false, nil + } + active, err := probe(ctx, session) + if err != nil { + return false, fmt.Errorf("probe archive writer: %w", err) + } + return active, nil +} + +func hashPath(path string) (snapshot, error) { + info, err := os.Lstat(path) + if err != nil { + return snapshot{}, err + } + if !info.Mode().IsRegular() { + return snapshot{}, errors.New("rollout path is not a regular file") + } + file, err := os.Open(path) + if err != nil { + return snapshot{}, err + } + hasher := sha256.New() + written, copyErr := io.Copy(hasher, file) + closeErr := file.Close() + if copyErr != nil || closeErr != nil { + return snapshot{}, errors.Join(copyErr, closeErr) + } + return snapshot{Bytes: written, SHA256: hex.EncodeToString(hasher.Sum(nil))}, nil +} + +func hashBytes(data []byte) string { + digest := sha256.Sum256(data) + return hex.EncodeToString(digest[:]) +} + +func pathMatches(path string, want snapshot) (bool, error) { + got, err := hashPath(path) + if err != nil { + return false, err + } + return got == want, nil +} + +func inspectPath(path string, want snapshot) (bool, bool, error) { + got, err := hashPath(path) + if errors.Is(err, os.ErrNotExist) { + return false, false, nil + } + if err != nil { + return false, false, err + } + return true, got == want, nil +} + +func validSessionID(sessionID string) bool { + return sessionID != "" && sessionID != "." && sessionID != ".." && filepath.Base(sessionID) == sessionID && !strings.ContainsAny(sessionID, "/\\\x00") +} + +func relativeWithin(root string, target string) (string, error) { + relative, err := filepath.Rel(filepath.Clean(root), filepath.Clean(target)) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) || filepath.IsAbs(relative) { + return "", errors.New("path is outside root") + } + return relative, nil +} + +func syncArchiveRename(sourcePath string, targetPath string) error { + sourceDirectory := filepath.Dir(sourcePath) + targetDirectory := filepath.Dir(targetPath) + if err := syncArchiveDirectory(sourceDirectory); err != nil { + return err + } + if targetDirectory == sourceDirectory { + return nil + } + return syncArchiveDirectory(targetDirectory) +} + +func writeJournal(path string, value journal) error { + if value.Version != 1 || !validSessionID(value.SessionID) || !filepath.IsAbs(value.SourcePath) || !filepath.IsAbs(value.TargetPath) || value.Bytes < 0 || len(value.SHA256) != 64 || value.Phase != phasePrepared && value.Phase != phaseRenamed { + return errors.New("complete archive journal metadata is required") + } + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + directory := filepath.Dir(path) + if err := os.MkdirAll(directory, 0o700); err != nil { + return err + } + temporary, err := os.CreateTemp(directory, ".archive-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return err + } + return syncArchiveDirectory(directory) +} + +func readJournal(path string) (journal, error) { + data, err := os.ReadFile(path) + if err != nil { + return journal{}, err + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var value journal + if err := decoder.Decode(&value); err != nil { + return journal{}, err + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + if err == nil { + err = errors.New("multiple JSON values") + } + return journal{}, err + } + if value.Version != 1 || !validSessionID(value.SessionID) || !filepath.IsAbs(value.SourcePath) || !filepath.IsAbs(value.TargetPath) || value.Bytes < 0 || len(value.SHA256) != 64 || value.Phase != phasePrepared && value.Phase != phaseRenamed { + return journal{}, errors.New("invalid archive journal") + } + return value, nil +} diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go new file mode 100644 index 0000000..b8babd5 --- /dev/null +++ b/internal/archive/archive_test.go @@ -0,0 +1,338 @@ +package archive + +import ( + "context" + "database/sql" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/samekind/codexfold/internal/codex" + _ "modernc.org/sqlite" +) + +func TestArchiveDryRunAndApplyMatchOfficialFileAndStateBehavior(t *testing.T) { + fixture := archiveFixture(t) + originalGlobal, err := os.ReadFile(fixture.globalPath) + if err != nil { + t.Fatal(err) + } + dry, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{Now: fixture.now}) + if err != nil { + t.Fatal(err) + } + if !dry.DryRun || dry.Archived || dry.TargetPath != filepath.Join(fixture.home, "archived_sessions", filepath.Base(fixture.session.RolloutPath)) { + t.Fatalf("dry-run result = %#v", dry) + } + assertActiveSource(t, fixture) + + result, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, Now: fixture.now, WriterActive: idleWriter, + }) + if err != nil { + t.Fatal(err) + } + if result.DryRun || !result.Archived || result.Bytes != int64(len(fixture.source)) || result.SHA256 == "" { + t.Fatalf("archive result = %#v", result) + } + if _, err := os.Lstat(fixture.session.RolloutPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("active source remains after archive: %v", err) + } + archived, err := os.ReadFile(result.TargetPath) + if err != nil || string(archived) != string(fixture.source) { + t.Fatalf("archived bytes changed: %q err=%v", archived, err) + } + var path string + var archivedFlag int + var archivedAt sql.NullInt64 + var updatedAt int64 + var updatedAtMillis int64 + if err := fixture.db.QueryRow(`select rollout_path, archived, archived_at, updated_at, updated_at_ms from threads where id = ?`, fixture.session.ID).Scan( + &path, &archivedFlag, &archivedAt, &updatedAt, &updatedAtMillis, + ); err != nil { + t.Fatal(err) + } + if path != result.TargetPath || archivedFlag != 1 || !archivedAt.Valid || archivedAt.Int64 != fixture.now.Unix() || updatedAt != 100 || updatedAtMillis != 301 { + t.Fatalf("archived database row = path=%s archived=%d archived_at=%#v updated=%d/%d", path, archivedFlag, archivedAt, updatedAt, updatedAtMillis) + } + afterGlobal, err := os.ReadFile(fixture.globalPath) + if err != nil || string(afterGlobal) != string(originalGlobal) { + t.Fatalf("archive changed global state: %q err=%v", afterGlobal, err) + } + if _, err := os.Lstat(JournalPath(fixture.store, fixture.session.ID)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("completed archive left a journal: %v", err) + } +} + +func TestArchiveApplyRequiresWriterProbe(t *testing.T) { + fixture := archiveFixture(t) + _, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{Apply: true}) + if err == nil { + t.Fatal("archive apply accepted a missing writer probe") + } + assertActiveSource(t, fixture) +} + +func TestArchiveRejectsWriterRouteChangeAndSourceMutationBeforeRename(t *testing.T) { + t.Run("writer", func(t *testing.T) { + fixture := archiveFixture(t) + _, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, + WriterActive: func(context.Context, codex.Session) (bool, error) { + return true, nil + }, + }) + if err == nil { + t.Fatal("active writer was not rejected") + } + assertActiveSource(t, fixture) + }) + + t.Run("route change", func(t *testing.T) { + fixture := archiveFixture(t) + _, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, WriterActive: idleWriter, + BeforeRename: func() error { + _, err := fixture.db.Exec(`update threads set rollout_path = ? where id = ?`, filepath.Join(fixture.home, "other.jsonl"), fixture.session.ID) + return err + }, + }) + if err == nil { + t.Fatal("concurrent route change was not rejected") + } + if data, readErr := os.ReadFile(fixture.session.RolloutPath); readErr != nil || string(data) != string(fixture.source) { + t.Fatalf("route-race source changed: %q err=%v", data, readErr) + } + }) + + t.Run("source mutation", func(t *testing.T) { + fixture := archiveFixture(t) + _, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, WriterActive: idleWriter, + BeforeRename: func() error { + return os.WriteFile(fixture.session.RolloutPath, append(append([]byte(nil), fixture.source...), []byte("{\"changed\":true}\n")...), 0o600) + }, + }) + if err == nil { + t.Fatal("concurrent source mutation was not rejected") + } + var archived int + if dbErr := fixture.db.QueryRow(`select archived from threads where id = ?`, fixture.session.ID).Scan(&archived); dbErr != nil || archived != 0 { + t.Fatalf("source-race database changed: archived=%d err=%v", archived, dbErr) + } + }) +} + +func TestArchiveFailureAfterRenameRollsBackFileDatabaseAndJournal(t *testing.T) { + fixture := archiveFixture(t) + _, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, WriterActive: idleWriter, + AfterRename: func() error { + return errors.New("injected failure") + }, + }) + if err == nil { + t.Fatal("injected archive failure returned nil") + } + assertActiveSource(t, fixture) + if _, err := os.Lstat(filepath.Join(fixture.home, "archived_sessions", filepath.Base(fixture.session.RolloutPath))); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("failed archive left target: %v", err) + } + if _, err := os.Lstat(JournalPath(fixture.store, fixture.session.ID)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("rolled-back archive left journal: %v", err) + } +} + +func TestArchiveCommitFailureDoesNotGuessTransactionOutcome(t *testing.T) { + originalCommit := commitArchiveTransaction + t.Cleanup(func() { commitArchiveTransaction = originalCommit }) + + t.Run("not committed rolls back", func(t *testing.T) { + fixture := archiveFixture(t) + commitArchiveTransaction = func(context.Context, *sql.Conn) error { + return errors.New("injected commit failure") + } + _, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, WriterActive: idleWriter, + }) + if err == nil { + t.Fatal("commit failure returned nil") + } + assertActiveSource(t, fixture) + if _, statErr := os.Lstat(JournalPath(fixture.store, fixture.session.ID)); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("rolled-back commit failure left journal: %v", statErr) + } + }) + + t.Run("committed but acknowledgement failed leaves recovery journal", func(t *testing.T) { + fixture := archiveFixture(t) + commitArchiveTransaction = func(ctx context.Context, conn *sql.Conn) error { + if _, err := conn.ExecContext(ctx, `commit`); err != nil { + return err + } + return errors.New("lost commit acknowledgement") + } + result, err := Archive(context.Background(), fixture.home, fixture.store, fixture.session, Options{ + Apply: true, WriterActive: idleWriter, + }) + if err == nil { + t.Fatal("ambiguous commit acknowledgement returned nil") + } + if _, statErr := os.Lstat(result.TargetPath); statErr != nil { + t.Fatalf("committed archive target missing: %v", statErr) + } + if _, statErr := os.Lstat(JournalPath(fixture.store, fixture.session.ID)); statErr != nil { + t.Fatalf("ambiguous commit did not retain recovery journal: %v", statErr) + } + recovered, recoverErr := Recover(context.Background(), fixture.home, fixture.store, fixture.session.ID) + if recoverErr != nil || !recovered.Finalized || recovered.RolledBack { + t.Fatalf("recover committed archive = %#v err=%v", recovered, recoverErr) + } + }) +} + +func TestRecoverRollsBackRenamedFileOrFinalizesCommittedState(t *testing.T) { + t.Run("rollback", func(t *testing.T) { + fixture := archiveFixture(t) + target := filepath.Join(fixture.home, "archived_sessions", filepath.Base(fixture.session.RolloutPath)) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(fixture.session.RolloutPath, target); err != nil { + t.Fatal(err) + } + if err := writeJournal(JournalPath(fixture.store, fixture.session.ID), journal{ + Version: 1, SessionID: fixture.session.ID, SourcePath: fixture.session.RolloutPath, TargetPath: target, + Bytes: int64(len(fixture.source)), SHA256: hashBytes(fixture.source), Phase: phaseRenamed, + }); err != nil { + t.Fatal(err) + } + result, err := Recover(context.Background(), fixture.home, fixture.store, fixture.session.ID) + if err != nil || !result.RolledBack || result.Finalized { + t.Fatalf("rollback recovery = %#v err=%v", result, err) + } + assertActiveSource(t, fixture) + }) + + t.Run("finalize", func(t *testing.T) { + fixture := archiveFixture(t) + target := filepath.Join(fixture.home, "archived_sessions", filepath.Base(fixture.session.RolloutPath)) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(fixture.session.RolloutPath, target); err != nil { + t.Fatal(err) + } + if _, err := fixture.db.Exec(`update threads set rollout_path = ?, archived = 1, archived_at = ? where id = ?`, target, fixture.now.Unix(), fixture.session.ID); err != nil { + t.Fatal(err) + } + if err := writeJournal(JournalPath(fixture.store, fixture.session.ID), journal{ + Version: 1, SessionID: fixture.session.ID, SourcePath: fixture.session.RolloutPath, TargetPath: target, + Bytes: int64(len(fixture.source)), SHA256: hashBytes(fixture.source), Phase: phaseRenamed, + }); err != nil { + t.Fatal(err) + } + result, err := Recover(context.Background(), fixture.home, fixture.store, fixture.session.ID) + if err != nil || result.RolledBack || !result.Finalized { + t.Fatalf("finalize recovery = %#v err=%v", result, err) + } + if data, readErr := os.ReadFile(target); readErr != nil || string(data) != string(fixture.source) { + t.Fatalf("finalized target changed: %q err=%v", data, readErr) + } + }) +} + +func TestRecoverRejectsUnsafeIdentityAndJournalPaths(t *testing.T) { + fixture := archiveFixture(t) + if _, err := Recover(context.Background(), fixture.home, fixture.store, "../session"); err == nil { + t.Fatal("recovery accepted an unsafe session ID") + } + outside := filepath.Join(t.TempDir(), "outside.jsonl") + if err := os.WriteFile(outside, fixture.source, 0o600); err != nil { + t.Fatal(err) + } + if err := writeJournal(JournalPath(fixture.store, fixture.session.ID), journal{ + Version: 1, SessionID: fixture.session.ID, SourcePath: outside, + TargetPath: filepath.Join(fixture.home, "archived_sessions", filepath.Base(outside)), + Bytes: int64(len(fixture.source)), SHA256: hashBytes(fixture.source), Phase: phasePrepared, + }); err != nil { + t.Fatal(err) + } + if _, err := Recover(context.Background(), fixture.home, fixture.store, fixture.session.ID); err == nil { + t.Fatal("recovery accepted a journal outside the active sessions tree") + } +} + +type fixture struct { + home string + store string + db *sql.DB + session codex.Session + source []byte + globalPath string + now time.Time +} + +func archiveFixture(t *testing.T) fixture { + t.Helper() + home := t.TempDir() + store := filepath.Join(home, "fold-store") + rollout := filepath.Join(home, "sessions", "2026", "07", "16", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(rollout), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"type\":\"session_meta\"}\n{\"value\":1}\n") + if err := os.WriteFile(rollout, source, 0o600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(` + create table threads ( + id text primary key, + rollout_path text not null, + archived integer not null, + archived_at integer, + updated_at integer not null, + updated_at_ms integer not null + ); + insert into threads values ('session', ?, 0, null, 100, 200); + insert into threads values ('newer', '/tmp/newer.jsonl', 0, null, 300, 300); + `, rollout); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + globalPath := filepath.Join(home, ".codex-global-state.json") + if err := os.WriteFile(globalPath, []byte("{\"selectedThreadId\":\"session\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + return fixture{ + home: home, store: store, db: db, + session: codex.Session{ID: "session", RolloutPath: rollout}, + source: source, globalPath: globalPath, now: time.Unix(1_800_000_000, 0), + } +} + +func assertActiveSource(t *testing.T, fixture fixture) { + t.Helper() + data, err := os.ReadFile(fixture.session.RolloutPath) + if err != nil || string(data) != string(fixture.source) { + t.Fatalf("active source = %q err=%v", data, err) + } + var path string + var archived int + if err := fixture.db.QueryRow(`select rollout_path, archived from threads where id = ?`, fixture.session.ID).Scan(&path, &archived); err != nil { + t.Fatal(err) + } + if path != fixture.session.RolloutPath || archived != 0 { + t.Fatalf("active row = path=%s archived=%d", path, archived) + } +} + +func idleWriter(context.Context, codex.Session) (bool, error) { + return false, nil +} diff --git a/internal/archive/sync_unix.go b/internal/archive/sync_unix.go new file mode 100644 index 0000000..8fb845c --- /dev/null +++ b/internal/archive/sync_unix.go @@ -0,0 +1,14 @@ +//go:build !windows + +package archive + +import "os" + +func syncArchiveDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} diff --git a/internal/archive/sync_windows.go b/internal/archive/sync_windows.go new file mode 100644 index 0000000..8409326 --- /dev/null +++ b/internal/archive/sync_windows.go @@ -0,0 +1,5 @@ +//go:build windows + +package archive + +func syncArchiveDirectory(string) error { return nil } diff --git a/internal/buildid/buildid.go b/internal/buildid/buildid.go new file mode 100644 index 0000000..30532dd --- /dev/null +++ b/internal/buildid/buildid.go @@ -0,0 +1,40 @@ +package buildid + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" +) + +func CurrentSHA256() (string, error) { + executable, err := os.Executable() + if err != nil { + return "", err + } + return FileSHA256(executable) +} + +func FileSHA256(path string) (string, error) { + if !filepath.IsAbs(path) { + return "", errors.New("absolute executable path is required") + } + file, err := os.Open(filepath.Clean(path)) + if err != nil { + return "", err + } + hasher := sha256.New() + _, copyErr := io.Copy(hasher, file) + closeErr := file.Close() + if copyErr != nil || closeErr != nil { + return "", errors.Join(copyErr, closeErr) + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func ValidSHA256(value string) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == sha256.Size +} diff --git a/internal/cli/archive.go b/internal/cli/archive.go new file mode 100644 index 0000000..0d79900 --- /dev/null +++ b/internal/cli/archive.go @@ -0,0 +1,121 @@ +package cli + +import ( + "context" + "errors" + "fmt" + + archivepkg "github.com/samekind/codexfold/internal/archive" + "github.com/samekind/codexfold/internal/codex" + "github.com/spf13/cobra" +) + +type archiveFlags struct { + codexHome string + storeDir string + apply bool + json bool +} + +func newArchiveCommand() *cobra.Command { + var flags archiveFlags + command := &cobra.Command{ + Use: "archive ", + Short: "Preview or explicitly archive one selected Codex session", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(flags.codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, flags.storeDir) + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + session, err := findSession(sessions, args[0]) + if err != nil { + return err + } + writers, err := probeArchiveWriters(command.Context(), sessions) + if err != nil { + return err + } + result, err := archivepkg.Archive(command.Context(), home, store, session, archivepkg.Options{ + Apply: flags.apply, + WriterActive: func(_ context.Context, selected codex.Session) (bool, error) { + return writers[selected.ID], nil + }, + }) + if err != nil { + return err + } + if flags.json { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=%t archived=%t session=%s bytes=%s sha256=%s target=%s\n", + result.DryRun, result.Archived, result.SessionID, formatBytes(result.Bytes), result.SHA256, result.TargetPath) + return err + }, + } + addArchiveFlags(command, &flags) + command.AddCommand(newArchiveRecoverCommand()) + return command +} + +func newArchiveRecoverCommand() *cobra.Command { + var flags archiveFlags + command := &cobra.Command{ + Use: "recover ", + Short: "Recover one interrupted archive transaction", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + if !flags.apply { + return errors.New("archive recovery requires --apply") + } + home, err := codex.ResolveHome(flags.codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, flags.storeDir) + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + writers, err := probeArchiveWriters(command.Context(), sessions) + if err != nil { + return err + } + if writers[args[0]] { + return errors.New("cannot recover an archive transaction while the selected session has an active writer") + } + result, err := archivepkg.Recover(command.Context(), home, store, args[0]) + if err != nil { + return err + } + if flags.json { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "rolled_back=%t finalized=%t session=%s source=%s target=%s\n", + result.RolledBack, result.Finalized, result.SessionID, result.SourcePath, result.TargetPath) + return err + }, + } + addArchiveFlags(command, &flags) + return command +} + +func addArchiveFlags(command *cobra.Command, flags *archiveFlags) { + command.Flags().StringVar(&flags.codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&flags.storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().BoolVar(&flags.apply, "apply", false, "Apply the explicit archive or recovery mutation") + command.Flags().BoolVar(&flags.json, "json", false, "Emit JSON output") +} + +func probeArchiveWriters(ctx context.Context, sessions []codex.Session) (map[string]bool, error) { + writers, err := enrollmentWriterProbe(ctx, sessions) + if err != nil { + return nil, fmt.Errorf("probe native session writers: %w", err) + } + return writers, nil +} diff --git a/internal/cli/archive_test.go b/internal/cli/archive_test.go new file mode 100644 index 0000000..a8a3cd5 --- /dev/null +++ b/internal/cli/archive_test.go @@ -0,0 +1,213 @@ +package cli + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + archivepkg "github.com/samekind/codexfold/internal/archive" + "github.com/samekind/codexfold/internal/codex" + _ "modernc.org/sqlite" +) + +func TestArchiveCommandIsDryRunFirstAndMatchesOfficialApply(t *testing.T) { + fixture := archiveCLIFixture(t) + allowArchiveWriterProbe(t, nil) + + var output bytes.Buffer + root := NewRootCommand() + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"archive", fixture.sessionID, "--codex-home", fixture.home, "--store", fixture.store, "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("archive dry-run: %v", err) + } + var dry archivepkg.Result + if err := json.Unmarshal(output.Bytes(), &dry); err != nil || !dry.DryRun || dry.Archived { + t.Fatalf("archive dry-run = %#v err=%v output=%s", dry, err, output.String()) + } + if data, err := os.ReadFile(fixture.sourcePath); err != nil || string(data) != string(fixture.source) { + t.Fatalf("dry-run changed source: %q err=%v", data, err) + } + + output.Reset() + root = NewRootCommand() + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"archive", fixture.sessionID, "--codex-home", fixture.home, "--store", fixture.store, "--apply", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("archive apply: %v", err) + } + var applied archivepkg.Result + if err := json.Unmarshal(output.Bytes(), &applied); err != nil || applied.DryRun || !applied.Archived { + t.Fatalf("archive apply = %#v err=%v output=%s", applied, err, output.String()) + } + if data, err := os.ReadFile(applied.TargetPath); err != nil || string(data) != string(fixture.source) { + t.Fatalf("archived source = %q err=%v", data, err) + } + var rolloutPath string + var archived int + if err := fixture.db.QueryRow(`select rollout_path, archived from threads where id = ?`, fixture.sessionID).Scan(&rolloutPath, &archived); err != nil { + t.Fatal(err) + } + if rolloutPath != applied.TargetPath || archived != 1 { + t.Fatalf("archive route = %s archived=%d", rolloutPath, archived) + } +} + +func TestArchiveCommandFailsClosedWhenWriterProbeFailsOrReportsWriter(t *testing.T) { + t.Run("probe failure", func(t *testing.T) { + fixture := archiveCLIFixture(t) + allowArchiveWriterProbe(t, errors.New("lsof unavailable")) + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"archive", fixture.sessionID, "--codex-home", fixture.home, "--store", fixture.store, "--apply"}) + if err := root.Execute(); err == nil { + t.Fatal("archive accepted a failed native writer probe") + } + assertArchiveCLIActive(t, fixture) + }) + + t.Run("active writer", func(t *testing.T) { + fixture := archiveCLIFixture(t) + previous := enrollmentWriterProbe + enrollmentWriterProbe = func(context.Context, []codex.Session) (map[string]bool, error) { + return map[string]bool{fixture.sessionID: true}, nil + } + t.Cleanup(func() { enrollmentWriterProbe = previous }) + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"archive", fixture.sessionID, "--codex-home", fixture.home, "--store", fixture.store, "--apply"}) + if err := root.Execute(); err == nil { + t.Fatal("archive accepted an active native writer") + } + assertArchiveCLIActive(t, fixture) + }) +} + +func TestArchiveRecoverCommandRequiresApplyAndRestoresInterruptedRename(t *testing.T) { + fixture := archiveCLIFixture(t) + allowArchiveWriterProbe(t, nil) + target := filepath.Join(fixture.home, "archived_sessions", filepath.Base(fixture.sourcePath)) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(fixture.sourcePath, target); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(fixture.source) + journal := map[string]any{ + "version": 1, "session_id": fixture.sessionID, + "source_path": fixture.sourcePath, "target_path": target, + "bytes": len(fixture.source), "sha256": hex.EncodeToString(digest[:]), "phase": "renamed", + } + journalData, err := json.MarshalIndent(journal, "", " ") + if err != nil { + t.Fatal(err) + } + journalPath := archivepkg.JournalPath(fixture.store, fixture.sessionID) + if err := os.MkdirAll(filepath.Dir(journalPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(journalPath, append(journalData, '\n'), 0o600); err != nil { + t.Fatal(err) + } + + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"archive", "recover", fixture.sessionID, "--codex-home", fixture.home, "--store", fixture.store}) + if err := root.Execute(); err == nil { + t.Fatal("archive recovery ran without --apply") + } + if _, err := os.Stat(target); err != nil { + t.Fatalf("recovery preview changed target: %v", err) + } + + var output bytes.Buffer + root = NewRootCommand() + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"archive", "recover", fixture.sessionID, "--codex-home", fixture.home, "--store", fixture.store, "--apply", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("archive recover apply: %v", err) + } + var recovered archivepkg.RecoveryResult + if err := json.Unmarshal(output.Bytes(), &recovered); err != nil || !recovered.RolledBack || recovered.Finalized { + t.Fatalf("archive recover = %#v err=%v output=%s", recovered, err, output.String()) + } + assertArchiveCLIActive(t, fixture) +} + +type archiveCLIState struct { + home string + store string + db *sql.DB + sessionID string + sourcePath string + source []byte +} + +func archiveCLIFixture(t *testing.T) archiveCLIState { + t.Helper() + home := t.TempDir() + store := filepath.Join(home, "fold-store") + sourcePath := filepath.Join(home, "sessions", "2026", "07", "16", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(sourcePath), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"type\":\"session_meta\"}\n{\"value\":1}\n") + if err := os.WriteFile(sourcePath, source, 0o600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(` + create table threads ( + id text primary key, title text, cwd text, rollout_path text, + model_provider text, model text, updated_at integer, updated_at_ms integer, + archived integer, archived_at integer, git_branch text + ); + insert into threads values ('session', 'Session', '/workspace', ?, 'provider', 'model', 100, 200, 0, null, 'main'); + `, sourcePath); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + return archiveCLIState{home: home, store: store, db: db, sessionID: "session", sourcePath: sourcePath, source: source} +} + +func allowArchiveWriterProbe(t *testing.T, probeErr error) { + t.Helper() + previous := enrollmentWriterProbe + enrollmentWriterProbe = func(context.Context, []codex.Session) (map[string]bool, error) { + return map[string]bool{}, probeErr + } + t.Cleanup(func() { enrollmentWriterProbe = previous }) +} + +func assertArchiveCLIActive(t *testing.T, fixture archiveCLIState) { + t.Helper() + data, err := os.ReadFile(fixture.sourcePath) + if err != nil || string(data) != string(fixture.source) { + t.Fatalf("active source = %q err=%v", data, err) + } + var rolloutPath string + var archived int + if err := fixture.db.QueryRow(`select rollout_path, archived from threads where id = ?`, fixture.sessionID).Scan(&rolloutPath, &archived); err != nil { + t.Fatal(err) + } + if rolloutPath != fixture.sourcePath || archived != 0 { + t.Fatalf("active route = %s archived=%d", rolloutPath, archived) + } +} diff --git a/internal/cli/contains.go b/internal/cli/contains.go index d199e6c..4738223 100644 --- a/internal/cli/contains.go +++ b/internal/cli/contains.go @@ -3,8 +3,8 @@ package cli import ( "fmt" - "github.com/jstar0/codexfold/internal/codex" - "github.com/jstar0/codexfold/internal/contain" + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/contain" "github.com/spf13/cobra" ) diff --git a/internal/cli/content_boundary_test.go b/internal/cli/content_boundary_test.go new file mode 100644 index 0000000..a70e102 --- /dev/null +++ b/internal/cli/content_boundary_test.go @@ -0,0 +1,45 @@ +package cli + +import ( + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestContentChangingReconcilePackageHasOneExplicitCLIBoundary(t *testing.T) { + _, currentFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("resolve test source path") + } + repoRoot := filepath.Clean(filepath.Join(filepath.Dir(currentFile), "../..")) + internalRoot := filepath.Join(repoRoot, "internal") + fset := token.NewFileSet() + err := filepath.Walk(internalRoot, func(path string, info fs.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + if info.IsDir() || !strings.HasSuffix(info.Name(), ".go") || strings.HasSuffix(info.Name(), "_test.go") { + return nil + } + file, err := parser.ParseFile(fset, path, nil, 0) + if err != nil { + return err + } + for _, imported := range file.Imports { + if imported.Path.Value != `"github.com/samekind/codexfold/internal/reconcile"` { + continue + } + if filepath.Clean(path) != filepath.Join(repoRoot, "internal", "cli", "fs_reconcile.go") { + t.Errorf("content-changing reconcile package imported by %s", path) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/cli/fold.go b/internal/cli/fold.go index 8206fee..d051337 100644 --- a/internal/cli/fold.go +++ b/internal/cli/fold.go @@ -5,8 +5,8 @@ import ( "fmt" "path/filepath" - "github.com/jstar0/codexfold/internal/codex" - "github.com/jstar0/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/fold" "github.com/spf13/cobra" ) @@ -32,7 +32,7 @@ func newFoldCommand() *cobra.Command { if err != nil { return err } - result, err := fold.Fold(command.Context(), session, options) + result, err := fold.Fold(command.Context(), toFoldSession(session), options) if err != nil { return err } @@ -176,6 +176,13 @@ func findSession(sessions []codex.Session, sessionID string) (codex.Session, err return codex.Session{}, fmt.Errorf("Codex session not found: %s", sessionID) } +func toFoldSession(session codex.Session) fold.Session { + return fold.Session{ + ID: session.ID, Title: session.Title, CWD: session.CWD, + RolloutPath: session.RolloutPath, Archived: session.Archived, + } +} + func writeJSON(command *cobra.Command, value any) error { encoder := json.NewEncoder(command.OutOrStdout()) encoder.SetIndent("", " ") diff --git a/internal/cli/fork_family.go b/internal/cli/fork_family.go new file mode 100644 index 0000000..4a7c702 --- /dev/null +++ b/internal/cli/fork_family.go @@ -0,0 +1,106 @@ +package cli + +import ( + "fmt" + + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/family" + "github.com/spf13/cobra" +) + +func newForkFamilyCommand() *cobra.Command { + command := &cobra.Command{Use: "fork-family", Short: "Report fork graph and exact content evidence without mutation"} + command.AddCommand(newForkFamilyShowCommand()) + command.AddCommand(newForkFamilyCompareCommand()) + return command +} + +func newForkFamilyShowCommand() *cobra.Command { + var codexHome string + var jsonOutput bool + command := &cobra.Command{ + Use: "show ", + Short: "List the spawn-edge family and active or archived state", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + edges, err := codex.LoadSpawnEdges(home) + if err != nil { + return err + } + report, err := family.Build(args[0], sessions, edges) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, report) + } + if _, err := fmt.Fprintf(command.OutOrStdout(), "seed=%s members=%d edges=%d missing=%d\n", report.SeedID, len(report.Members), len(report.Edges), len(report.MissingSessionIDs)); err != nil { + return err + } + for _, member := range report.Members { + if _, err := fmt.Fprintf(command.OutOrStdout(), "session=%s relation=%s archived=%t path=%s\n", member.ID, member.RelationToSeed, member.Archived, member.RolloutPath); err != nil { + return err + } + } + return nil + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newForkFamilyCompareCommand() *cobra.Command { + var codexHome string + var jsonOutput bool + command := &cobra.Command{ + Use: "compare ", + Short: "Compare two explicitly selected rollouts using exact record evidence", + Args: cobra.ExactArgs(2), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + left, err := findSession(sessions, args[0]) + if err != nil { + return err + } + right, err := findSession(sessions, args[1]) + if err != nil { + return err + } + edges, err := codex.LoadSpawnEdges(home) + if err != nil { + return err + } + comparison, err := family.Compare(command.Context(), left, right, edges) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, comparison) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "left=%s right=%s graph=%s relation=%s exact=%t shared_prefix=%d shared=%d left_archived=%t right_archived=%t\n", + comparison.LeftID, comparison.RightID, comparison.GraphRelation, comparison.Relation, + comparison.VerifiedExact, comparison.SharedPrefixRecords, comparison.SharedRecords, + comparison.LeftArchived, comparison.RightArchived) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} diff --git a/internal/cli/fs.go b/internal/cli/fs.go new file mode 100644 index 0000000..66b56f5 --- /dev/null +++ b/internal/cli/fs.go @@ -0,0 +1,1939 @@ +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "math" + "os" + "os/exec" + "path/filepath" + "runtime" + "runtime/debug" + "strings" + "sync" + "time" + + "github.com/samekind/codexfold/internal/cdc" + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/compat" + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/fsctl" + "github.com/samekind/codexfold/internal/fskitproto" + "github.com/samekind/codexfold/internal/mountfs" + "github.com/samekind/codexfold/internal/pack" + "github.com/samekind/codexfold/internal/service" + "github.com/samekind/codexfold/internal/storage" + "github.com/samekind/codexfold/internal/vfs" + "github.com/spf13/cobra" +) + +type FSMigrateResult struct { + SessionID string `json:"session_id"` + Native vfs.NativeFile `json:"native"` + Target string `json:"target"` + Shadow fsctl.ShadowResult `json:"shadow"` + DryRun bool `json:"dry_run"` + Routed bool `json:"routed"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` +} + +type FSCompatibilityResult struct { + Installed []compat.ClientVersion `json:"installed,omitempty"` + Contracts int `json:"contracts"` + DetectionErrors []string `json:"detection_errors,omitempty"` + Evaluation compat.Evaluation `json:"evaluation"` +} + +type FSServeResult struct { + MountPoint string `json:"mount_point"` + ManagedSessions int `json:"managed_sessions"` + Frontend string `json:"frontend"` + ResourcePath string `json:"resource_path,omitempty"` + DryRun bool `json:"dry_run"` +} + +type FSRollbackResult struct { + SessionID string `json:"session_id"` + From string `json:"from"` + Target vfs.NativeFile `json:"target"` + RetiredState string `json:"retired_state,omitempty"` + DryRun bool `json:"dry_run"` + Routed bool `json:"routed"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` +} + +type FSCompactResult struct { + SessionID string `json:"session_id"` + CurrentGeneration uint64 `json:"current_generation"` + NextGeneration uint64 `json:"next_generation"` + Bytes int64 `json:"bytes,omitempty"` + SHA256 string `json:"sha256,omitempty"` + DryRun bool `json:"dry_run"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` +} + +type FSRecoverResult struct { + SessionIDs []string `json:"session_ids"` + Recovered int `json:"recovered"` + DryRun bool `json:"dry_run"` +} + +type FSNativeValidationResult struct { + Healthy bool `json:"healthy"` + Report mountfs.NativePreflightReport `json:"report"` + Issues []mountfs.NativePreflightIssue `json:"issues,omitempty"` +} + +type compatibilityFlags struct { + contractsPath string + cliPath string + desktopPath string +} + +var mountHealthProbe = service.ProbeMount + +func newFSCommand() *cobra.Command { + command := &cobra.Command{Use: "fs", Short: "Operate the transparent session filesystem"} + command.AddCommand(newFSStatusCommand()) + command.AddCommand(newFSDoctorCommand()) + command.AddCommand(newFSValidateNativeCommand()) + command.AddCommand(newFSCompatibilityCommand()) + command.AddCommand(newFSCompatibilityImportCommand()) + command.AddCommand(newFSBenchmarkCommand()) + command.AddCommand(newFSServeCommand()) + command.AddCommand(newFSNativeSupervisorCommand()) + command.AddCommand(newFSMigrateCommand()) + command.AddCommand(newFSRollbackCommand()) + command.AddCommand(newFSCompactCommand()) + command.AddCommand(newFSRecoverCommand()) + command.AddCommand(newFSEnrollCommand()) + command.AddCommand(newFSRepairRolloutCommand()) + command.AddCommand(newFSReconcileRolloutCommand()) + command.AddCommand(newFSNamespaceCommand()) + command.AddCommand(newFSServiceCommand()) + return command +} + +func newFSValidateNativeCommand() *cobra.Command { + var codexHome string + var nativeRoot string + var auditAll bool + var jsonOutput bool + command := &cobra.Command{ + Use: "validate-native", + Short: "Validate active native rollout JSONL before writer routing", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + root := nativeRoot + if root == "" { + root = filepath.Join(home, "fold-native") + } + result := FSNativeValidationResult{Healthy: true} + if auditAll { + audit, err := mountfs.AuditNativeWriterRollouts(command.Context(), root) + if err != nil { + return err + } + result.Report = audit.NativePreflightReport + result.Issues = audit.Issues + result.Healthy = len(result.Issues) == 0 + } else { + filesystem := mountfs.NewCanonical() + filesystem.SetNativeRoot(root) + result.Report, err = filesystem.ValidateNativeWriterRollouts(command.Context()) + if err != nil { + result.Healthy = false + result.Issues = []mountfs.NativePreflightIssue{{Message: err.Error()}} + } + } + if jsonOutput { + if err := writeJSON(command, result); err != nil { + return err + } + } else { + if _, err := fmt.Fprintf(command.OutOrStdout(), "healthy=%t files=%d bytes=%d validated=%d incremental=%d cached=%d issues=%d\n", result.Healthy, result.Report.Files, result.Report.Bytes, result.Report.ValidatedFiles, result.Report.IncrementalFiles, result.Report.CachedFiles, len(result.Issues)); err != nil { + return err + } + } + if !result.Healthy { + return fmt.Errorf("native rollout validation failed with %d issue(s)", len(result.Issues)) + } + return nil + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&nativeRoot, "native-root", "", "Native rollout root; defaults to /fold-native") + command.Flags().BoolVar(&auditAll, "audit-all", false, "Bypass the cache and report every invalid active rollout") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSStatusCommand() *cobra.Command { + var codexHome string + var storeDir string + var jsonOutput bool + command := &cobra.Command{ + Use: "status", + Short: "Report the highest verified filesystem capability", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + status, err := fsctl.NewStatus(verifiedCapability(), runtime.GOOS) + if err != nil { + return err + } + status.Storage, err = storage.Scan(command.Context(), storage.Options{StoreDir: store, AllowMetadataIssues: true}) + if err != nil { + return err + } + status.StorageLimits, err = storage.LoadLimits(store) + if err != nil { + return err + } + status.AvailableBytes, err = storage.AvailableBytes(store) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, status) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "capability=%s platform=%s logical=%s physical=%s available=%s\n", status.Capability, status.Platform, formatBytes(status.Storage.LogicalSessionBytes), formatBytes(status.Storage.TotalPhysicalBytes), formatBytes(status.AvailableBytes)) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSDoctorCommand() *cobra.Command { + var codexHome string + var storeDir string + var mountPoint string + var definitionPath string + var jsonOutput bool + command := &cobra.Command{ + Use: "doctor", + Short: "Verify filesystem storage, state, route, client, daemon, and mount components", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + mount := defaultMountPoint(home, mountPoint) + report := fsDoctor(command.Context(), home, store, mount, definitionPath) + if jsonOutput { + return writeJSON(command, report) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "healthy=%t issues=%d daemon=%t mount=%t pack=%t manifest=%t\n", report.Healthy, report.IssueCount, report.ComponentHealth[fsctl.ComponentDaemon], report.ComponentHealth[fsctl.ComponentMount], report.ComponentHealth[fsctl.ComponentPack], report.ComponentHealth[fsctl.ComponentManifest]) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + addServiceDefinitionFlags(command, &definitionPath) + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSCompatibilityCommand() *cobra.Command { + var codexHome string + var storeDir string + var flags compatibilityFlags + var jsonOutput bool + command := &cobra.Command{ + Use: "compatibility", + Short: "Evaluate installed Codex clients against exact-version contracts", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + result, err := evaluateCompatibility(command.Context(), resolveFoldStore(home, storeDir), flags) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "approved=%t quarantine=%t installed=%d contracts=%d detection_errors=%d\n", result.Evaluation.Approved, result.Evaluation.Quarantine, len(result.Installed), result.Contracts, len(result.DetectionErrors)) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + addCompatibilityFlags(command, &flags) + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSBenchmarkCommand() *cobra.Command { + var codexHome string + var storeDir string + var jsonOutput bool + var options fsctl.BenchmarkOptions + command := &cobra.Command{ + Use: "benchmark ", + Short: "Compare native and packed virtual reads without changing routes", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return err + } + var managedState *vfs.SessionState + for index := range states { + if states[index].SessionID == args[0] { + state := states[index] + managedState = &state + break + } + } + + var nativePath string + var virtual fsctl.Readable + var closeVirtual func() error + var cleanup func() + if managedState != nil { + managed, resolver, openErr := openManagedSession(command.Context(), store, *managedState) + if openErr != nil { + return openErr + } + defer resolver.Close() + reader, openErr := managed.OpenReader() + if openErr != nil { + return openErr + } + closeVirtual = reader.Close + defer func() { _ = closeVirtual() }() + + benchmarkDir, openErr := os.MkdirTemp("", "codexfold-benchmark-") + if openErr != nil { + return openErr + } + cleanup = func() { _ = os.RemoveAll(benchmarkDir) } + defer cleanup() + materialized, materializeErr := managed.MaterializeCurrent(command.Context(), filepath.Join(benchmarkDir, "visible.jsonl"), false) + if materializeErr != nil { + return materializeErr + } + nativePath = materialized.Path + virtual = reader + } else { + session, manifest, resolver, view, openErr := openFoldView(home, store, args[0]) + if openErr != nil { + return openErr + } + defer resolver.Close() + if manifest.Source.SHA256 == "" { + return errors.New("manifest source digest is missing") + } + nativePath = session.RolloutPath + virtual = view + } + report, err := fsctl.Benchmark(command.Context(), nativePath, virtual, options) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, report) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "native=%.0fB/s virtual=%.0fB/s random_p95=%s go_sys=%s\n", report.Native.BytesPerSecond, report.Virtual.BytesPerSecond, report.Random.P95, formatBytes(int64(report.GoSysBytes))) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().IntVar(&options.SequentialBlockBytes, "sequential-block-bytes", 0, "Sequential read block size") + command.Flags().IntVar(&options.RandomBlockBytes, "random-block-bytes", 0, "Random read block size") + command.Flags().IntVar(&options.RandomReads, "random-reads", 0, "Random read count") + command.Flags().Int64Var(&options.Seed, "seed", 1, "Deterministic random seed") + command.Flags().BoolVar(&options.BypassOSCache, "bypass-os-cache", false, "Request OS cache bypass for the native and packed reads") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSServeCommand() *cobra.Command { + var codexHome string + var storeDir string + var mountPoint string + var apply bool + var foreground bool + var canonicalNamespace bool + var nativeRoot string + var frontend string + var nativeFSKitSocket string + var nativeFSKitResource string + var operationTracePath string + var enrollmentInterval time.Duration + var enrollmentStableFor time.Duration + var enrollmentBatchSize int + var enrollmentCanary bool + var jsonOutput bool + command := &cobra.Command{ + Use: "serve", + Short: "Mount managed sessions and hot-load newly enrolled state", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + if apply { + if err := requireFilesystemActivationAllowed(home); err != nil { + return err + } + } + if canonicalNamespace { + if nativeRoot == "" || !filepath.IsAbs(nativeRoot) { + return errors.New("canonical namespace requires an absolute native root") + } + nativeRoot = filepath.Clean(nativeRoot) + } + if frontend != "fuse" && frontend != "native-fskit" { + return errors.New("filesystem frontend must be fuse or native-fskit") + } + if frontend == "native-fskit" { + if runtime.GOOS != "darwin" { + return errors.New("native-fskit frontend is available only on macOS") + } + if !canonicalNamespace { + return errors.New("native-fskit frontend requires the canonical namespace") + } + } + if enrollmentInterval < 0 || enrollmentStableFor < 0 || enrollmentBatchSize < 0 { + return errors.New("enrollment timing and batch values cannot be negative") + } + if enrollmentInterval > 0 { + if !canonicalNamespace { + return errors.New("periodic enrollment requires the canonical namespace") + } + if enrollmentStableFor <= 0 || enrollmentBatchSize <= 0 { + return errors.New("periodic enrollment requires a positive stable window and batch size") + } + } + if enrollmentCanary && enrollmentInterval <= 0 { + return errors.New("enrollment canary requires periodic enrollment") + } + store := resolveFoldStore(home, storeDir) + mount := defaultMountPoint(home, mountPoint) + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return err + } + if nativeFSKitResource == "" { + nativeFSKitResource = filepath.Join(store, "fs", "native-fskit") + } + if nativeFSKitSocket == "" { + nativeFSKitSocket = defaultNativeFSKitSocket(home, nativeFSKitResource) + } + result := FSServeResult{MountPoint: mount, ManagedSessions: len(states), Frontend: frontend, DryRun: !apply} + if frontend == "native-fskit" { + result.ResourcePath = nativeFSKitResource + } + if !apply { + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=true frontend=%s mount=%s sessions=%d resource=%s\n", frontend, mount, len(states), result.ResourcePath) + return err + } + processLock, err := service.AcquireProcessLock(filepath.Join(store, "fs", "service.lock")) + if err != nil { + return err + } + defer processLock.Close() + if canonicalNamespace { + for _, state := range states { + if _, err := recoverInterruptedCanonicalMigration(home, store, nativeRoot, state); err != nil { + return err + } + } + states, err = vfs.DiscoverSessionStates(store) + if err != nil { + return err + } + result.ManagedSessions = len(states) + } + var operationRecorder func(string) + if operationTracePath != "" { + recorder, closer, err := newOperationRecorder(operationTracePath) + if err != nil { + return err + } + operationRecorder = recorder + defer closer.Close() + } + if canonicalNamespace { + for _, directory := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(nativeRoot, directory), 0o700); err != nil { + return err + } + } + } + filesystem := mountfs.New() + if canonicalNamespace { + filesystem = mountfs.NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + if frontend == "native-fskit" { + filesystem.SetNativeNamespaceRefreshMount(mount) + } + if err := filesystem.RecoverNativeAppendTransactions(); err != nil { + return fmt.Errorf("recover native append transactions: %w", err) + } + if _, err := filesystem.ValidateNativeWriterRollouts(command.Context()); err != nil { + return fmt.Errorf("validate native writer rollouts: %w", err) + } + } + ctx, cancel := context.WithCancel(command.Context()) + defer cancel() + var nativeWatcherDone chan error + if frontend == "native-fskit" { + nativeWatcherDone = make(chan error, 1) + go func() { + err := filesystem.WatchNativeNamespace(ctx) + nativeWatcherDone <- err + if err != nil && !errors.Is(err, context.Canceled) { + cancel() + } + }() + } + var enrollmentDone chan struct{} + if enrollmentInterval > 0 { + enrollmentDone = make(chan struct{}) + flags := enrollmentFlags{ + codexHome: home, storeDir: store, mountPoint: mount, nativeRoot: nativeRoot, + stableFor: enrollmentStableFor, batchSize: enrollmentBatchSize, + canonicalNamespace: canonicalNamespace, canary: enrollmentCanary, + } + go func() { + defer close(enrollmentDone) + runPeriodicEnrollment(ctx, flags, enrollmentInterval, func(result FSEnrollmentApplyResult, cycleErr error) { + if cycleErr != nil { + if !errors.Is(cycleErr, context.Canceled) { + _, _ = fmt.Fprintf(command.ErrOrStderr(), "enrollment cycle failed: %v\n", cycleErr) + } + return + } + if len(result.Plan.Selected) == 0 && result.Apply.Applied == 0 { + return + } + _, _ = fmt.Fprintf(command.ErrOrStderr(), "enrollment cycle sessions=%d selected=%d applied=%d\n", len(result.Plan.Decisions), len(result.Plan.Selected), result.Apply.Applied) + }) + }() + } + known := make(map[string]uint64) + knownRoutes := make(map[string]string) + knownPacks := make(map[string]string) + var loadMu sync.Mutex + openState := func(state vfs.SessionState) (*vfs.Session, *pack.Resolver, error) { + managed, resolver, err := openManagedSession(ctx, store, state) + if err != nil { + return nil, nil, err + } + return managed, resolver, nil + } + filesystem.SetOwnedSessionLoader(func(sessionID string) (*vfs.Session, io.Closer, error) { + loadMu.Lock() + defer loadMu.Unlock() + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return nil, nil, err + } + for _, state := range states { + if state.SessionID == sessionID { + managed, resolver, err := openState(state) + if err == nil { + known[state.SessionID] = state.Generation + knownPacks[state.SessionID] = resolver.Generation() + } + return managed, resolver, err + } + } + return nil, nil, os.ErrNotExist + }) + load := func() error { + loadMu.Lock() + defer loadMu.Unlock() + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return err + } + currentPack, err := pack.CurrentGeneration(store) + if err != nil { + return err + } + routes := make(map[string]string) + if canonicalNamespace { + routes, err = discoverCanonicalRoutes(home, mount, store, states, codex.LoadSessions) + if err != nil { + return err + } + } + seen := make(map[string]struct{}, len(states)) + for _, state := range states { + seen[state.SessionID] = struct{}{} + if canonicalNamespace { + route, exists := routes[state.SessionID] + handled, err := syncCanonicalRetirement(store, home, nativeRoot, filesystem, state, route, exists, known, knownRoutes, knownPacks, currentPack, openState) + if err != nil { + return err + } + if handled { + continue + } + if !exists { + if _, mounted := known[state.SessionID]; mounted { + if err := filesystem.RemoveSession(state.SessionID); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + delete(known, state.SessionID) + delete(knownRoutes, state.SessionID) + delete(knownPacks, state.SessionID) + } + continue + } + generation, generationKnown := known[state.SessionID] + if generationKnown && generation == state.Generation && knownPacks[state.SessionID] == currentPack { + if knownRoutes[state.SessionID] == route { + continue + } + if err := filesystem.MoveSessionAt(state.SessionID, route); err != nil { + return err + } + knownRoutes[state.SessionID] = route + if err := writeMountAcknowledgement(store, state.SessionID, state.Generation, route); err != nil { + return err + } + continue + } + managed, resolver, err := openState(state) + if err != nil { + return err + } + if err := filesystem.UpsertSessionAtOwned(state.SessionID, route, managed, resolver); err != nil { + return err + } + known[state.SessionID] = state.Generation + knownRoutes[state.SessionID] = route + knownPacks[state.SessionID] = resolver.Generation() + if err := writeMountAcknowledgement(store, state.SessionID, state.Generation, route); err != nil { + return err + } + continue + } + if known[state.SessionID] == state.Generation && knownPacks[state.SessionID] == currentPack { + continue + } + managed, resolver, err := openState(state) + if err != nil { + return err + } + if err := filesystem.UpsertSessionOwned(state.SessionID, managed, resolver); err != nil { + return err + } + known[state.SessionID] = state.Generation + knownPacks[state.SessionID] = resolver.Generation() + } + for sessionID := range known { + if _, exists := seen[sessionID]; exists { + continue + } + if err := filesystem.RemoveSession(sessionID); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + delete(known, sessionID) + delete(knownRoutes, sessionID) + delete(knownPacks, sessionID) + } + return nil + } + if err := load(); err != nil { + return err + } + storageMaintenanceDone := startStorageMaintenance(ctx, command.ErrOrStderr(), store, startupStorageGC) + runtimeMemoryMaintenanceDone := startRuntimeMemoryMaintenance(ctx, filesystem) + watcherDone := make(chan struct{}) + watcherErrors := make(chan error, 1) + go func() { + defer close(watcherDone) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := load(); err != nil { + watcherErrors <- err + cancel() + return + } + } + } + }() + var mountErr error + if frontend == "native-fskit" { + mountErr = mountfs.ServeNativeFSKit(ctx, filesystem, mountfs.NativeFSKitServerOptions{ + SocketPath: nativeFSKitSocket, ResourcePath: nativeFSKitResource, Recorder: operationRecorder, + PrewarmSharedMemoryWindows: 4, + }) + } else { + mountErr = mountfs.Mount(ctx, mountfs.HostOptions{MountPoint: mount, Filesystem: filesystem, Foreground: foreground, OperationRecorder: operationRecorder}) + } + cancel() + <-watcherDone + <-storageMaintenanceDone + <-runtimeMemoryMaintenanceDone + if enrollmentDone != nil { + <-enrollmentDone + } + var nativeWatcherErr error + if nativeWatcherDone != nil { + nativeWatcherErr = <-nativeWatcherDone + if errors.Is(nativeWatcherErr, context.Canceled) { + nativeWatcherErr = nil + } + } + sessionCloseErr := filesystem.CloseSessions() + select { + case watcherErr := <-watcherErrors: + return errors.Join(watcherErr, sessionCloseErr) + default: + return errors.Join(mountErr, nativeWatcherErr, sessionCloseErr) + } + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + command.Flags().BoolVar(&apply, "apply", false, "Start the filesystem host") + command.Flags().BoolVar(&foreground, "foreground", true, "Keep the FUSE host in the foreground") + command.Flags().BoolVar(&canonicalNamespace, "canonical-namespace", false, "Expose sessions and archived_sessions as a shared virtual namespace") + command.Flags().StringVar(&nativeRoot, "native-root", "", "Backing root for unmanaged canonical session files") + command.Flags().StringVar(&frontend, "frontend", "fuse", "Filesystem frontend: fuse or native-fskit") + command.Flags().StringVar(&nativeFSKitSocket, "fskit-socket", "", "Native FSKit daemon Unix socket; defaults to a short per-home path in /private/tmp") + command.Flags().StringVar(&nativeFSKitResource, "fskit-resource", "", "Native FSKit resource; defaults to the security-scoped /fs/native-fskit directory") + command.Flags().StringVar(&operationTracePath, "operation-trace", "", "Absolute path for sanitized FUSE operation names") + command.Flags().DurationVar(&enrollmentInterval, "enrollment-interval", 0, "Periodic stable-session enrollment interval; zero disables the loop") + command.Flags().DurationVar(&enrollmentStableFor, "enrollment-stable-for", time.Hour, "Required unchanged interval before periodic enrollment") + command.Flags().IntVar(&enrollmentBatchSize, "enrollment-batch-size", 1, "Maximum sessions enrolled per periodic cycle") + command.Flags().BoolVar(&enrollmentCanary, "enrollment-canary", false, "Allow periodic enrollment only in an explicitly isolated Codex home while capability remains preview") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output for dry-run") + return command +} + +func defaultNativeFSKitSocket(home string, resourcePath string) string { + if fskitproto.UsesDirectoryResource(resourcePath) { + return filepath.Join(filepath.Clean(resourcePath), "daemon.sock") + } + digest := sha256.Sum256([]byte(filepath.Clean(home))) + userHome, err := os.UserHomeDir() + if err == nil && runtime.GOOS == "darwin" { + return filepath.Join(userHome, "Library", "Containers", "vip.jstar.codexfold.fskitprofileprobe.module", "Data", "tmp", fmt.Sprintf("cf-%s.sock", hex.EncodeToString(digest[:4]))) + } + return filepath.Join("/private/tmp", fmt.Sprintf("codexfold-fskit-%d-%s.sock", os.Getuid(), hex.EncodeToString(digest[:4]))) +} + +func syncCanonicalRetirement( + store string, + home string, + nativeRoot string, + filesystem *mountfs.Filesystem, + state vfs.SessionState, + route string, + routeExists bool, + known map[string]uint64, + knownRoutes map[string]string, + knownPacks map[string]string, + currentPack string, + openState func(vfs.SessionState) (*vfs.Session, *pack.Resolver, error), +) (bool, error) { + retirement, retiring, err := readRetirementRequest(store, state.SessionID) + if err != nil { + return false, err + } + if !retiring { + return false, removeIfExists(filepath.Join(store, "fs", "sessions", state.SessionID, retirementAcknowledgementFilename)) + } + reject := func(message string) (bool, error) { + rejected := retirement + rejected.Error = message + return true, writeRetirementAcknowledgement(store, state.SessionID, rejected) + } + if !routeExists || retirement.Route != route { + return reject("retirement request does not match the current session route") + } + generation := known[state.SessionID] + if generation != state.Generation || knownRoutes[state.SessionID] != route || knownPacks[state.SessionID] != currentPack { + managed, resolver, err := openState(state) + if err != nil { + return true, err + } + if err := filesystem.UpsertSessionAtOwned(state.SessionID, route, managed, resolver); err != nil { + return true, err + } + generation = managed.State().Generation + known[state.SessionID] = generation + knownRoutes[state.SessionID] = route + knownPacks[state.SessionID] = resolver.Generation() + } + if retirement.Generation != generation { + return reject("retirement request generation does not match the current session state") + } + nativeTargetPath, err := canonicalNativeRoute(home, nativeRoot, filepath.Join(home, filepath.FromSlash(strings.TrimPrefix(route, "/")))) + if err != nil { + return true, err + } + nativeTarget, targetErr := hashPath(nativeTargetPath) + if targetErr != nil || nativeTarget.Bytes != retirement.Bytes || nativeTarget.SHA256 != retirement.SHA256 { + return reject("native rollback target is unavailable or changed") + } + if err := filesystem.PreferNativeSession(state.SessionID); err != nil { + return true, err + } + if err := writeRetirementAcknowledgement(store, state.SessionID, retirement); err != nil { + return true, err + } + return true, nil +} + +func newFSMigrateCommand() *cobra.Command { + var codexHome string + var storeDir string + var mountPoint string + var nativeRoot string + var mountWait time.Duration + var apply, canonicalNamespace, compatibilityCanary bool + var jsonOutput bool + var compatibility compatibilityFlags + command := &cobra.Command{ + Use: "migrate ", + Short: "Shadow and optionally route an eligible session to the mounted filesystem", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + if apply { + if err := requireFilesystemActivationAllowed(home); err != nil { + return err + } + } + store := resolveFoldStore(home, storeDir) + session, manifest, resolver, view, err := openFoldView(home, store, args[0]) + if err != nil { + return err + } + defer resolver.Close() + if !session.Archived { + return errors.New("only archived sessions are eligible for filesystem migration") + } + mount := defaultMountPoint(home, mountPoint) + sourcePath := session.RolloutPath + target := filepath.Join(mount, session.ID+".jsonl") + if canonicalNamespace { + if nativeRoot == "" { + nativeRoot = filepath.Join(home, "fold-native") + } + sourcePath, err = canonicalNativeRoute(home, nativeRoot, session.RolloutPath) + if err != nil { + return err + } + target, err = canonicalMountRoute(home, mount, session.RolloutPath) + if err != nil { + return err + } + } + if _, err := mountfs.ValidateNativeRollout(command.Context(), sourcePath); err != nil { + return fmt.Errorf("native rollout is not eligible for transparent routing: %w", err) + } + shadow, err := fsctl.Shadow(command.Context(), sourcePath, view, fsctl.ShadowOptions{RandomReads: 10000, Seed: 1}) + if err != nil { + return err + } + native := vfs.NativeFile{Path: sourcePath, Bytes: shadow.Bytes, SHA256: shadow.SHA256} + result := FSMigrateResult{SessionID: session.ID, Native: native, Target: target, Shadow: shadow, DryRun: !apply} + if apply { + if err := requireStorageHealth(command.Context(), store); err != nil { + return err + } + if compatibilityCanary { + userHome, err := os.UserHomeDir() + if err != nil { + return err + } + if err := validateCompatibilityCanary(home, filepath.Join(userHome, ".codex"), store, canonicalNamespace, compatibility); err != nil { + return err + } + } else { + compatibilityResult, err := evaluateCompatibility(command.Context(), store, compatibility) + if err != nil { + return err + } + if len(compatibilityResult.DetectionErrors) != 0 || !compatibilityResult.Evaluation.Approved { + return errors.New("installed Codex client versions are not covered by compatibility contracts") + } + } + if err := mountHealthProbe(mount); err != nil { + return fmt.Errorf("filesystem mount point is not healthy: %w", err) + } + projectedPersistent := int64(1 << 20) + if canonicalNamespace { + projectedPersistent += native.Bytes + } + storageAssessment, err := assessStoreMutation(command.Context(), store, storage.Projection{Operation: "fs-migrate", AdditionalPersistentBytes: projectedPersistent}) + if err != nil { + return err + } + canonicalSource := "" + canonicalRoute := "" + if canonicalNamespace { + if _, err := os.Stat(filepath.Join(store, "fs", "sessions", session.ID, "state.json")); err == nil { + return errors.New("session is already managed") + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + canonicalSource = native.Path + canonicalRoute, err = canonicalNamespaceRoute(home, mount, session.RolloutPath) + if err != nil { + return err + } + retained, err := retainCanonicalSnapshot(command.Context(), store, session.ID, native, nil) + if err != nil { + return err + } + native = retained + result.Native = retained + } + rollbackMigration := func(cause error) error { + if !canonicalNamespace { + if _, err := os.Stat(filepath.Join(store, "fs", "sessions", session.ID)); errors.Is(err, os.ErrNotExist) { + return cause + } else if err != nil { + return errors.Join(cause, err) + } + if _, err := retireManagedState(store, session.ID); err != nil { + return errors.Join(cause, err) + } + return cause + } + return rollbackCanonicalMigration(store, session.ID, canonicalSource, native.Path, cause) + } + managed, migrationLease, err := vfs.OpenSessionWithWriter(command.Context(), vfs.SessionOptions{Root: store, ManifestPath: fold.ManifestPath(store, session.ID), Manifest: manifest, Reader: resolver, NativeSnapshot: native}) + if err != nil { + return rollbackMigration(err) + } + defer migrationLease.Close() + if canonicalNamespace { + if err := waitForMountAcknowledgement(command.Context(), store, session.ID, managed.State().Generation, canonicalRoute, mountWait); err != nil { + return rollbackMigration(fmt.Errorf("wait for canonical mount acknowledgement: %w", err)) + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return rollbackMigration(err) + } + current, err := findSession(sessions, session.ID) + if err != nil || filepath.Clean(current.RolloutPath) != filepath.Clean(session.RolloutPath) { + return rollbackMigration(errors.New("canonical Codex route changed during migration")) + } + if _, err := waitForTargetMatch(command.Context(), target, vfs.NativeFile{Bytes: shadow.Bytes, SHA256: shadow.SHA256}, mountWait); err != nil { + return rollbackMigration(fmt.Errorf("verify managed target before canonical cutover: %w", err)) + } + if err := finalizeCanonicalSnapshotSource(canonicalSource, native); err != nil { + return rollbackMigration(err) + } + } + targetFile, err := waitForTarget(command.Context(), target, mountWait) + if err != nil { + return rollbackMigration(fmt.Errorf("verify mounted target: %w", err)) + } + if targetFile.Bytes != shadow.Bytes || targetFile.SHA256 != shadow.SHA256 { + return rollbackMigration(errors.New("mounted target differs from the shadow-verified native session")) + } + if !canonicalNamespace { + if _, err := codex.RouteSession(command.Context(), codex.RouteOptions{CodexHome: home, SessionID: session.ID, ExpectedPath: session.RolloutPath, Target: codex.RouteTarget{Path: target, Bytes: targetFile.Bytes, SHA256: targetFile.SHA256}}); err != nil { + return err + } + } else { + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + current, err := findSession(sessions, session.ID) + if err != nil || filepath.Clean(current.RolloutPath) != filepath.Clean(session.RolloutPath) { + return rollbackMigration(errors.New("canonical Codex route changed during migration")) + } + } + result.Routed = true + result.DryRun = false + result.Storage = storage.CompleteAccounting(command.Context(), storageAssessment, store) + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "session=%s shadow=%t dry_run=%t routed=%t target=%s\n", result.SessionID, result.Shadow.Verified, result.DryRun, result.Routed, result.Target) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + command.Flags().BoolVar(&canonicalNamespace, "canonical-namespace", false, "Enroll the session at its canonical Codex path without changing SQLite routing") + command.Flags().StringVar(&nativeRoot, "native-root", "", "Canonical native snapshot root; defaults to /fold-native") + command.Flags().DurationVar(&mountWait, "mount-wait", 15*time.Second, "Maximum wait for the mounted session target") + command.Flags().BoolVar(&apply, "apply", false, "Enroll and route the session after all gates pass") + command.Flags().BoolVar(&compatibilityCanary, "compatibility-canary", false, "Allow an isolated canonical canary with both client checks explicitly skipped") + addCompatibilityFlags(command, &compatibility) + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func rollbackCanonicalMigration(store string, sessionID string, sourcePath string, retainedPath string, cause error) error { + // Keep the managed route live until an exact native source is available. + // If restoration fails, retiring state here would remove both recovery paths. + if err := restoreCanonicalSnapshotSource(sourcePath, retainedPath); err != nil { + return errors.Join(cause, err) + } + stateDirectory := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + if _, err := os.Stat(stateDirectory); errors.Is(err, os.ErrNotExist) { + return cause + } else if err != nil { + return errors.Join(cause, err) + } + if _, err := retireManagedState(store, sessionID); err != nil { + return errors.Join(cause, err) + } + return cause +} + +func newFSRollbackCommand() *cobra.Command { + var codexHome string + var storeDir string + var mountPoint string + var nativeRoot string + var targetPath string + var mountWait time.Duration + var apply, canonicalNamespace bool + var jsonOutput bool + command := &cobra.Command{ + Use: "rollback ", + Short: "Route a managed session to a verified native file containing its latest visible bytes", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + state, err := managedState(store, args[0]) + if err != nil { + return err + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + current, err := findSession(sessions, args[0]) + if err != nil { + return err + } + currentNativeFallback := !canonicalNamespace && isGeneratedNativeFallbackPath(current.RolloutPath, store, state.SessionID) + mount := defaultMountPoint(home, mountPoint) + if canonicalNamespace { + if nativeRoot == "" { + nativeRoot = filepath.Join(home, "fold-native") + } + canonicalTarget, err := canonicalNativeRoute(home, nativeRoot, current.RolloutPath) + if err != nil { + return err + } + if targetPath != "" && filepath.Clean(targetPath) != filepath.Clean(canonicalTarget) { + return errors.New("canonical rollback target must remain inside the retained native namespace") + } + targetPath = canonicalTarget + } else if targetPath == "" { + targetPath = filepath.Join(store, "fs", "fallbacks", state.SessionID, "fallback-current.jsonl") + } + result := FSRollbackResult{SessionID: state.SessionID, From: current.RolloutPath, Target: vfs.NativeFile{Path: filepath.Clean(targetPath)}, DryRun: !apply} + if apply { + if currentNativeFallback { + storageAssessment, err := assessStoreMutation(command.Context(), store, storage.Projection{Operation: "fs-rollback"}) + if err != nil { + return err + } + target, err := hashPath(current.RolloutPath) + if err != nil { + return err + } + retiredState, err := retireManagedState(store, state.SessionID) + if err != nil { + return err + } + result.Target = target + result.RetiredState = retiredState + result.Routed = true + result.DryRun = false + result.Storage = storage.CompleteAccounting(command.Context(), storageAssessment, store) + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "session=%s dry_run=%t routed=%t from=%s target=%s\n", result.SessionID, result.DryRun, result.Routed, result.From, result.Target.Path) + return err + } + if canonicalNamespace { + if err := mountHealthProbe(mount); err != nil { + return fmt.Errorf("canonical filesystem mount is not healthy: %w", err) + } + } + managed, resolver, err := openManagedSession(command.Context(), store, state) + if err != nil { + return err + } + defer resolver.Close() + rollbackLease, err := managed.OpenWriter() + if errors.Is(err, vfs.ErrWriterBusy) { + return errors.New("cannot rollback while the session has an active writer") + } + if err != nil { + return err + } + defer rollbackLease.Close() + visible, err := managed.VisibleInfo() + if err != nil { + return err + } + reclaimableBytes := int64(0) + if info, err := os.Stat(targetPath); err == nil && info.Mode().IsRegular() { + reclaimableBytes = info.Size() + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + storageAssessment, err := assessStoreMutation(command.Context(), store, storage.Projection{ + Operation: "fs-rollback", AdditionalPersistentBytes: visible.Size, TemporaryBytes: visible.Size, + TemporaryPersistentOverlapBytes: visible.Size, ReclaimableBytes: reclaimableBytes, + }) + if err != nil { + return err + } + target, err := managed.MaterializeCurrent(command.Context(), filepath.Clean(targetPath), true) + if err != nil { + return err + } + if canonicalNamespace { + mountedTarget, err := canonicalMountRoute(home, mount, current.RolloutPath) + if err != nil { + return err + } + canonicalRoute, err := canonicalNamespaceRoute(home, mount, current.RolloutPath) + if err != nil { + return err + } + retirement, err := createRetirementRequest(store, state.SessionID, managed.State().Generation, canonicalRoute, target) + if err != nil { + return err + } + recoveryWait := mountWait + if recoveryWait < 15*time.Second { + recoveryWait = 15 * time.Second + } + restoreManagedRoute := func(cause error, retiredState string, retiredSnapshot string) error { + var restoreErrors []error + if retiredSnapshot != "" { + if err := restoreCanonicalNativeSnapshot(state.NativeSnapshot.Path, retiredSnapshot); err != nil { + restoreErrors = append(restoreErrors, err) + } + } + var restored vfs.SessionState + if retiredState == "" { + directory := filepath.Join(store, "fs", "sessions", state.SessionID) + if err := clearRetirementControl(directory); err != nil { + restoreErrors = append(restoreErrors, err) + } else { + restored, err = vfs.RepublishSessionState(filepath.Join(directory, "state.json")) + if err != nil { + restoreErrors = append(restoreErrors, err) + } + } + } else if err := clearRetirementControl(retiredState); err != nil { + restoreErrors = append(restoreErrors, err) + } else if err := restoreManagedState(store, state.SessionID, retiredState); err != nil { + restoreErrors = append(restoreErrors, err) + } else { + restored, err = managedState(store, state.SessionID) + if err != nil { + restoreErrors = append(restoreErrors, err) + } + } + if restored.Generation != 0 { + if err := waitForMountAcknowledgement(command.Context(), store, state.SessionID, restored.Generation, canonicalRoute, recoveryWait); err != nil { + restoreErrors = append(restoreErrors, fmt.Errorf("wait for restored managed route: %w", err)) + } else if _, err := waitForTargetMatch(command.Context(), mountedTarget, target, recoveryWait); err != nil { + restoreErrors = append(restoreErrors, fmt.Errorf("verify restored managed route: %w", err)) + } + } + return errors.Join(append([]error{cause}, restoreErrors...)...) + } + if err := waitForRetirementAcknowledgement(command.Context(), store, state.SessionID, retirement, mountWait); err != nil { + return restoreManagedRoute(err, "", "") + } + _, err = waitForTargetMatch(command.Context(), mountedTarget, target, mountWait) + if err != nil { + return restoreManagedRoute(fmt.Errorf("verify canonical native rollback: %w", err), "", "") + } + nativeTarget, err := hashPath(target.Path) + if err != nil || nativeTarget.Bytes != target.Bytes || nativeTarget.SHA256 != target.SHA256 { + if err == nil { + err = errors.New("canonical native rollback target changed before retirement") + } + return restoreManagedRoute(fmt.Errorf("verify canonical native rollback target: %w", err), "", "") + } + retiredState, err := retireManagedState(store, state.SessionID) + if err != nil { + return restoreManagedRoute(err, "", "") + } + retiredSnapshot, err := retireCanonicalNativeSnapshot(store, nativeRoot, state.SessionID, state.NativeSnapshot.Path, target.Path, retiredState) + if err != nil { + return restoreManagedRoute(err, retiredState, "") + } + if err := clearRetirementControl(retiredState); err != nil { + return restoreManagedRoute(err, retiredState, retiredSnapshot) + } + result.RetiredState = retiredState + } else { + if _, err := codex.RouteSession(command.Context(), codex.RouteOptions{CodexHome: home, SessionID: state.SessionID, ExpectedPath: current.RolloutPath, Target: codex.RouteTarget{Path: target.Path, Bytes: target.Bytes, SHA256: target.SHA256}}); err != nil { + return err + } + retiredState, err := retireManagedState(store, state.SessionID) + if err != nil { + return err + } + result.RetiredState = retiredState + } + result.Target = target + result.Routed = true + result.DryRun = false + result.Storage = storage.CompleteAccounting(command.Context(), storageAssessment, store) + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "session=%s dry_run=%t routed=%t from=%s target=%s\n", result.SessionID, result.DryRun, result.Routed, result.From, result.Target.Path) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + command.Flags().BoolVar(&canonicalNamespace, "canonical-namespace", false, "Restore current bytes to canonical native backing without changing SQLite routing") + command.Flags().StringVar(&nativeRoot, "native-root", "", "Canonical native rollback root; defaults to /fold-native") + command.Flags().DurationVar(&mountWait, "mount-wait", 15*time.Second, "Maximum wait for native passthrough after state retirement") + command.Flags().StringVar(&targetPath, "to", "", "Native rollback target; defaults to the managed session directory") + command.Flags().BoolVar(&apply, "apply", false, "Materialize current bytes and update the Codex route") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSCompactCommand() *cobra.Command { + var codexHome string + var storeDir string + var idleFor time.Duration + var apply bool + var jsonOutput bool + command := &cobra.Command{ + Use: "compact ", + Short: "Fold the latest visible bytes into a new verified immutable generation", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + state, err := managedState(store, args[0]) + if err != nil { + return err + } + result := FSCompactResult{SessionID: state.SessionID, CurrentGeneration: state.Generation, NextGeneration: state.Generation + 1, DryRun: !apply} + if apply { + managed, resolver, err := openManagedSession(command.Context(), store, state) + if err != nil { + return err + } + defer resolver.Close() + visible, err := managed.VisibleInfo() + if err != nil { + return err + } + persistentBytes, err := conservativeStoredBytes(visible.Size) + if err != nil { + return err + } + if persistentBytes > math.MaxInt64-persistentBytes { + return errors.New("compact storage byte estimate overflow") + } + persistentBytes *= 2 + storageAssessment, err := assessStoreMutation(command.Context(), store, storage.Projection{ + Operation: "fs-compact", AdditionalPersistentBytes: persistentBytes, TemporaryBytes: visible.Size, + }) + if err != nil { + return err + } + var preparedResolver *pack.Resolver + defer func() { + if preparedResolver != nil { + _ = preparedResolver.Close() + } + }() + compact, err := managed.Compact(command.Context(), vfs.CompactOptions{IdleFor: idleFor, Prepare: func(ctx context.Context, current vfs.NativeFile, generation uint64) (vfs.PreparedGeneration, error) { + currentManifest, err := fold.LoadManifestPath(state.ManifestPath) + if err != nil { + return vfs.PreparedGeneration{}, err + } + manifestPath := filepath.Join(store, "manifests", "generations", state.SessionID, fmt.Sprintf("%020d.json", generation)) + options := fold.FoldOptions{ + StoreDir: store, ManifestPathOverride: manifestPath, Apply: true, Overwrite: true, + FieldThreshold: currentManifest.Settings.FieldThreshold, MaxJSONLineBytes: currentManifest.Settings.MaxJSONLineBytes, + CDC: cdc.Options{MinBytes: currentManifest.Settings.CDCMinBytes, AverageBytes: currentManifest.Settings.CDCAverageBytes, MaxBytes: currentManifest.Settings.CDCMaxBytes}, + } + if _, err := fold.Fold(ctx, fold.Session{ID: state.SessionID, Title: currentManifest.Session.Title, CWD: currentManifest.Session.CWD, RolloutPath: current.Path, Archived: true}, options); err != nil { + return vfs.PreparedGeneration{}, err + } + if _, err := pack.Build(ctx, store, pack.BuildOptions{}); err != nil { + return vfs.PreparedGeneration{}, err + } + manifest, err := fold.LoadManifestPath(manifestPath) + if err != nil { + return vfs.PreparedGeneration{}, err + } + preparedResolver, err = pack.Open(store, pack.OpenOptions{}) + if err != nil { + return vfs.PreparedGeneration{}, err + } + view, err := vfs.NewView(manifest, preparedResolver) + if err != nil { + return vfs.PreparedGeneration{}, err + } + return vfs.PreparedGeneration{ManifestPath: manifestPath, Manifest: manifest, View: view}, nil + }}) + if err != nil { + return err + } + result.NextGeneration = compact.Generation + result.Bytes = compact.Bytes + result.SHA256 = compact.SHA256 + result.DryRun = false + result.Storage = storage.CompleteAccounting(command.Context(), storageAssessment, store) + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "session=%s dry_run=%t generation=%d->%d bytes=%s sha256=%s\n", result.SessionID, result.DryRun, result.CurrentGeneration, result.NextGeneration, formatBytes(result.Bytes), result.SHA256) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().DurationVar(&idleFor, "idle-for", 0, "Minimum stable time before compaction") + command.Flags().BoolVar(&apply, "apply", false, "Commit the new compacted generation") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSRecoverCommand() *cobra.Command { + var codexHome string + var storeDir string + var all bool + var apply bool + var jsonOutput bool + command := &cobra.Command{ + Use: "recover [session-id]", + Short: "Inspect or recover interrupted managed session operations", + Args: cobra.MaximumNArgs(1), + RunE: func(command *cobra.Command, args []string) error { + if len(args) == 0 && !all { + return errors.New("provide a session ID or --all") + } + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return err + } + selected := make([]vfs.SessionState, 0) + for _, state := range states { + if all || state.SessionID == args[0] { + selected = append(selected, state) + } + } + if !all && len(selected) == 0 { + return fmt.Errorf("managed session not found: %s", args[0]) + } + result := FSRecoverResult{DryRun: !apply} + for _, state := range selected { + result.SessionIDs = append(result.SessionIDs, state.SessionID) + if !apply { + continue + } + managed, resolver, err := openManagedSession(command.Context(), store, state) + if err != nil { + return err + } + if err := managed.Recover(command.Context()); err != nil { + _ = resolver.Close() + return err + } + recoveredState := managed.State() + _ = resolver.Close() + if _, err := recoverInterruptedCanonicalMigration(home, store, filepath.Join(home, "fold-native"), recoveredState); err != nil { + return err + } + result.Recovered++ + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=%t selected=%d recovered=%d\n", result.DryRun, len(result.SessionIDs), result.Recovered) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().BoolVar(&all, "all", false, "Recover every managed session") + command.Flags().BoolVar(&apply, "apply", false, "Apply deterministic journal recovery") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func recoverInterruptedCanonicalMigration(home string, store string, nativeRoot string, state vfs.SessionState) (recovered bool, resultErr error) { + retainedPath := filepath.Join(store, "fs", "snapshots", state.SessionID, "native.jsonl") + if filepath.Clean(state.NativeSnapshot.Path) != filepath.Clean(retainedPath) { + return false, nil + } + if _, pending, err := readRetirementRequest(store, state.SessionID); err != nil { + return false, err + } else if pending { + return false, nil + } + guard, acquired, err := vfs.TryAcquireWriterLeaseGuard(store, state.SessionID) + if err != nil { + return false, err + } + if !acquired { + return false, nil + } + defer func() { resultErr = errors.Join(resultErr, guard.Close()) }() + sessions, err := codex.LoadSessions(home) + if err != nil { + return false, err + } + current, err := findSession(sessions, state.SessionID) + if err != nil { + return false, err + } + sourcePath, err := canonicalNativeRoute(home, nativeRoot, current.RolloutPath) + if err != nil { + return false, err + } + source, err := hashPath(sourcePath) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + if state.BackingPath != "" { + return false, nil + } + delta, err := os.Stat(state.DeltaPath) + if err != nil { + return false, err + } + if delta.Size() != 0 { + return false, nil + } + if source.Bytes != state.BaseBytes || source.SHA256 != state.BaseSHA256 || source.Bytes != state.NativeSnapshot.Bytes || source.SHA256 != state.NativeSnapshot.SHA256 { + return false, errors.New("interrupted canonical migration source no longer matches the managed base") + } + retiredState, err := retireManagedState(store, state.SessionID) + if err != nil { + return false, err + } + if _, err := retireCanonicalNativeSnapshot(store, nativeRoot, state.SessionID, state.NativeSnapshot.Path, sourcePath, retiredState); err != nil { + if restoreErr := restoreManagedState(store, state.SessionID, retiredState); restoreErr != nil { + return false, errors.Join(err, restoreErr) + } + return false, err + } + return true, nil +} + +func addCompatibilityFlags(command *cobra.Command, flags *compatibilityFlags) { + defaults := defaultCompatibilityFlags() + command.Flags().StringVar(&flags.contractsPath, "contracts", "", "Compatibility contract directory; defaults to /compatibility") + command.Flags().StringVar(&flags.cliPath, "cli", defaults.cliPath, "Codex CLI path, or 'none' to skip CLI evaluation") + command.Flags().StringVar(&flags.desktopPath, "desktop-app", defaults.desktopPath, "Codex desktop application path, or 'none' to skip desktop evaluation") +} + +func defaultCompatibilityFlags() compatibilityFlags { + desktop := "none" + if runtime.GOOS == "darwin" { + desktop = "/Applications/ChatGPT.app" + } + return compatibilityFlags{cliPath: "codex", desktopPath: desktop} +} + +func evaluateCompatibility(ctx context.Context, store string, flags compatibilityFlags) (FSCompatibilityResult, error) { + contractsPath := flags.contractsPath + if contractsPath == "" { + contractsPath = filepath.Join(store, "compatibility") + } + contracts, err := compat.LoadAll(contractsPath) + if err != nil { + return FSCompatibilityResult{}, err + } + result := FSCompatibilityResult{Contracts: len(contracts)} + if flags.cliPath != "none" { + binary := flags.cliPath + if !strings.ContainsRune(binary, filepath.Separator) { + resolved, err := exec.LookPath(binary) + if err != nil { + result.DetectionErrors = append(result.DetectionErrors, "cli: "+err.Error()) + } else { + binary = resolved + } + } + if len(result.DetectionErrors) == 0 { + client, err := compat.DetectCLIVersion(ctx, binary) + if err != nil { + result.DetectionErrors = append(result.DetectionErrors, "cli: "+err.Error()) + } else { + result.Installed = append(result.Installed, client) + } + } + } + if flags.desktopPath != "none" { + if _, err := os.Stat(flags.desktopPath); err != nil { + result.DetectionErrors = append(result.DetectionErrors, "desktop: "+err.Error()) + } else { + client, err := compat.DetectDesktopVersion(ctx, flags.desktopPath) + if err != nil { + result.DetectionErrors = append(result.DetectionErrors, "desktop: "+err.Error()) + } else { + result.Installed = append(result.Installed, client) + } + } + } + result.Evaluation = compat.Evaluate(result.Installed, contracts) + if len(result.Installed) == 0 { + result.Evaluation = compat.Evaluation{Approved: false, Quarantine: true} + } + return result, nil +} + +func validateCompatibilityCanary(home string, defaultHome string, store string, canonical bool, flags compatibilityFlags) error { + home = filepath.Clean(home) + defaultHome = filepath.Clean(defaultHome) + store = filepath.Clean(store) + if !canonical { + return errors.New("compatibility canary requires canonical namespace mode") + } + if home == defaultHome { + return errors.New("compatibility canary is forbidden for the real Codex home") + } + relative, err := filepath.Rel(home, store) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) || filepath.IsAbs(relative) { + return errors.New("compatibility canary store must be inside the isolated Codex home") + } + if flags.cliPath != "none" || flags.desktopPath != "none" { + return errors.New("compatibility canary requires --cli none and --desktop-app none") + } + return nil +} + +func openFoldView(home string, store string, sessionID string) (codex.Session, fold.Manifest, *pack.Resolver, *vfs.View, error) { + sessions, err := codex.LoadSessions(home) + if err != nil { + return codex.Session{}, fold.Manifest{}, nil, nil, err + } + session, err := findSession(sessions, sessionID) + if err != nil { + return codex.Session{}, fold.Manifest{}, nil, nil, err + } + manifest, err := fold.LoadManifest(store, session.ID) + if err != nil { + return codex.Session{}, fold.Manifest{}, nil, nil, err + } + resolver, err := pack.Open(store, pack.OpenOptions{}) + if err != nil { + return codex.Session{}, fold.Manifest{}, nil, nil, err + } + view, err := vfs.NewView(manifest, resolver) + if err != nil { + _ = resolver.Close() + return codex.Session{}, fold.Manifest{}, nil, nil, err + } + return session, manifest, resolver, view, nil +} + +func openManagedSession(ctx context.Context, store string, state vfs.SessionState) (*vfs.Session, *pack.Resolver, error) { + manifest, err := fold.LoadManifestPath(state.ManifestPath) + if err != nil { + return nil, nil, err + } + resolver, err := pack.Open(store, pack.OpenOptions{}) + if err != nil { + return nil, nil, err + } + managed, err := vfs.OpenSession(ctx, vfs.SessionOptions{Root: store, ManifestPath: state.ManifestPath, Manifest: manifest, Reader: resolver, NativeSnapshot: state.NativeSnapshot}) + if err != nil { + _ = resolver.Close() + return nil, nil, err + } + return managed, resolver, nil +} + +func managedState(store string, sessionID string) (vfs.SessionState, error) { + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return vfs.SessionState{}, err + } + for _, state := range states { + if state.SessionID == sessionID { + return state, nil + } + } + return vfs.SessionState{}, fmt.Errorf("managed session not found: %s", sessionID) +} + +func requireStorageHealth(ctx context.Context, store string) error { + packReport, err := pack.Doctor(ctx, store) + if err != nil { + return err + } + if packReport.IssueCount != 0 { + return fmt.Errorf("pack doctor reported %d issues", packReport.IssueCount) + } + foldReport, err := fold.Doctor(ctx, store) + if err != nil { + return err + } + if foldReport.IssueCount != 0 { + return fmt.Errorf("fold doctor reported %d issues", foldReport.IssueCount) + } + return nil +} + +func assessStoreMutation(ctx context.Context, store string, projection storage.Projection) (storage.Assessment, error) { + guard, err := storage.DefaultGuard(store) + if err != nil { + return storage.Assessment{}, err + } + return guard.Check(ctx, projection) +} + +func conservativeStoredBytes(rawBytes int64) (int64, error) { + if rawBytes < 0 { + return 0, errors.New("storage byte estimate cannot be negative") + } + overhead := rawBytes/16 + 1<<20 + if rawBytes > math.MaxInt64-overhead { + return 0, errors.New("storage byte estimate overflow") + } + return rawBytes + overhead, nil +} + +func startupStorageGC(ctx context.Context, store string) (storage.StorageGCResult, bool, error) { + if err := requireStorageHealth(ctx, store); err != nil { + return storage.StorageGCResult{}, false, nil + } + result, err := storage.Collect(ctx, storage.GCOptions{StoreDir: store, Apply: true}) + return result, true, err +} + +func startStorageMaintenance( + ctx context.Context, + diagnostics io.Writer, + store string, + run func(context.Context, string) (storage.StorageGCResult, bool, error), +) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + defer debug.FreeOSMemory() + _, _, err := run(ctx, store) + if err != nil && !errors.Is(err, context.Canceled) { + _, _ = fmt.Fprintf(diagnostics, "storage maintenance failed: %v\n", err) + } + }() + return done +} + +func startRuntimeMemoryMaintenance(ctx context.Context, filesystem *mountfs.Filesystem) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if !filesystem.IOIdleFor(3 * time.Second) { + continue + } + var memory runtime.MemStats + runtime.ReadMemStats(&memory) + if runtimeMemoryReclaimable(memory, 64<<20) { + debug.FreeOSMemory() + } + } + } + }() + return done +} + +func runtimeMemoryReclaimable(memory runtime.MemStats, threshold uint64) bool { + return memory.HeapIdle > memory.HeapReleased && memory.HeapIdle-memory.HeapReleased >= threshold +} + +func fsDoctor(ctx context.Context, home string, store string, mount string, definitionPath string) fsctl.DoctorReport { + var serviceStatus service.Status + platform, platformErr := service.CurrentPlatform() + definition, definitionErr := resolveServiceDefinitionPath(definitionPath) + if platformErr == nil && definitionErr == nil { + serviceStatus, platformErr = platformServiceStatus(ctx, platform, mount, definition) + } + if platformErr != nil || definitionErr != nil { + serviceStatus.DaemonError = errors.Join(platformErr, definitionErr).Error() + } + var storageInventory storage.Inventory + var storageLimits storage.Limits + var availableBytes int64 + checks := []fsctl.Check{ + {Component: fsctl.ComponentDaemon, Run: func(context.Context) error { + if !serviceStatus.DaemonRunning { + return errors.New(serviceStatus.DaemonError) + } + if !serviceStatus.Build.Healthy { + return errors.New(serviceStatus.Build.Error) + } + return nil + }}, + {Component: fsctl.ComponentMount, Run: func(context.Context) error { + if !serviceStatus.MountHealthy { + return errors.New(serviceStatus.MountError) + } + return nil + }}, + {Component: fsctl.ComponentPack, Run: func(ctx context.Context) error { + report, err := pack.Doctor(ctx, store) + if err != nil { + return err + } + if report.IssueCount != 0 { + return fmt.Errorf("pack doctor reported %d issues", report.IssueCount) + } + return nil + }}, + {Component: fsctl.ComponentManifest, Run: func(ctx context.Context) error { + report, err := fold.Doctor(ctx, store) + if err != nil { + return err + } + if report.IssueCount != 0 { + return fmt.Errorf("fold doctor reported %d issues", report.IssueCount) + } + return nil + }}, + {Component: fsctl.ComponentStorage, Run: func(ctx context.Context) error { + var err error + storageInventory, err = storage.Scan(ctx, storage.Options{StoreDir: store, AllowMetadataIssues: true}) + if err != nil { + return err + } + storageLimits, err = storage.LoadLimits(store) + if err != nil { + return err + } + availableBytes, err = storage.AvailableBytes(store) + return err + }}, + } + states, stateErr := vfs.DiscoverSessionStates(store) + stateCheck := func(kind string) fsctl.Check { + return fsctl.Check{Component: kind, Run: func(context.Context) error { + if stateErr != nil { + return stateErr + } + for _, state := range states { + paths := []string{state.DeltaPath} + if kind == fsctl.ComponentBacking && state.BackingPath != "" { + paths = []string{state.BackingPath} + } + for _, path := range paths { + if _, err := os.Stat(path); err != nil { + return err + } + } + } + return nil + }} + } + checks = append(checks, stateCheck(fsctl.ComponentDelta), stateCheck(fsctl.ComponentBacking)) + checks = append(checks, + fsctl.Check{Component: fsctl.ComponentRoute, Run: func(context.Context) error { + sessions, err := codex.LoadSessions(home) + if err != nil { + return err + } + for _, session := range sessions { + if _, err := os.Stat(session.RolloutPath); err != nil { + return fmt.Errorf("session %s route: %w", session.ID, err) + } + } + return nil + }}, + fsctl.Check{Component: fsctl.ComponentFallback, Run: func(context.Context) error { + if stateErr != nil { + return stateErr + } + for _, state := range states { + if _, err := os.Stat(state.NativeSnapshot.Path); err != nil { + return err + } + } + return nil + }}, + fsctl.Check{Component: fsctl.ComponentJournal, Run: func(context.Context) error { + if stateErr != nil { + return stateErr + } + for _, state := range states { + path := filepath.Join(store, "fs", "sessions", state.SessionID, "journal.jsonl") + if file, err := os.Open(path); err == nil { + _ = file.Close() + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + } + return nil + }}, + fsctl.Check{Component: fsctl.ComponentClient, Run: func(ctx context.Context) error { + result, err := evaluateCompatibility(ctx, store, defaultCompatibilityFlags()) + if err != nil { + return err + } + if len(result.DetectionErrors) != 0 || !result.Evaluation.Approved { + return errors.New("installed Codex clients are not covered by exact compatibility contracts") + } + return nil + }}, + ) + report := fsctl.Doctor(ctx, checks) + report.Storage = storageInventory + report.StorageLimits = storageLimits + report.AvailableBytes = availableBytes + return report +} + +func defaultMountPoint(home string, explicit string) string { + if explicit != "" { + return filepath.Clean(explicit) + } + return filepath.Join(home, "fold-fs") +} + +func verifiedCapability() fsctl.Capability { return fsctl.FSEnginePreview } + +func waitForTarget(ctx context.Context, target string, timeout time.Duration) (vfs.NativeFile, error) { + if timeout <= 0 { + timeout = 5 * time.Second + } + deadline := time.Now().Add(timeout) + for { + file, err := hashPath(target) + if err == nil { + return file, nil + } + if !errors.Is(err, os.ErrNotExist) { + return vfs.NativeFile{}, err + } + if time.Now().After(deadline) { + return vfs.NativeFile{}, os.ErrNotExist + } + select { + case <-ctx.Done(): + return vfs.NativeFile{}, ctx.Err() + case <-time.After(50 * time.Millisecond): + } + } +} + +func waitForTargetMatch(ctx context.Context, target string, expected vfs.NativeFile, timeout time.Duration) (vfs.NativeFile, error) { + if timeout <= 0 { + timeout = 5 * time.Second + } + deadline := time.Now().Add(timeout) + for { + file, err := hashPath(target) + if err == nil && file.Bytes == expected.Bytes && file.SHA256 == expected.SHA256 { + return file, nil + } + if err != nil && !errors.Is(err, os.ErrNotExist) { + return vfs.NativeFile{}, err + } + if time.Now().After(deadline) { + return vfs.NativeFile{}, errors.New("timed out waiting for matching mounted session") + } + select { + case <-ctx.Done(): + return vfs.NativeFile{}, ctx.Err() + case <-time.After(25 * time.Millisecond): + } + } +} + +func hashPath(path string) (vfs.NativeFile, error) { + file, err := os.Open(path) + if err != nil { + return vfs.NativeFile{}, err + } + hasher := sha256.New() + bytesRead, copyErr := io.Copy(hasher, file) + closeErr := file.Close() + if copyErr != nil { + return vfs.NativeFile{}, copyErr + } + if closeErr != nil { + return vfs.NativeFile{}, closeErr + } + return vfs.NativeFile{Path: path, Bytes: bytesRead, SHA256: hex.EncodeToString(hasher.Sum(nil))}, nil +} diff --git a/internal/cli/fs_activation.go b/internal/cli/fs_activation.go new file mode 100644 index 0000000..67c1951 --- /dev/null +++ b/internal/cli/fs_activation.go @@ -0,0 +1,43 @@ +package cli + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/samekind/codexfold/internal/fsctl" +) + +func requireFilesystemActivationAllowed(home string) error { + capability := verifiedCapability() + if capability != fsctl.FSEnginePreview && capability != fsctl.PlatformCanary { + return nil + } + + userHome, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("resolve real Codex home: %w", err) + } + realHomes := []string{filepath.Join(userHome, ".codex")} + if configured := strings.TrimSpace(os.Getenv("CODEX_HOME")); configured != "" { + realHomes = append(realHomes, configured) + } + for _, realHome := range realHomes { + if sameActivationPath(home, realHome) { + return fmt.Errorf("real Codex home activation is disabled while filesystem capability is %s; only an isolated compatibility canary is allowed", capability) + } + } + return nil +} + +func sameActivationPath(left string, right string) bool { + left = filepath.Clean(left) + right = filepath.Clean(right) + if left == right { + return true + } + resolvedLeft, leftErr := filepath.EvalSymlinks(left) + resolvedRight, rightErr := filepath.EvalSymlinks(right) + return leftErr == nil && rightErr == nil && filepath.Clean(resolvedLeft) == filepath.Clean(resolvedRight) +} diff --git a/internal/cli/fs_activation_test.go b/internal/cli/fs_activation_test.go new file mode 100644 index 0000000..83e6abd --- /dev/null +++ b/internal/cli/fs_activation_test.go @@ -0,0 +1,73 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPreviewRejectsEveryRealCodexHomeActivationEntryPoint(t *testing.T) { + userHome := t.TempDir() + t.Setenv("HOME", userHome) + t.Setenv("CODEX_HOME", "") + + codexHome := filepath.Join(userHome, ".codex") + store := filepath.Join(codexHome, "fold-store") + mount := filepath.Join(codexHome, "fold-fs") + native := filepath.Join(codexHome, "fold-native") + if err := os.MkdirAll(codexHome, 0o700); err != nil { + t.Fatal(err) + } + definition := filepath.Join(userHome, "com.codexfold.fs.plist") + if err := os.WriteFile(definition, []byte("fixture"), 0o600); err != nil { + t.Fatal(err) + } + + previousProbe := mountHealthProbe + mountHealthProbe = func(string) error { return nil } + t.Cleanup(func() { mountHealthProbe = previousProbe }) + + want := "real Codex home activation is disabled while filesystem capability is fs-engine-preview" + tests := []struct { + name string + args []string + }{ + { + name: "serve", + args: []string{"fs", "serve", "--codex-home", codexHome, "--store", store, "--mount", mount, "--apply"}, + }, + { + name: "service install", + args: []string{ + "fs", "service", "install", "--codex-home", codexHome, "--store", store, "--mount", mount, + "--native-root", native, "--canonical-namespace", "--plist", definition, "--apply", + }, + }, + { + name: "service start", + args: []string{"fs", "service", "start", "--codex-home", codexHome, "--mount", mount, "--plist", definition, "--apply"}, + }, + { + name: "namespace activate", + args: []string{ + "fs", "namespace", "activate", "--codex-home", codexHome, "--mount", mount, + "--native-root", native, "--apply", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs(test.args) + err := root.Execute() + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("activation error = %v, want %q", err, want) + } + }) + } +} diff --git a/internal/cli/fs_compatibility_import.go b/internal/cli/fs_compatibility_import.go new file mode 100644 index 0000000..2eb7be5 --- /dev/null +++ b/internal/cli/fs_compatibility_import.go @@ -0,0 +1,72 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/compat" + "github.com/spf13/cobra" +) + +type FSCompatibilityImportResult struct { + Contract compat.Contract `json:"contract"` + Path string `json:"path,omitempty"` + DryRun bool `json:"dry_run"` +} + +func newFSCompatibilityImportCommand() *cobra.Command { + var codexHome, storeDir, tracePath, clientKind, clientVersion, platform string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "compatibility-import", + Short: "Import a sanitized exact-version contract from a real filesystem trace", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if !filepath.IsAbs(tracePath) || (clientKind != "cli" && clientKind != "desktop") || clientVersion == "" || platform == "" { + return errors.New("absolute trace path, cli or desktop client kind, version, and platform are required") + } + trace, err := os.Open(tracePath) + if err != nil { + return err + } + contract, parseErr := compat.ParseFSUsage(trace, compat.ContractOptions{Platform: platform, ClientKind: clientKind, ClientVersion: clientVersion}) + closeErr := trace.Close() + if parseErr != nil { + return parseErr + } + if closeErr != nil { + return closeErr + } + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + result := FSCompatibilityImportResult{Contract: contract, DryRun: !apply} + if apply { + result.Path, err = compat.Save(filepath.Join(resolveFoldStore(home, storeDir), "compatibility"), contract) + if err != nil { + return err + } + result.DryRun = false + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "kind=%s version=%s operations=%d dry_run=%t path=%s\n", contract.ClientKind, contract.ClientVersion, len(contract.Operations), result.DryRun, result.Path) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&tracePath, "trace", "", "Absolute path to a real fs_usage-compatible trace") + command.Flags().StringVar(&clientKind, "client-kind", "", "Client kind: cli or desktop") + command.Flags().StringVar(&clientVersion, "client-version", "", "Exact client version represented by the trace") + command.Flags().StringVar(&platform, "platform", runtime.GOOS, "Trace platform") + command.Flags().BoolVar(&apply, "apply", false, "Persist the sanitized contract") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} diff --git a/internal/cli/fs_enroll.go b/internal/cli/fs_enroll.go new file mode 100644 index 0000000..652f685 --- /dev/null +++ b/internal/cli/fs_enroll.go @@ -0,0 +1,313 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/enroll" + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/fsctl" + "github.com/samekind/codexfold/internal/mountfs" + "github.com/samekind/codexfold/internal/pack" + "github.com/samekind/codexfold/internal/storage" + "github.com/samekind/codexfold/internal/vfs" + "github.com/spf13/cobra" +) + +type enrollmentFlags struct { + codexHome string + storeDir string + mountPoint string + nativeRoot string + stableFor time.Duration + batchSize int + canonicalNamespace bool + canary bool + jsonOutput bool +} + +type FSEnrollmentApplyResult struct { + Plan enroll.Plan `json:"plan"` + Apply enroll.ApplyResult `json:"apply"` +} + +type enrollmentCycleReporter func(FSEnrollmentApplyResult, error) + +var runEnrollmentCommand = func(ctx context.Context, args []string) error { + binary, err := os.Executable() + if err != nil { + return err + } + output, err := exec.CommandContext(ctx, binary, args...).CombinedOutput() + if err != nil { + return fmt.Errorf("%s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output))) + } + return nil +} + +var runServiceEnrollmentCycle = runEnrollmentCycle + +func newFSEnrollCommand() *cobra.Command { + command := &cobra.Command{Use: "enroll", Short: "Plan and apply bounded automatic session enrollment"} + command.AddCommand(newFSEnrollPlanCommand()) + command.AddCommand(newFSEnrollApplyCommand()) + return command +} + +func newFSEnrollPlanCommand() *cobra.Command { + var flags enrollmentFlags + var record bool + command := &cobra.Command{ + Use: "plan", + Short: "Report eligible and blocked sessions without changing routes", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + plan, store, err := buildEnrollmentPlan(command.Context(), flags) + if err != nil { + return err + } + if record { + if err := enroll.SaveObservations(enrollmentObservationPath(store), plan.Observations); err != nil { + return err + } + } + if flags.jsonOutput { + return writeJSON(command, plan) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "sessions=%d selected=%d observations=%d\n", len(plan.Decisions), len(plan.Selected), len(plan.Observations)) + return err + }, + } + addEnrollmentFlags(command, &flags) + command.Flags().BoolVar(&record, "record-observations", false, "Persist this read-only stability observation for the next planning cycle") + return command +} + +func newFSEnrollApplyCommand() *cobra.Command { + var flags enrollmentFlags + var apply bool + command := &cobra.Command{ + Use: "apply", + Short: "Apply the selected bounded enrollment batch", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if !apply { + return errors.New("enrollment apply requires --apply") + } + result, err := runEnrollmentCycle(command.Context(), flags) + if err != nil { + return err + } + if flags.jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "selected=%d applied=%d changed=%d managed=%d\n", result.Apply.Selected, result.Apply.Applied, result.Apply.SkippedChanged, result.Apply.SkippedManaged) + return err + }, + } + addEnrollmentFlags(command, &flags) + command.Flags().BoolVar(&apply, "apply", false, "Run the bounded fold, pack, and canonical migration transactions") + return command +} + +func runEnrollmentCycle(ctx context.Context, flags enrollmentFlags) (FSEnrollmentApplyResult, error) { + home, err := codex.ResolveHome(flags.codexHome) + if err != nil { + return FSEnrollmentApplyResult{}, err + } + if err := requireFilesystemActivationAllowed(home); err != nil { + return FSEnrollmentApplyResult{}, err + } + plan, store, err := buildEnrollmentPlan(ctx, flags) + if err != nil { + return FSEnrollmentApplyResult{}, err + } + if err := enroll.SaveObservations(enrollmentObservationPath(store), plan.Observations); err != nil { + return FSEnrollmentApplyResult{}, err + } + mount := defaultMountPoint(home, flags.mountPoint) + nativeRoot := flags.nativeRoot + if nativeRoot == "" { + nativeRoot = filepath.Join(home, "fold-native") + } + applied, err := enroll.Apply(ctx, plan, enroll.ApplyOptions{ + Limit: flags.batchSize, + IsManaged: func(_ context.Context, sessionID string) (bool, error) { + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return false, err + } + for _, state := range states { + if state.SessionID == sessionID { + return true, nil + } + } + return false, nil + }, + Apply: func(ctx context.Context, decision enroll.Decision) error { + if _, err := mountfs.ValidateNativeRollout(ctx, decision.RolloutPath); err != nil { + return fmt.Errorf("native rollout is not eligible for transparent routing: %w", err) + } + return applyEnrollmentCommands(ctx, home, store, mount, nativeRoot, decision.SessionID, flags.canary) + }, + }) + result := FSEnrollmentApplyResult{Plan: plan, Apply: applied} + return result, err +} + +func runPeriodicEnrollment(ctx context.Context, flags enrollmentFlags, interval time.Duration, report enrollmentCycleReporter) { + if interval <= 0 { + return + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + result, err := runServiceEnrollmentCycle(ctx, flags) + if report != nil { + report(result, err) + } + if ctx.Err() != nil { + return + } + } + } +} + +func addEnrollmentFlags(command *cobra.Command, flags *enrollmentFlags) { + command.Flags().StringVar(&flags.codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&flags.storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(&flags.mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + command.Flags().BoolVar(&flags.canonicalNamespace, "canonical-namespace", false, "Plan canonical-path enrollment without changing SQLite routes") + command.Flags().StringVar(&flags.nativeRoot, "native-root", "", "Canonical native backing root; defaults to /fold-native") + command.Flags().DurationVar(&flags.stableFor, "stable-for", time.Hour, "Required unchanged observation window") + command.Flags().IntVar(&flags.batchSize, "batch-size", 1, "Maximum sessions selected per cycle") + command.Flags().BoolVar(&flags.canary, "enrollment-canary", false, "Allow explicit enrollment only in an isolated Codex home while capability remains preview") + command.Flags().BoolVar(&flags.jsonOutput, "json", false, "Emit JSON output") +} + +func buildEnrollmentPlan(ctx context.Context, flags enrollmentFlags) (enroll.Plan, string, error) { + home, err := codex.ResolveHome(flags.codexHome) + if err != nil { + return enroll.Plan{}, "", err + } + store := resolveFoldStore(home, flags.storeDir) + mount := defaultMountPoint(home, flags.mountPoint) + if flags.canary { + userHome, err := os.UserHomeDir() + if err != nil { + return enroll.Plan{}, "", err + } + if err := validateCompatibilityCanary(home, filepath.Join(userHome, ".codex"), store, flags.canonicalNamespace, compatibilityFlags{cliPath: "none", desktopPath: "none"}); err != nil { + return enroll.Plan{}, "", err + } + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return enroll.Plan{}, "", err + } + writers, err := enrollmentWriterProbe(ctx, sessions) + if err != nil { + return enroll.Plan{}, "", fmt.Errorf("probe native session writers: %w", err) + } + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return enroll.Plan{}, "", err + } + managed := make(map[string]struct{}, len(states)) + for _, state := range states { + managed[state.SessionID] = struct{}{} + } + observations, err := enroll.LoadObservations(enrollmentObservationPath(store)) + if err != nil { + return enroll.Plan{}, "", err + } + doctorHealthy := requireEnrollmentStorageHealth(ctx, store) == nil + compatibilityApproved := flags.canary + if !flags.canary { + compatibility, err := evaluateCompatibility(ctx, store, defaultCompatibilityFlags()) + if err == nil { + compatibilityApproved = len(compatibility.DetectionErrors) == 0 && compatibility.Evaluation.Approved + } + } + mountHealthy := mountHealthProbe(mount) == nil + guard, err := storage.DefaultGuard(store) + if err != nil { + return enroll.Plan{}, "", err + } + plan, err := enroll.Build(ctx, enroll.Input{ + Sessions: sessions, Managed: managed, Previous: observations, Now: time.Now(), + Policy: enroll.Policy{StableFor: flags.stableFor, BatchSize: flags.batchSize, ArchivedOnly: true}, + Gates: enroll.Gates{ + DoctorHealthy: doctorHealthy, CompatibilityApproved: compatibilityApproved, MountHealthy: mountHealthy, + CanonicalNamespace: flags.canonicalNamespace, EnrollmentAllowed: flags.canary || automaticEnrollmentAllowed(verifiedCapability()), + }, + WriterActive: func(_ context.Context, session codex.Session) (bool, error) { + return writers[session.ID], nil + }, + Budget: guard, + }) + return plan, store, err +} + +func requireEnrollmentStorageHealth(ctx context.Context, store string) error { + foldReport, err := fold.Doctor(ctx, store) + if err != nil { + return err + } + if foldReport.IssueCount != 0 { + return fmt.Errorf("fold doctor reported %d issues", foldReport.IssueCount) + } + packReport, err := pack.Doctor(ctx, store) + if err != nil { + return err + } + if packReport.IssueCount == 0 { + return nil + } + if _, currentErr := os.Lstat(filepath.Join(filepath.Clean(store), "packs", "CURRENT")); errors.Is(currentErr, os.ErrNotExist) { + states, stateErr := vfs.DiscoverSessionStates(store) + if stateErr != nil { + return stateErr + } + if len(states) == 0 { + return nil + } + } + return fmt.Errorf("pack doctor reported %d issues", packReport.IssueCount) +} + +func automaticEnrollmentAllowed(capability fsctl.Capability) bool { + return capability == fsctl.CrossPlatformReady || strings.HasPrefix(string(capability), "production-ready:") +} + +func enrollmentObservationPath(store string) string { + return filepath.Join(filepath.Clean(store), "enrollment", "observations.json") +} + +func applyEnrollmentCommands(ctx context.Context, home string, store string, mount string, nativeRoot string, sessionID string, canary bool) error { + commands := [][]string{ + {"fold", sessionID, "--codex-home", home, "--store", store, "--apply", "--overwrite"}, + {"pack", "build", "--codex-home", home, "--store", store}, + {"fs", "migrate", sessionID, "--codex-home", home, "--store", store, "--mount", mount, "--canonical-namespace", "--native-root", nativeRoot, "--apply"}, + } + if canary { + commands[2] = append(commands[2], "--compatibility-canary", "--cli", "none", "--desktop-app", "none") + } + for _, command := range commands { + if err := runEnrollmentCommand(ctx, command); err != nil { + return err + } + } + return nil +} diff --git a/internal/cli/fs_enroll_writer.go b/internal/cli/fs_enroll_writer.go new file mode 100644 index 0000000..8cccb0e --- /dev/null +++ b/internal/cli/fs_enroll_writer.go @@ -0,0 +1,62 @@ +package cli + +import ( + "path/filepath" + "strings" + + "github.com/samekind/codexfold/internal/codex" +) + +var enrollmentWriterProbe = detectEnrollmentWriters + +func parseEnrollmentWriterSnapshot(output []byte, sessions []codex.Session) map[string]bool { + aliases := make(map[string][]string, len(sessions)*2) + for _, session := range sessions { + for _, path := range enrollmentPathAliases(session.RolloutPath) { + aliases[path] = append(aliases[path], session.ID) + } + } + writers := make(map[string]bool) + var access string + var name string + flush := func() { + if name == "" || !strings.ContainsAny(access, "wu") { + access = "" + name = "" + return + } + for _, sessionID := range aliases[canonicalEnrollmentPath(name)] { + writers[sessionID] = true + } + access = "" + name = "" + } + for _, line := range strings.Split(string(output), "\n") { + if line == "" { + continue + } + switch line[0] { + case 'p', 'f': + flush() + case 'a': + access = line[1:] + case 'n': + name = strings.TrimSuffix(line[1:], " (deleted)") + } + } + flush() + return writers +} + +func enrollmentPathAliases(path string) []string { + path = canonicalEnrollmentPath(path) + aliases := []string{path} + if resolved, err := filepath.EvalSymlinks(path); err == nil && resolved != path { + aliases = append(aliases, resolved) + } + return aliases +} + +func canonicalEnrollmentPath(path string) string { + return filepath.Clean(path) +} diff --git a/internal/cli/fs_enroll_writer_other.go b/internal/cli/fs_enroll_writer_other.go new file mode 100644 index 0000000..d22caed --- /dev/null +++ b/internal/cli/fs_enroll_writer_other.go @@ -0,0 +1,14 @@ +//go:build !darwin && !linux && !windows + +package cli + +import ( + "context" + "errors" + + "github.com/samekind/codexfold/internal/codex" +) + +func detectEnrollmentWriters(context.Context, []codex.Session) (map[string]bool, error) { + return nil, errors.New("native writer probe is unavailable on this platform") +} diff --git a/internal/cli/fs_enroll_writer_test.go b/internal/cli/fs_enroll_writer_test.go new file mode 100644 index 0000000..938471a --- /dev/null +++ b/internal/cli/fs_enroll_writer_test.go @@ -0,0 +1,92 @@ +package cli + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/enroll" +) + +func TestParseEnrollmentWriterSnapshotBlocksWriteAndUpdateDescriptorsOnly(t *testing.T) { + root := t.TempDir() + sessions := []codex.Session{ + {ID: "read", RolloutPath: filepath.Join(root, "read.jsonl")}, + {ID: "write", RolloutPath: filepath.Join(root, "write.jsonl")}, + {ID: "update", RolloutPath: filepath.Join(root, "update.jsonl")}, + } + output := []byte("p1\nf3\nar\nn" + sessions[0].RolloutPath + "\n" + + "f4\naw\nn" + sessions[1].RolloutPath + "\n" + + "f5\nau\nn" + sessions[2].RolloutPath + "\n") + writers := parseEnrollmentWriterSnapshot(output, sessions) + if writers["read"] || !writers["write"] || !writers["update"] { + t.Fatalf("writer snapshot = %#v", writers) + } +} + +func TestParseEnrollmentWriterSnapshotBlocksEverySessionSharingAPath(t *testing.T) { + path := filepath.Join(t.TempDir(), "shared.jsonl") + sessions := []codex.Session{{ID: "first", RolloutPath: path}, {ID: "second", RolloutPath: path}} + writers := parseEnrollmentWriterSnapshot([]byte("p1\nf3\naw\nn"+path+"\n"), sessions) + if !writers["first"] || !writers["second"] { + t.Fatalf("shared-path writer snapshot = %#v", writers) + } +} + +func TestEnrollmentWriterProbeFailureFailsClosed(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + oldProbe := enrollmentWriterProbe + defer func() { enrollmentWriterProbe = oldProbe }() + enrollmentWriterProbe = func(context.Context, []codex.Session) (map[string]bool, error) { + return nil, context.DeadlineExceeded + } + _, _, err := buildEnrollmentPlan(context.Background(), enrollmentFlags{ + codexHome: home, storeDir: storeDir, mountPoint: filepath.Join(home, "mount"), + nativeRoot: filepath.Join(home, "fold-native"), canonicalNamespace: true, canary: true, + }) + if err == nil { + t.Fatal("writer probe failure did not stop enrollment planning") + } +} + +func TestEnrollmentPlanBlocksSessionReportedByNativeWriterProbe(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + oldProbe := enrollmentWriterProbe + defer func() { enrollmentWriterProbe = oldProbe }() + enrollmentWriterProbe = func(context.Context, []codex.Session) (map[string]bool, error) { + return map[string]bool{"session": true}, nil + } + oldMountProbe := mountHealthProbe + mountHealthProbe = func(string) error { return nil } + defer func() { mountHealthProbe = oldMountProbe }() + info, err := os.Stat(nativePath) + if err != nil { + t.Fatal(err) + } + if err := enroll.SaveObservations(enrollmentObservationPath(storeDir), enroll.Observations{"session": { + Path: nativePath, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), StableSinceUnixNano: time.Now().Add(-time.Hour).UnixNano(), + }}); err != nil { + t.Fatal(err) + } + plan, _, err := buildEnrollmentPlan(context.Background(), enrollmentFlags{ + codexHome: home, storeDir: storeDir, mountPoint: filepath.Join(home, "mount"), + nativeRoot: filepath.Join(home, "fold-native"), canonicalNamespace: true, + canary: true, stableFor: time.Nanosecond, batchSize: 1, + }) + if err != nil { + t.Fatal(err) + } + if len(plan.Selected) != 0 || len(plan.Decisions) != 1 { + t.Fatalf("writer-active plan = %#v", plan) + } + found := false + for _, reason := range plan.Decisions[0].Reasons { + found = found || reason == enroll.ReasonWriterActive + } + if !found { + t.Fatalf("writer-active reason missing: %#v", plan.Decisions[0]) + } +} diff --git a/internal/cli/fs_enroll_writer_unix.go b/internal/cli/fs_enroll_writer_unix.go new file mode 100644 index 0000000..e87a1a6 --- /dev/null +++ b/internal/cli/fs_enroll_writer_unix.go @@ -0,0 +1,31 @@ +//go:build darwin || linux + +package cli + +import ( + "context" + "fmt" + "os" + "os/exec" + + "github.com/samekind/codexfold/internal/codex" +) + +func detectEnrollmentWriters(ctx context.Context, sessions []codex.Session) (map[string]bool, error) { + if len(sessions) == 0 { + return map[string]bool{}, nil + } + lsof := "/usr/sbin/lsof" + if _, err := os.Stat(lsof); err != nil { + resolved, lookErr := exec.LookPath("lsof") + if lookErr != nil { + return nil, fmt.Errorf("native writer probe requires lsof: %w", lookErr) + } + lsof = resolved + } + output, err := exec.CommandContext(ctx, lsof, "-n", "-P", "-F", "pfan").Output() + if err != nil { + return nil, fmt.Errorf("run native writer probe: %w", err) + } + return parseEnrollmentWriterSnapshot(output, sessions), nil +} diff --git a/internal/cli/fs_enroll_writer_windows.go b/internal/cli/fs_enroll_writer_windows.go new file mode 100644 index 0000000..a895b48 --- /dev/null +++ b/internal/cli/fs_enroll_writer_windows.go @@ -0,0 +1,46 @@ +//go:build windows + +package cli + +import ( + "context" + "errors" + "fmt" + + "github.com/samekind/codexfold/internal/codex" + "golang.org/x/sys/windows" +) + +func detectEnrollmentWriters(ctx context.Context, sessions []codex.Session) (map[string]bool, error) { + writers := make(map[string]bool) + for _, session := range sessions { + if err := ctx.Err(); err != nil { + return nil, err + } + path, err := windows.UTF16PtrFromString(session.RolloutPath) + if err != nil { + return nil, fmt.Errorf("encode rollout path for native handle probe: %w", err) + } + handle, err := windows.CreateFile( + path, + windows.GENERIC_READ|windows.GENERIC_WRITE, + 0, + nil, + windows.OPEN_EXISTING, + windows.FILE_ATTRIBUTE_NORMAL, + 0, + ) + if err == nil { + if closeErr := windows.CloseHandle(handle); closeErr != nil { + return nil, fmt.Errorf("close native rollout probe: %w", closeErr) + } + continue + } + if errors.Is(err, windows.ERROR_SHARING_VIOLATION) || errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + writers[session.ID] = true + continue + } + return nil, fmt.Errorf("probe native rollout handle %s: %w", session.ID, err) + } + return writers, nil +} diff --git a/internal/cli/fs_namespace.go b/internal/cli/fs_namespace.go new file mode 100644 index 0000000..ca68f09 --- /dev/null +++ b/internal/cli/fs_namespace.go @@ -0,0 +1,192 @@ +package cli + +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/sessionns" + "github.com/samekind/codexfold/internal/vfs" + "github.com/spf13/cobra" +) + +type FSNamespaceResult struct { + sessionns.Result + DryRun bool `json:"dry_run"` +} + +func newFSNamespaceCommand() *cobra.Command { + command := &cobra.Command{Use: "namespace", Short: "Manage the canonical Codex session directory namespace"} + command.AddCommand(newFSNamespaceStatusCommand()) + command.AddCommand(newFSNamespaceActivateCommand()) + command.AddCommand(newFSNamespaceDeactivateCommand()) + command.AddCommand(newFSNamespaceRecoverCommand()) + return command +} + +func newFSNamespaceStatusCommand() *cobra.Command { + var codexHome, mountPoint, nativeRoot string + var jsonOutput bool + command := &cobra.Command{ + Use: "status", + Short: "Inspect the canonical namespace without changing it", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + options, err := namespaceOptions(codexHome, mountPoint, nativeRoot) + if err != nil { + return err + } + result, err := sessionns.Inspect(options) + if err != nil { + return err + } + return writeNamespaceResult(command, FSNamespaceResult{Result: result, DryRun: true}, jsonOutput) + }, + } + addNamespaceFlags(command, &codexHome, &mountPoint, &nativeRoot) + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSNamespaceActivateCommand() *cobra.Command { + var codexHome, mountPoint, nativeRoot string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "activate", + Short: "Atomically route Codex session directories through the canonical mount", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + options, err := namespaceOptions(codexHome, mountPoint, nativeRoot) + if err != nil { + return err + } + if apply { + if err := requireFilesystemActivationAllowed(options.Home); err != nil { + return err + } + } + if !apply { + result, err := sessionns.Inspect(options) + if err != nil { + return err + } + return writeNamespaceResult(command, FSNamespaceResult{Result: result, DryRun: true}, jsonOutput) + } + if err := mountHealthProbe(options.Mount); err != nil { + return fmt.Errorf("canonical filesystem mount is not healthy: %w", err) + } + result, err := sessionns.Activate(options) + if err != nil { + return err + } + return writeNamespaceResult(command, FSNamespaceResult{Result: result}, jsonOutput) + }, + } + addNamespaceFlags(command, &codexHome, &mountPoint, &nativeRoot) + command.Flags().BoolVar(&apply, "apply", false, "Move native directories and install canonical links") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSNamespaceDeactivateCommand() *cobra.Command { + var codexHome, storeDir, mountPoint, nativeRoot string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "deactivate", + Short: "Restore ordinary Codex session directories from the retained native tree", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + options, err := namespaceOptions(codexHome, mountPoint, nativeRoot) + if err != nil { + return err + } + if !apply { + result, err := sessionns.Inspect(options) + if err != nil { + return err + } + return writeNamespaceResult(command, FSNamespaceResult{Result: result, DryRun: true}, jsonOutput) + } + states, err := vfs.DiscoverSessionStates(resolveFoldStore(options.Home, storeDir)) + if err != nil { + return err + } + if len(states) != 0 { + return errors.New("rollback all managed sessions before deactivating the namespace") + } + if err := mountHealthProbe(options.Mount); err == nil { + return errors.New("stop the filesystem service before deactivating the namespace") + } + result, err := sessionns.Deactivate(options) + if err != nil { + return err + } + return writeNamespaceResult(command, FSNamespaceResult{Result: result}, jsonOutput) + }, + } + addNamespaceFlags(command, &codexHome, &mountPoint, &nativeRoot) + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().BoolVar(&apply, "apply", false, "Remove canonical links and restore native directories") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSNamespaceRecoverCommand() *cobra.Command { + var codexHome, mountPoint, nativeRoot string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "recover", + Short: "Recover an interrupted namespace activation or deactivation", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + options, err := namespaceOptions(codexHome, mountPoint, nativeRoot) + if err != nil { + return err + } + var result sessionns.Result + if apply { + result, err = sessionns.Recover(options) + } else { + result, err = sessionns.Inspect(options) + } + if err != nil { + return err + } + return writeNamespaceResult(command, FSNamespaceResult{Result: result, DryRun: !apply}, jsonOutput) + }, + } + addNamespaceFlags(command, &codexHome, &mountPoint, &nativeRoot) + command.Flags().BoolVar(&apply, "apply", false, "Apply journal recovery") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func namespaceOptions(codexHome string, mountPoint string, nativeRoot string) (sessionns.Options, error) { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return sessionns.Options{}, err + } + mount := defaultMountPoint(home, mountPoint) + if nativeRoot == "" { + nativeRoot = filepath.Join(home, "fold-native") + } + if !filepath.IsAbs(nativeRoot) { + return sessionns.Options{}, errors.New("native root must be absolute") + } + return sessionns.Options{Home: home, Mount: mount, NativeRoot: filepath.Clean(nativeRoot), MountProbe: mountHealthProbe}, nil +} + +func addNamespaceFlags(command *cobra.Command, codexHome *string, mountPoint *string, nativeRoot *string) { + command.Flags().StringVar(codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(mountPoint, "mount", "", "Canonical mount path; defaults to /fold-fs") + command.Flags().StringVar(nativeRoot, "native-root", "", "Retained native tree; defaults to /fold-native") +} + +func writeNamespaceResult(command *cobra.Command, result FSNamespaceResult, jsonOutput bool) error { + if jsonOutput { + return writeJSON(command, result) + } + _, err := fmt.Fprintf(command.OutOrStdout(), "active=%t recovered=%t dry_run=%t home=%s mount=%s native_root=%s\n", result.Active, result.Recovered, result.DryRun, result.Home, result.Mount, result.NativeRoot) + return err +} diff --git a/internal/cli/fs_reconcile.go b/internal/cli/fs_reconcile.go new file mode 100644 index 0000000..53ab222 --- /dev/null +++ b/internal/cli/fs_reconcile.go @@ -0,0 +1,114 @@ +package cli + +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/samekind/codexfold/internal/reconcile" + "github.com/spf13/cobra" +) + +type FSReconcileRolloutResult struct { + reconcile.Result + DryRun bool `json:"dry_run"` +} + +func newFSRepairRolloutCommand() *cobra.Command { + var outputPath, orphanPath string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "repair-rollout ", + Short: "Recover deterministically interleaved JSONL writes into a separate rollout", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + if !apply { + return errors.New("repair-rollout requires --apply and writes only to a separate --output") + } + if !filepath.IsAbs(args[0]) || !filepath.IsAbs(outputPath) { + return errors.New("source and --output paths must be absolute") + } + result, err := reconcile.RepairWithOptions(args[0], outputPath, reconcile.RepairOptions{AllowOrphans: orphanPath != "", OrphanPath: orphanPath, Context: command.Context()}) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), + "physical=%d invalid=%d reconstructed=%d conversations=%d preserved=%d reconstructed_conversations=%d conversation_verified=%t orphans=%d output=%d regressions=%d max_buffer=%d path=%s sha256=%s\n", + result.PhysicalLines, + result.InvalidPhysicalLines, + result.ReconstructedRecords, + result.SourceConversationRecords, + result.PreservedConversationRecords, + result.ReconstructedConversationRecords, + result.ConversationIntegrityVerified, + result.OrphanLines, + result.OutputRecords, + result.TimestampRegressions, + result.MaximumBufferedBytes, + result.OutputPath, + result.OutputSHA256, + ) + return err + }, + } + command.Flags().StringVar(&outputPath, "output", "", "Absolute output path for the repaired rollout") + command.Flags().StringVar(&orphanPath, "orphans", "", "Optional absolute path for unrecoverable raw fragments; enables salvage mode") + command.Flags().BoolVar(&apply, "apply", false, "Write a separately verified repaired rollout") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSReconcileRolloutCommand() *cobra.Command { + var outputPath string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "reconcile-rollout ", + Short: "Reconcile two monotonic rollout branches without replacing either source", + Args: cobra.ExactArgs(2), + RunE: func(command *cobra.Command, args []string) error { + if !filepath.IsAbs(args[0]) || !filepath.IsAbs(args[1]) { + return errors.New("base and branch paths must be absolute") + } + var result reconcile.Result + var err error + if apply { + if !filepath.IsAbs(outputPath) { + return errors.New("--output must be absolute with --apply") + } + result, err = reconcile.MergeWithOptions(args[0], args[1], outputPath, reconcile.MergeOptions{Context: command.Context()}) + } else { + result, err = reconcile.Analyze(args[0], args[1]) + } + if err != nil { + return err + } + wrapped := FSReconcileRolloutResult{Result: result, DryRun: !apply} + if jsonOutput { + return writeJSON(command, wrapped) + } + _, err = fmt.Fprintf(command.OutOrStdout(), + "base=%d branch=%d shared=%d base_only=%d added=%d output=%d regressions=%d/%d/%d dry_run=%t path=%s sha256=%s\n", + result.Base.Records, + result.Branch.Records, + result.SharedRecords, + result.BaseOnlyRecords, + result.AddedFromBranch, + result.OutputRecords, + result.Base.TimestampRegressions, + result.Branch.TimestampRegressions, + result.OutputRegressions, + wrapped.DryRun, + result.OutputPath, + result.OutputSHA256, + ) + return err + }, + } + command.Flags().StringVar(&outputPath, "output", "", "Absolute output path for the reconciled rollout") + command.Flags().BoolVar(&apply, "apply", false, "Write a separately verified reconciled rollout") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} diff --git a/internal/cli/fs_service.go b/internal/cli/fs_service.go new file mode 100644 index 0000000..b2acfc3 --- /dev/null +++ b/internal/cli/fs_service.go @@ -0,0 +1,1825 @@ +package cli + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/samekind/codexfold/internal/buildid" + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/mountfs" + "github.com/samekind/codexfold/internal/service" + "github.com/samekind/codexfold/internal/storage" + "github.com/samekind/codexfold/internal/vfs" + "github.com/spf13/cobra" +) + +const serviceLabel = "com.codexfold.fs" + +// Re-registering a signed FSKit extension after an in-place app Contents swap +// can take noticeably longer than an ordinary LaunchAgent restart. Keep the +// update transaction bounded, but allow the system extension service enough +// time to replace its prior endpoint before declaring a healthy mount failed. +const nativeFSKitStartupTimeout = 120 * time.Second + +type operationTrace struct { + mu sync.Mutex + file *os.File +} + +func newOperationRecorder(path string) (func(string), io.Closer, error) { + if !filepath.IsAbs(path) { + return nil, nil, errors.New("operation trace path must be absolute") + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, nil, err + } + file, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return nil, nil, err + } + trace := &operationTrace{file: file} + return trace.record, trace, nil +} + +func (t *operationTrace) record(operation string) { + t.mu.Lock() + defer t.mu.Unlock() + _, _ = fmt.Fprintf(t.file, "%d %s\n", time.Now().UnixNano(), operation) +} + +func (t *operationTrace) Close() error { + t.mu.Lock() + defer t.mu.Unlock() + if err := t.file.Sync(); err != nil { + _ = t.file.Close() + return err + } + return t.file.Close() +} + +type FSServiceActionResult struct { + Action string `json:"action"` + Path string `json:"path,omitempty"` + SupervisorPath string `json:"supervisor_path,omitempty"` + DryRun bool `json:"dry_run"` +} + +type FSServiceInstallResult struct { + Path string `json:"path"` + DryRun bool `json:"dry_run"` + Bytes int `json:"bytes"` + SupervisorPath string `json:"supervisor_path,omitempty"` + SupervisorBytes int `json:"supervisor_bytes,omitempty"` + FSKitAppPath string `json:"fskit_app_path,omitempty"` + FSKitLauncherPath string `json:"fskit_launcher_path,omitempty"` + FSKitResourcePath string `json:"fskit_resource_path,omitempty"` + FSKitAppChanged bool `json:"fskit_app_changed,omitempty"` + BinarySourcePath string `json:"binary_source_path,omitempty"` + BinaryCurrentSHA256 string `json:"binary_current_sha256,omitempty"` + BinaryCandidateSHA256 string `json:"binary_candidate_sha256,omitempty"` + BinaryChanged bool `json:"binary_changed,omitempty"` +} + +type FSServiceBinaryUpdateResult struct { + Candidate string `json:"candidate"` + Target string `json:"target"` + CurrentSHA256 string `json:"current_sha256"` + CandidateSHA256 string `json:"candidate_sha256"` + Changed bool `json:"changed"` + DryRun bool `json:"dry_run"` +} + +type FSUpdatePreflightResult struct { + DoctorHealthy bool `json:"doctor_healthy"` + Compatibility FSCompatibilityResult `json:"compatibility"` + Decision service.UpdateDecision `json:"decision"` + QuarantinedSessions int `json:"quarantined_sessions"` +} + +func newFSServiceCommand() *cobra.Command { + command := &cobra.Command{Use: "service", Short: "Manage the transparent filesystem service"} + command.AddCommand(newFSServiceInstallCommand()) + command.AddCommand(newFSServiceStartCommand()) + command.AddCommand(newFSServiceStopCommand()) + command.AddCommand(newFSServiceRestartCommand()) + command.AddCommand(newFSServiceStatusCommand()) + command.AddCommand(newFSServiceUpdateBinaryCommand()) + command.AddCommand(newFSServiceUpdatePreflightCommand()) + addPlatformServiceCommands(command) + return command +} + +func newFSServiceUpdateBinaryCommand() *cobra.Command { + var codexHome, mountPoint, definitionPath string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "update-binary ", + Short: "Atomically replace, restart, verify, and roll back the service binary", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + candidate, err := filepath.Abs(args[0]) + if err != nil { + return err + } + definition, err := resolveServiceDefinitionPath(definitionPath) + if err != nil { + return err + } + platform, err := service.CurrentPlatform() + if err != nil { + return err + } + target, err := service.DefinitionBinary(platform, definition) + if err != nil { + return err + } + currentSHA256, err := buildid.FileSHA256(target) + if err != nil { + return err + } + candidateSHA256, err := buildid.FileSHA256(candidate) + if err != nil { + return err + } + result := FSServiceBinaryUpdateResult{ + Candidate: candidate, Target: target, CurrentSHA256: currentSHA256, + CandidateSHA256: candidateSHA256, Changed: currentSHA256 != candidateSHA256, DryRun: !apply, + } + if apply && result.Changed { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + if err := requireFilesystemActivationAllowed(home); err != nil { + return err + } + mount := defaultMountPoint(home, mountPoint) + update, err := service.StageBinaryUpdate(candidate, target) + if err != nil { + return err + } + if err := stopPlatformService(command.Context(), platform, definition); err != nil { + _ = update.Commit() + return fmt.Errorf("stop filesystem service before binary update: %w", err) + } + if err := update.Promote(); err != nil { + rollbackErr := update.Rollback() + var cleanupErr error + var restartErr error + if rollbackErr == nil { + cleanupErr = update.Commit() + restartErr = startPlatformService(command.Context(), platform, definition, mount) + } + return errors.Join(fmt.Errorf("promote filesystem service binary: %w", err), rollbackErr, cleanupErr, restartErr) + } + if err := startPlatformService(command.Context(), platform, definition, mount); err != nil { + _ = stopPlatformService(command.Context(), platform, definition) + rollbackErr := update.Rollback() + var restartErr error + if rollbackErr == nil { + restartErr = startPlatformService(command.Context(), platform, definition, mount) + _ = update.Commit() + } + return errors.Join(fmt.Errorf("start verified filesystem service binary: %w", err), rollbackErr, restartErr) + } + if err := update.Commit(); err != nil { + return fmt.Errorf("remove filesystem binary rollback artifact: %w", err) + } + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=%t changed=%t target=%s current=%s candidate=%s\n", result.DryRun, result.Changed, result.Target, result.CurrentSHA256, result.CandidateSHA256) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + addServiceDefinitionFlags(command, &definitionPath) + command.Flags().BoolVar(&apply, "apply", false, "Stop, replace, restart, and verify the service binary") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSServiceInstallCommand() *cobra.Command { + var codexHome, storeDir, mountPoint, binaryPath, binarySource, plistPath, logDir, nativeRoot, operationTracePath string + var frontend, fskitResource, fskitAppPath, fskitAppSource, label string + var enrollmentInterval, enrollmentStableFor time.Duration + var enrollmentBatchSize int + var apply, canonicalNamespace, enrollmentCanary, jsonOutput bool + command := &cobra.Command{ + Use: "install", + Short: "Render and optionally start the native platform service", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) (runErr error) { + if label == "" { + label = serviceLabel + } + home, store, mount, binary, plist, logs, err := resolveServicePaths(codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir, label) + if err != nil { + return err + } + var binaryCurrentSHA256, binaryCandidateSHA256 string + if binarySource != "" { + binarySource, err = filepath.Abs(binarySource) + if err != nil { + return err + } + if filepath.Clean(binarySource) == filepath.Clean(binary) { + return errors.New("candidate binary must be separate from the installed target") + } + binaryCurrentSHA256, err = buildid.FileSHA256(binary) + if err != nil { + return err + } + binaryCandidateSHA256, err = buildid.FileSHA256(binarySource) + if err != nil { + return err + } + } + if apply { + if err := requireFilesystemActivationAllowed(home); err != nil { + return err + } + } + platform, err := service.CurrentPlatform() + if err != nil { + return err + } + if frontend == "" { + frontend = "fuse" + } + if label != serviceLabel && platform != service.PlatformLaunchd { + return errors.New("custom service labels are supported only for macOS LaunchAgents") + } + hadExistingDefinition := false + if apply { + if info, statErr := os.Stat(plist); statErr == nil { + if !info.Mode().IsRegular() { + return errors.New("installed service definition is not a regular file") + } + hadExistingDefinition = true + } else if !errors.Is(statErr, os.ErrNotExist) { + return statErr + } + } + if frontend == "native-fskit" { + if platform != service.PlatformLaunchd { + return errors.New("native-fskit service frontend is available only on macOS") + } + canonicalNamespace = true + userHome, err := os.UserHomeDir() + if err != nil { + return err + } + if fskitAppPath == "" { + fskitAppPath = service.DefaultFSKitAppPath(userHome) + } + fskitAppPath, err = filepath.Abs(fskitAppPath) + if err != nil { + return err + } + } + var appTransaction fsKitAppTransaction + var binaryUpdate *service.BinaryUpdate + var definitionUpdates []*service.DefinitionUpdate + rollbackInstall := false + restartPreviousService := false + defer func() { + if !rollbackInstall { + return + } + rollbackContext, cancel := context.WithTimeout(context.Background(), 2*nativeFSKitStartupTimeout+15*time.Second) + defer cancel() + runErr = errors.Join(runErr, rollbackFailedServiceInstall( + rollbackContext, platform, plist, mount, definitionUpdates, appTransaction, binaryUpdate, + restartPreviousService, stopPlatformService, startPlatformService, + )) + }() + if apply && frontend == "native-fskit" && hadExistingDefinition { + rollbackInstall = true + restartPreviousService = true + stopErr := stopPlatformService(command.Context(), platform, plist) + if err := waitPlatformServiceInactive(command.Context(), platform, plist, mount, 30*time.Second); err != nil { + return errors.Join(stopErr, err) + } + } + if apply && binarySource != "" { + binaryUpdate, err = service.StageBinaryUpdate(binarySource, binary) + if err != nil { + return err + } + rollbackInstall = true + } + if apply && frontend == "native-fskit" { + if fskitAppSource != "" { + fskitAppSource, err = filepath.Abs(fskitAppSource) + if err != nil { + return err + } + } + appTransaction, err = prepareFSKitAppPlatform(command.Context(), fskitAppSource, fskitAppPath) + if err != nil { + return err + } + rollbackInstall = true + restartPreviousService = hadExistingDefinition && appTransaction.Changed() + if fskitResource == "" { + fskitResource = filepath.Join(appTransaction.AppGroupPath(), service.FSKitResourceDirectoryName) + } + } + if frontend == "native-fskit" && fskitResource == "" { + userHome, err := os.UserHomeDir() + if err != nil { + return err + } + fskitResource = service.DefaultFSKitResourcePath(userHome) + } + if frontend == "native-fskit" && apply { + within, err := filepath.Rel(appTransaction.AppGroupPath(), filepath.Clean(fskitResource)) + if err != nil || within == ".." || strings.HasPrefix(within, ".."+string(filepath.Separator)) { + return errors.New("native FSKit resource must remain inside the configured App Group") + } + } + launcherPath := "" + if frontend == "native-fskit" { + launcherPath, err = service.FSKitHostLauncherPath(fskitAppPath) + if err != nil { + return err + } + } + if canonicalNamespace { + if nativeRoot == "" { + nativeRoot = filepath.Join(home, "fold-native") + } + if !filepath.IsAbs(nativeRoot) { + return errors.New("canonical service native root must be absolute") + } + nativeRoot = filepath.Clean(nativeRoot) + } + if enrollmentCanary { + userHome, err := os.UserHomeDir() + if err != nil { + return err + } + if err := validateCompatibilityCanary(home, filepath.Join(userHome, ".codex"), store, canonicalNamespace, compatibilityFlags{cliPath: "none", desktopPath: "none"}); err != nil { + return err + } + } + options := service.Options{ + Label: label, BinaryPath: binary, CodexHome: home, StoreDir: store, MountPoint: mount, + StdoutPath: filepath.Join(logs, "stdout.log"), StderrPath: filepath.Join(logs, "stderr.log"), + CanonicalNamespace: canonicalNamespace, NativeRoot: nativeRoot, OperationTrace: operationTracePath, + EnrollmentInterval: enrollmentInterval, EnrollmentStableFor: enrollmentStableFor, + EnrollmentBatchSize: enrollmentBatchSize, EnrollmentCanary: enrollmentCanary, + Frontend: frontend, FSKitResource: fskitResource, LauncherPath: launcherPath, + } + definition, err := service.RenderDefinition(platform, options) + if err != nil { + return err + } + if apply && frontend == "fuse" && !mountfs.Available() { + return errors.New("service installation requires a platform FUSE build and an authorized host prerequisite") + } + if apply { + if err := os.MkdirAll(logs, 0o700); err != nil { + return err + } + } + written := service.InstallResult{Path: plist, DryRun: !apply, Bytes: len(definition)} + if apply { + update, err := service.StageDefinitionUpdate(plist, definition) + if err != nil { + return err + } + definitionUpdates = append(definitionUpdates, update) + rollbackInstall = true + } else if _, err := service.WriteDefinition(plist, definition, false); err != nil { + return err + } + result := FSServiceInstallResult{ + Path: written.Path, DryRun: written.DryRun, Bytes: written.Bytes, + BinarySourcePath: binarySource, BinaryCurrentSHA256: binaryCurrentSHA256, + BinaryCandidateSHA256: binaryCandidateSHA256, + BinaryChanged: binarySource != "" && binaryCurrentSHA256 != binaryCandidateSHA256, + } + if frontend == "native-fskit" { + result.FSKitAppPath = fskitAppPath + result.FSKitLauncherPath = launcherPath + result.FSKitResourcePath = fskitResource + if appTransaction != nil { + result.FSKitAppChanged = appTransaction.Changed() + } + supervisorDefinition, err := service.RenderLaunchdSupervisor(options) + if err != nil { + return err + } + supervisorPath := nativeFSKitSupervisorDefinitionPath(plist) + if apply { + update, err := service.StageDefinitionUpdate(supervisorPath, supervisorDefinition) + if err != nil { + return err + } + definitionUpdates = append(definitionUpdates, update) + } else if _, err := service.WriteDefinition(supervisorPath, supervisorDefinition, false); err != nil { + return err + } + result.SupervisorPath = supervisorPath + result.SupervisorBytes = len(supervisorDefinition) + } + if apply { + for _, update := range definitionUpdates { + if err := update.Promote(); err != nil { + return err + } + } + if binaryUpdate != nil { + if err := binaryUpdate.Promote(); err != nil { + return err + } + } + } + if apply { + restartPreviousService = restartPreviousService || hadExistingDefinition + if err := installPlatformService(command.Context(), platform, plist, binary, mount); err != nil { + return err + } + } + rollbackInstall = false + cleanupErr := commitDefinitionUpdates(definitionUpdates) + if appTransaction != nil { + if err := appTransaction.Commit(); err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("remove FSKit app rollback artifacts: %w", err)) + } + } + if binaryUpdate != nil { + if err := binaryUpdate.Commit(); err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("remove filesystem binary rollback artifacts: %w", err)) + } + } + if cleanupErr != nil { + return cleanupErr + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "dry_run=%t path=%s bytes=%d supervisor=%s\n", result.DryRun, result.Path, result.Bytes, result.SupervisorPath) + return err + }, + } + addServicePathFlags(command, &codexHome, &storeDir, &mountPoint, &binaryPath, &plistPath, &logDir) + command.Flags().StringVar(&binarySource, "binary-source", "", "Executable candidate to atomically install at --binary") + command.Flags().BoolVar(&canonicalNamespace, "canonical-namespace", false, "Start the service with the canonical Codex session namespace") + command.Flags().StringVar(&frontend, "frontend", "fuse", "Filesystem frontend: fuse or native-fskit") + command.Flags().StringVar(&label, "label", serviceLabel, "Service label; non-default labels are intended for isolated macOS validation") + command.Flags().StringVar(&fskitResource, "fskit-resource", "", "Native FSKit resource inside the App Group; defaults to /native-fskit") + command.Flags().StringVar(&fskitAppPath, "fskit-app", "", "Installed signed FSKit app; defaults to ~/Applications/CodexFoldFSKit.app") + command.Flags().StringVar(&fskitAppSource, "fskit-app-source", "", "Signed FSKit app candidate to atomically install or update at --fskit-app") + command.Flags().StringVar(&nativeRoot, "native-root", "", "Canonical native backing root; defaults to /fold-native") + command.Flags().StringVar(&operationTracePath, "operation-trace", "", "Absolute path for sanitized filesystem operation names") + command.Flags().DurationVar(&enrollmentInterval, "enrollment-interval", 0, "Periodic stable-session enrollment interval; zero disables the loop") + command.Flags().DurationVar(&enrollmentStableFor, "enrollment-stable-for", time.Hour, "Required unchanged interval before periodic enrollment") + command.Flags().IntVar(&enrollmentBatchSize, "enrollment-batch-size", 1, "Maximum sessions enrolled per periodic cycle") + command.Flags().BoolVar(&enrollmentCanary, "enrollment-canary", false, "Enable periodic apply only for an explicitly isolated Codex home while capability remains preview") + command.Flags().BoolVar(&apply, "apply", false, "Write, install, and start the native platform service") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func rollbackDefinitionUpdates(updates []*service.DefinitionUpdate) error { + var result error + for index := len(updates) - 1; index >= 0; index-- { + result = errors.Join(result, updates[index].Rollback()) + } + return errors.Join(result, commitDefinitionUpdates(updates)) +} + +func commitDefinitionUpdates(updates []*service.DefinitionUpdate) error { + var result error + for _, update := range updates { + result = errors.Join(result, update.Commit()) + } + return result +} + +type stopServiceOperation func(context.Context, service.Platform, string) error +type startServiceOperation func(context.Context, service.Platform, string, string) error + +func rollbackFailedServiceInstall( + ctx context.Context, + platform service.Platform, + definitionPath string, + mountPoint string, + definitionUpdates []*service.DefinitionUpdate, + appTransaction fsKitAppTransaction, + binaryUpdate *service.BinaryUpdate, + restartPreviousService bool, + stop stopServiceOperation, + start startServiceOperation, +) error { + if restartPreviousService && stop != nil { + // A failed start normally booted the jobs out already. This extra stop is + // best effort so rollback can also recover failures before the health gate. + _ = stop(ctx, platform, definitionPath) + } + result := rollbackDefinitionUpdates(definitionUpdates) + if appTransaction != nil { + result = errors.Join(result, appTransaction.Rollback(ctx)) + } + if binaryUpdate != nil { + result = errors.Join(result, binaryUpdate.Rollback(), binaryUpdate.Commit()) + } + if restartPreviousService { + if start == nil { + result = errors.Join(result, errors.New("service rollback restart operation is unavailable")) + } else { + result = errors.Join(result, start(ctx, platform, definitionPath, mountPoint)) + } + } + return result +} + +func waitPlatformServiceInactive(ctx context.Context, platform service.Platform, definitionPath string, mountPoint string, timeout time.Duration) error { + if timeout <= 0 { + timeout = 30 * time.Second + } + var lockPaths nativeFSKitProcessLockPaths + checkLocks := false + if platform == service.PlatformLaunchd { + frontend, err := service.DefinitionFrontend(platform, definitionPath) + if err != nil { + return err + } + if frontend == "native-fskit" { + lockPaths, err = nativeFSKitLaunchdLockPaths(definitionPath) + if err != nil { + return err + } + checkLocks = true + } + } + deadline := time.NewTimer(timeout) + defer deadline.Stop() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var daemonLock, supervisorLock service.ProcessLockStatus + var mountPresent bool + var mountPresenceErr error + for { + status, err := platformServiceStatus(ctx, platform, mountPoint, definitionPath) + if err != nil { + return err + } + if checkLocks { + daemonLock, err = service.InspectProcessLock(lockPaths.daemon) + if err != nil { + return fmt.Errorf("inspect daemon process lock: %w", err) + } + supervisorLock, err = service.InspectProcessLock(lockPaths.supervisor) + if err != nil { + return fmt.Errorf("inspect supervisor process lock: %w", err) + } + mountPresent, mountPresenceErr = service.MountPresent(mountPoint) + } + if nativeFSKitServiceInactive(status, daemonLock, supervisorLock, mountPresent, mountPresenceErr) { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline.C: + return fmt.Errorf( + "previous filesystem service did not stop cleanly: daemon=%t supervisor=%t mount_healthy=%t mount_present=%t mount_presence_error=%q daemon_lock=%t daemon_lock_pid=%d supervisor_lock=%t supervisor_lock_pid=%d", + status.DaemonRunning, status.SupervisorRunning, status.MountHealthy, + mountPresent, errorString(mountPresenceErr), + daemonLock.Held, daemonLock.PID, supervisorLock.Held, supervisorLock.PID, + ) + case <-ticker.C: + } + } +} + +func nativeFSKitServiceInactive(status service.Status, daemonLock service.ProcessLockStatus, supervisorLock service.ProcessLockStatus, mountPresent bool, mountPresenceErr error) bool { + return !status.DaemonRunning && !status.SupervisorRunning && !status.MountHealthy && + !mountPresent && mountPresenceErr == nil && !daemonLock.Held && !supervisorLock.Held +} + +func errorString(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +func newFSServiceStartCommand() *cobra.Command { + return newFSServiceLifecycleCommand("start", true) +} + +func newFSServiceStopCommand() *cobra.Command { + return newFSServiceLifecycleCommand("stop", false) +} + +func newFSServiceRestartCommand() *cobra.Command { + return newFSServiceLifecycleCommand("restart", true) +} + +func newFSServiceLifecycleCommand(action string, waitForMount bool) *cobra.Command { + var plistPath, codexHome, mountPoint string + var apply, jsonOutput bool + command := &cobra.Command{ + Use: action, + Short: action + " the filesystem service", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + definition, err := resolveServiceDefinitionPath(plistPath) + if err != nil { + return err + } + var home string + if apply && (action == "start" || action == "restart") { + home, err = codex.ResolveHome(codexHome) + if err != nil { + return err + } + if err := requireFilesystemActivationAllowed(home); err != nil { + return err + } + } + platform, err := service.CurrentPlatform() + if err != nil { + return err + } + result := FSServiceActionResult{Action: action, Path: definition, DryRun: !apply} + if apply { + frontend, err := service.DefinitionFrontend(platform, definition) + if err != nil { + return err + } + if frontend == "native-fskit" { + result.SupervisorPath = nativeFSKitSupervisorDefinitionPath(definition) + } + if waitForMount && frontend == "fuse" && !mountfs.Available() { + return errors.New("service start requires a platform FUSE build and an authorized host prerequisite") + } + if action == "start" { + if err := startPlatformService(command.Context(), platform, definition, defaultMountPoint(home, mountPoint)); err != nil { + return err + } + } else if action == "restart" { + if err := stopPlatformService(command.Context(), platform, definition); err != nil { + return err + } + if err := startPlatformService(command.Context(), platform, definition, defaultMountPoint(home, mountPoint)); err != nil { + return err + } + } else { + if err := stopPlatformService(command.Context(), platform, definition); err != nil { + return err + } + } + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "action=%s dry_run=%t path=%s supervisor=%s\n", action, result.DryRun, definition, result.SupervisorPath) + return err + }, + } + addServiceDefinitionFlags(command, &plistPath) + if waitForMount { + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + } + command.Flags().BoolVar(&apply, "apply", false, "Execute the native service-manager action") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newFSServiceStatusCommand() *cobra.Command { + var codexHome, mountPoint, definitionPath string + var jsonOutput bool + command := &cobra.Command{ + Use: "status", + Short: "Report daemon and mount health separately", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + platform, err := service.CurrentPlatform() + if err != nil { + return err + } + definition, err := resolveServiceDefinitionPath(definitionPath) + if err != nil { + return err + } + status, err := platformServiceStatus(command.Context(), platform, defaultMountPoint(home, mountPoint), definition) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, status) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "daemon=%t supervisor=%t mount=%t build=%t running_build=%s disk_build=%s binary=%s daemon_error=%q supervisor_error=%q mount_error=%q build_error=%q\n", status.DaemonRunning, status.SupervisorRunning, status.MountHealthy, status.Build.Healthy, status.Build.RunningBuildSHA256, status.Build.ConfiguredBuildSHA256, status.Build.ConfiguredBinaryPath, status.DaemonError, status.SupervisorError, status.MountError, status.Build.Error) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + addServiceDefinitionFlags(command, &definitionPath) + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func installPlatformService(ctx context.Context, platform service.Platform, definitionPath string, binaryPath string, mountPoint string) error { + label, err := service.DefinitionLabel(platform, definitionPath) + if err != nil { + return err + } + if platform == service.PlatformWindows { + manager := service.WindowsManager{} + _ = manager.Stop(ctx, label) + if err := manager.Install(ctx, label, binaryPath, definitionPath); err != nil { + return err + } + } + return startPlatformService(ctx, platform, definitionPath, mountPoint) +} + +func startPlatformService(ctx context.Context, platform service.Platform, definitionPath string, mountPoint string) error { + frontend, err := service.DefinitionFrontend(platform, definitionPath) + if err != nil { + return err + } + label, err := service.DefinitionLabel(platform, definitionPath) + if err != nil { + return err + } + switch platform { + case service.PlatformLaunchd: + manager := service.Manager{} + if frontend == "native-fskit" { + supervisorLabel := nativeFSKitSupervisorLabel(label) + supervisorDefinition := nativeFSKitSupervisorDefinitionPath(definitionPath) + _ = manager.Bootout(ctx, supervisorDefinition) + _ = manager.Bootout(ctx, definitionPath) + if err := manager.Enable(ctx, label); err != nil { + return err + } + if err := manager.Enable(ctx, supervisorLabel); err != nil { + return err + } + if err := manager.Bootstrap(ctx, definitionPath); err != nil { + return err + } + if err := manager.Kickstart(ctx, label); err != nil { + _ = manager.Bootout(ctx, definitionPath) + return err + } + if err := manager.Bootstrap(ctx, supervisorDefinition); err != nil { + _ = manager.Bootout(ctx, definitionPath) + return err + } + if err := manager.Kickstart(ctx, supervisorLabel); err != nil { + _ = manager.Bootout(ctx, supervisorDefinition) + _ = manager.Bootout(ctx, definitionPath) + return err + } + if err := waitLaunchdNativeFSKitHealthy(ctx, manager, label, definitionPath, mountPoint, nativeFSKitStartupTimeout); err != nil { + _ = manager.Bootout(ctx, supervisorDefinition) + _ = manager.Bootout(ctx, definitionPath) + return err + } + if err := verifyServiceBuild(service.PlatformLaunchd, definitionPath, mountPoint); err != nil { + _ = manager.Bootout(ctx, supervisorDefinition) + _ = manager.Bootout(ctx, definitionPath) + return err + } + return nil + } + _ = manager.Bootout(ctx, definitionPath) + if err := manager.Enable(ctx, label); err != nil { + return err + } + if err := manager.Bootstrap(ctx, definitionPath); err != nil { + return err + } + if err := manager.Kickstart(ctx, label); err != nil { + _ = manager.Bootout(ctx, definitionPath) + return err + } + if _, err := manager.WaitHealthy(ctx, label, mountPoint, 15*time.Second); err != nil { + _ = manager.Bootout(ctx, definitionPath) + return err + } + if err := verifyServiceBuild(service.PlatformLaunchd, definitionPath, mountPoint); err != nil { + _ = manager.Bootout(ctx, definitionPath) + return err + } + return nil + case service.PlatformSystemd: + unit, err := systemdServiceUnit(definitionPath, label) + if err != nil { + return err + } + manager := service.SystemdManager{} + _ = manager.Stop(ctx, unit) + if err := manager.Start(ctx, unit); err != nil { + return err + } + if _, err := manager.WaitHealthy(ctx, unit, mountPoint, 15*time.Second); err != nil { + _ = manager.Stop(ctx, unit) + return err + } + if err := verifyServiceBuild(service.PlatformSystemd, definitionPath, mountPoint); err != nil { + _ = manager.Stop(ctx, unit) + return err + } + return nil + case service.PlatformWindows: + manager := service.WindowsManager{} + _ = manager.Stop(ctx, label) + if err := manager.Start(ctx, label); err != nil { + return err + } + if _, err := manager.WaitHealthy(ctx, label, mountPoint, 15*time.Second); err != nil { + _ = manager.Stop(ctx, label) + return err + } + if err := verifyServiceBuild(service.PlatformWindows, definitionPath, mountPoint); err != nil { + _ = manager.Stop(ctx, label) + return err + } + return nil + default: + return errors.New("unknown service platform") + } +} + +func stopPlatformService(ctx context.Context, platform service.Platform, definitionPath string) error { + frontend, err := service.DefinitionFrontend(platform, definitionPath) + if err != nil { + return err + } + label, err := service.DefinitionLabel(platform, definitionPath) + if err != nil { + return err + } + switch platform { + case service.PlatformLaunchd: + if frontend == "native-fskit" { + manager := service.Manager{} + supervisorErr := manager.Bootout(ctx, nativeFSKitSupervisorDefinitionPath(definitionPath)) + daemonErr := manager.Bootout(ctx, definitionPath) + return errors.Join(supervisorErr, daemonErr) + } + return (service.Manager{}).Bootout(ctx, definitionPath) + case service.PlatformSystemd: + unit, err := systemdServiceUnit(definitionPath, label) + if err != nil { + return err + } + return (service.SystemdManager{}).Stop(ctx, unit) + case service.PlatformWindows: + return (service.WindowsManager{}).Stop(ctx, label) + default: + return errors.New("unknown service platform") + } +} + +func verifyServiceBuild(platform service.Platform, definitionPath string, mountPoint string) error { + status := service.InspectBuild(platform, definitionPath, mountPoint) + if !status.Healthy { + return fmt.Errorf("filesystem service build verification failed: %s", status.Error) + } + return nil +} + +func platformServiceStatus(ctx context.Context, platform service.Platform, mountPoint string, definitionPath string) (service.Status, error) { + frontend, err := service.DefinitionFrontend(platform, definitionPath) + if err != nil { + return service.Status{}, err + } + label, err := service.DefinitionLabel(platform, definitionPath) + if err != nil { + return service.Status{}, err + } + var status service.Status + switch platform { + case service.PlatformLaunchd: + manager := service.Manager{} + status = manager.Status(ctx, label, mountPoint) + if frontend == "native-fskit" { + lockPaths, err := nativeFSKitLaunchdLockPaths(definitionPath) + if err != nil { + return service.Status{}, err + } + status = validateLaunchdChildProcess(status, lockPaths.daemon, "daemon") + supervisor := validateLaunchdChildProcess( + manager.Status(ctx, nativeFSKitSupervisorLabel(label), mountPoint), + lockPaths.supervisor, + "supervisor", + ) + status.SupervisorRunning = supervisor.DaemonRunning + status.SupervisorPID = supervisor.DaemonPID + status.SupervisorError = supervisor.DaemonError + } + case service.PlatformSystemd: + unit, err := service.SystemdUnitName(label) + if err != nil { + return service.Status{}, err + } + status = (service.SystemdManager{}).Status(ctx, unit, mountPoint) + case service.PlatformWindows: + status = (service.WindowsManager{}).Status(ctx, label, mountPoint) + default: + return service.Status{}, errors.New("unknown service platform") + } + status.Build = service.InspectBuild(platform, definitionPath, mountPoint) + return status, nil +} + +type nativeFSKitProcessLockPaths struct { + daemon string + supervisor string +} + +func nativeFSKitLaunchdLockPaths(definitionPath string) (nativeFSKitProcessLockPaths, error) { + store, err := service.DefinitionStore(service.PlatformLaunchd, definitionPath) + if err != nil { + return nativeFSKitProcessLockPaths{}, err + } + resource, err := service.DefinitionFSKitResource(service.PlatformLaunchd, definitionPath) + if err != nil { + return nativeFSKitProcessLockPaths{}, err + } + return nativeFSKitProcessLockPaths{ + daemon: filepath.Join(store, "fs", "service.lock"), + supervisor: filepath.Join(resource, service.NativeFSKitSupervisorLockName), + }, nil +} + +func validateLaunchdChildProcess(status service.Status, lockPath string, role string) service.Status { + if !status.DaemonRunning { + return status + } + lockStatus, err := service.InspectProcessLock(lockPath) + if err != nil { + status.DaemonRunning = false + status.DaemonError = fmt.Sprintf("inspect %s process lock: %v", role, err) + return status + } + if !lockStatus.Held { + status.DaemonRunning = false + status.DaemonError = fmt.Sprintf("%s host is running without an active child process lock", role) + return status + } + parentPID, err := service.ProcessParentPID(lockStatus.PID) + if err != nil { + status.DaemonRunning = false + status.DaemonError = fmt.Sprintf("inspect %s child process %d: %v", role, lockStatus.PID, err) + return status + } + if parentPID != status.DaemonPID { + status.DaemonRunning = false + status.DaemonError = fmt.Sprintf("%s process lock owner %d belongs to host %d, not launchd host %d", role, lockStatus.PID, parentPID, status.DaemonPID) + } + return status +} + +func waitLaunchdNativeFSKitHealthy(ctx context.Context, manager service.Manager, label string, definitionPath string, mountPoint string, timeout time.Duration) error { + if timeout <= 0 { + timeout = 15 * time.Second + } + lockPaths, err := nativeFSKitLaunchdLockPaths(definitionPath) + if err != nil { + return err + } + deadline := time.NewTimer(timeout) + defer deadline.Stop() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var daemon, supervisor service.Status + supervisorLabel := nativeFSKitSupervisorLabel(label) + for { + daemon = validateLaunchdChildProcess(manager.Status(ctx, label, mountPoint), lockPaths.daemon, "daemon") + supervisor = validateLaunchdChildProcess(manager.Status(ctx, supervisorLabel, mountPoint), lockPaths.supervisor, "supervisor") + if daemon.DaemonRunning && supervisor.DaemonRunning && daemon.MountHealthy { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline.C: + return fmt.Errorf("native FSKit service did not become healthy: daemon=%t supervisor=%t mount=%t daemon_error=%q supervisor_error=%q mount_error=%q", daemon.DaemonRunning, supervisor.DaemonRunning, daemon.MountHealthy, daemon.DaemonError, supervisor.DaemonError, daemon.MountError) + case <-ticker.C: + } + } +} + +func systemdServiceUnit(definitionPath string, label string) (string, error) { + unit, err := service.SystemdUnitName(label) + if err != nil { + return "", err + } + if filepath.Base(definitionPath) != unit { + return "", fmt.Errorf("systemd definition filename must be %s", unit) + } + return unit, nil +} + +func newFSServiceUpdatePreflightCommand() *cobra.Command { + var codexHome, storeDir string + var compatibility compatibilityFlags + var automatic, promote, applyQuarantine, jsonOutput bool + command := &cobra.Command{ + Use: "update-preflight", + Short: "Gate service updates and optionally route unknown-version sessions to current native bytes", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + store := resolveFoldStore(home, storeDir) + doctorErr := requireStorageHealth(command.Context(), store) + compatibilityResult, err := evaluateCompatibility(command.Context(), store, compatibility) + if err != nil { + return err + } + fallbackReady, err := managedRoutesMatchCurrentBytes(command.Context(), home, store) + if err != nil { + fallbackReady = false + } + decision := service.EvaluateUpdate(service.UpdateInput{Capability: verifiedCapability(), DoctorHealthy: doctorErr == nil, Compatibility: compatibilityResult.Evaluation, NativeFallbackReady: fallbackReady, Automatic: automatic, ExplicitPromotion: promote}) + result := FSUpdatePreflightResult{DoctorHealthy: doctorErr == nil, Compatibility: compatibilityResult, Decision: decision} + if decision.Quarantine && decision.RequiresNativeFallback && applyQuarantine { + count, err := quarantineManagedRoutes(command.Context(), home, store) + if err != nil { + return err + } + result.QuarantinedSessions = count + result.Decision = service.EvaluateUpdate(service.UpdateInput{Capability: verifiedCapability(), DoctorHealthy: doctorErr == nil, Compatibility: compatibilityResult.Evaluation, NativeFallbackReady: true, Automatic: automatic, ExplicitPromotion: promote}) + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "allowed=%t quarantine=%t requires_fallback=%t doctor=%t quarantined=%d reason=%q\n", result.Decision.Allowed, result.Decision.Quarantine, result.Decision.RequiresNativeFallback, result.DoctorHealthy, result.QuarantinedSessions, result.Decision.Reason) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + addCompatibilityFlags(command, &compatibility) + command.Flags().BoolVar(&automatic, "automatic", false, "Evaluate an unattended update") + command.Flags().BoolVar(&promote, "promote", false, "Explicitly approve preview or canary promotion") + command.Flags().BoolVar(&applyQuarantine, "apply-quarantine", false, "Route managed sessions to verified current native bytes when clients are unknown") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func managedRoutesMatchCurrentBytes(ctx context.Context, home string, store string) (bool, error) { + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return false, err + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return false, err + } + byID := make(map[string]codex.Session, len(sessions)) + for _, session := range sessions { + byID[session.ID] = session + } + for _, state := range states { + current, ok := byID[state.SessionID] + if !ok { + return false, fmt.Errorf("Codex route missing for managed session %s", state.SessionID) + } + if !isGeneratedNativeFallbackPath(current.RolloutPath, store, state.SessionID) { + return false, nil + } + if _, err := hashPath(current.RolloutPath); err != nil { + return false, err + } + } + return true, nil +} + +func quarantineManagedRoutes(ctx context.Context, home string, store string) (int, error) { + states, err := vfs.DiscoverSessionStates(store) + if err != nil { + return 0, err + } + sessions, err := codex.LoadSessions(home) + if err != nil { + return 0, err + } + byID := make(map[string]codex.Session, len(sessions)) + for _, session := range sessions { + byID[session.ID] = session + } + count := 0 + for _, state := range states { + current, ok := byID[state.SessionID] + if !ok { + return count, fmt.Errorf("Codex route missing for managed session %s", state.SessionID) + } + if isGeneratedNativeFallbackPath(current.RolloutPath, store, state.SessionID) { + if _, err := hashPath(current.RolloutPath); err != nil { + return count, err + } + continue + } + managed, resolver, err := openManagedSession(ctx, store, state) + if err != nil { + return count, err + } + targetDirectory := filepath.Join(store, "fs", "fallbacks", state.SessionID) + if err := os.MkdirAll(targetDirectory, 0o700); err != nil { + return count, err + } + if err := os.Chmod(targetDirectory, 0o700); err != nil { + return count, err + } + targetPath := filepath.Join(targetDirectory, "quarantine-current.jsonl") + target, err := managed.MaterializeCurrent(ctx, targetPath, true) + _ = resolver.Close() + if err != nil { + return count, err + } + if filepath.Clean(current.RolloutPath) == filepath.Clean(target.Path) { + continue + } + if _, err := codex.RouteSession(ctx, codex.RouteOptions{CodexHome: home, SessionID: state.SessionID, ExpectedPath: current.RolloutPath, Target: codex.RouteTarget{Path: target.Path, Bytes: target.Bytes, SHA256: target.SHA256}}); err != nil { + return count, err + } + if _, err := retireManagedState(store, state.SessionID); err != nil { + return count, err + } + count++ + } + return count, nil +} + +func isGeneratedNativeFallbackPath(path string, store string, sessionID string) bool { + if path == "" || store == "" || sessionID == "" { + return false + } + directory := filepath.Clean(filepath.Dir(path)) + legacyDirectory := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + fallbackDirectory := filepath.Join(filepath.Clean(store), "fs", "fallbacks", sessionID) + if directory != legacyDirectory && directory != fallbackDirectory { + return false + } + switch filepath.Base(path) { + case "fallback-current.jsonl", "quarantine-current.jsonl": + return true + default: + return false + } +} + +func retireManagedState(store string, sessionID string) (string, error) { + if store == "" || sessionID == "" { + return "", errors.New("store and session ID are required") + } + source := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + retiredRoot := filepath.Join(filepath.Clean(store), "fs", "retired") + if err := os.MkdirAll(retiredRoot, 0o700); err != nil { + return "", err + } + target := filepath.Join(retiredRoot, fmt.Sprintf("%s-%d", sessionID, time.Now().UnixNano())) + if err := os.Rename(source, target); err != nil { + return "", err + } + return target, nil +} + +func restoreManagedState(store string, sessionID string, retiredPath string) error { + if store == "" || sessionID == "" || retiredPath == "" { + return errors.New("store, session ID, and retired state path are required") + } + target := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + if err := os.Rename(filepath.Clean(retiredPath), target); err != nil { + return err + } + if _, err := vfs.RepublishSessionState(filepath.Join(target, "state.json")); err != nil { + return fmt.Errorf("republish restored managed state: %w", err) + } + return nil +} + +func retainCanonicalSnapshot(ctx context.Context, store string, sessionID string, source vfs.NativeFile, budget storage.Checker) (vfs.NativeFile, error) { + if store == "" || !validSessionID(sessionID) || source.Path == "" { + return vfs.NativeFile{}, errors.New("store, session ID, and source snapshot are required") + } + sourcePath := filepath.Clean(source.Path) + verified, err := hashPath(sourcePath) + if err != nil { + return vfs.NativeFile{}, fmt.Errorf("verify canonical native snapshot: %w", err) + } + if verified.Bytes != source.Bytes || verified.SHA256 != source.SHA256 { + return vfs.NativeFile{}, errors.New("canonical native snapshot changed during migration") + } + if budget == nil { + guard, err := storage.DefaultGuard(store) + if err != nil { + return vfs.NativeFile{}, err + } + budget = guard + } + if _, err := budget.Check(ctx, storage.Projection{Operation: "retain-migration-snapshot", AdditionalPersistentBytes: source.Bytes}); err != nil { + return vfs.NativeFile{}, err + } + retainedDir := filepath.Join(filepath.Clean(store), "fs", "snapshots", sessionID) + retainedPath := filepath.Join(retainedDir, "native.jsonl") + if err := os.MkdirAll(retainedDir, 0o700); err != nil { + return vfs.NativeFile{}, err + } + if _, err := os.Lstat(retainedPath); err == nil { + return vfs.NativeFile{}, errors.New("retained canonical snapshot already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return vfs.NativeFile{}, err + } + if err := os.Link(sourcePath, retainedPath); err != nil { + if !errors.Is(err, syscall.EXDEV) { + return vfs.NativeFile{}, fmt.Errorf("stage canonical native snapshot: %w", err) + } + if err := copyCanonicalSnapshot(sourcePath, retainedPath); err != nil { + return vfs.NativeFile{}, fmt.Errorf("copy canonical native snapshot: %w", err) + } + } + retained, err := hashPath(retainedPath) + if err == nil && (retained.Bytes != source.Bytes || retained.SHA256 != source.SHA256) { + err = errors.New("retained canonical snapshot does not match source") + } + if err != nil { + _ = os.Remove(retainedPath) + return vfs.NativeFile{}, err + } + retained.Path = retainedPath + return retained, nil +} + +func finalizeCanonicalSnapshotSource(sourcePath string, retained vfs.NativeFile) error { + sourcePath = filepath.Clean(sourcePath) + retained.Path = filepath.Clean(retained.Path) + source, err := hashPath(sourcePath) + if err != nil { + return fmt.Errorf("verify canonical source before cutover: %w", err) + } + hidden, err := hashPath(retained.Path) + if err != nil { + return fmt.Errorf("verify retained canonical snapshot before cutover: %w", err) + } + if source.Bytes != retained.Bytes || source.SHA256 != retained.SHA256 || hidden.Bytes != retained.Bytes || hidden.SHA256 != retained.SHA256 { + return errors.New("canonical source changed before cutover") + } + if err := os.Remove(sourcePath); err != nil { + return fmt.Errorf("hide canonical source after mount acknowledgement: %w", err) + } + return nil +} + +func copyCanonicalSnapshot(sourcePath string, retainedPath string) error { + source, err := os.Open(sourcePath) + if err != nil { + return err + } + defer source.Close() + target, err := os.OpenFile(retainedPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + if _, err := io.Copy(target, source); err != nil { + _ = target.Close() + _ = os.Remove(retainedPath) + return err + } + if err := target.Sync(); err != nil { + _ = target.Close() + _ = os.Remove(retainedPath) + return err + } + return target.Close() +} + +func restoreCanonicalSnapshotSource(originalPath string, retainedPath string) error { + originalPath = filepath.Clean(originalPath) + retainedPath = filepath.Clean(retainedPath) + if originalPath == "" || retainedPath == "" { + return errors.New("original and retained snapshot paths are required") + } + if _, err := os.Lstat(originalPath); err == nil { + original, originalErr := hashPath(originalPath) + retained, retainedErr := hashPath(retainedPath) + if originalErr != nil || retainedErr != nil || original.Bytes != retained.Bytes || original.SHA256 != retained.SHA256 { + return errors.New("cannot discard retained snapshot while canonical source differs") + } + if err := os.Remove(retainedPath); err != nil { + return err + } + _ = os.Remove(filepath.Dir(retainedPath)) + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := os.MkdirAll(filepath.Dir(originalPath), 0o700); err != nil { + return err + } + if err := os.Rename(retainedPath, originalPath); err != nil { + return fmt.Errorf("restore canonical native snapshot: %w", err) + } + return nil +} + +type mountAcknowledgement struct { + Generation uint64 `json:"generation"` + Route string `json:"route"` +} + +const ( + retirementRequestFilename = "retire.request.json" + retirementAcknowledgementFilename = "retire.ack.json" +) + +type retirementControl struct { + Token string `json:"token"` + Generation uint64 `json:"generation"` + Route string `json:"route"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + Error string `json:"error,omitempty"` +} + +func createRetirementRequest(store string, sessionID string, generation uint64, route string, target vfs.NativeFile) (retirementControl, error) { + if store == "" || !validSessionID(sessionID) || generation == 0 || route == "" || target.Bytes < 0 || len(target.SHA256) != 64 { + return retirementControl{}, errors.New("complete retirement request metadata is required") + } + directory := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + if pending, exists, err := readRetirementRequest(store, sessionID); err != nil { + return retirementControl{}, err + } else if exists { + if pending.Generation != generation || pending.Route != route || pending.Bytes != target.Bytes || pending.SHA256 != target.SHA256 { + return retirementControl{}, errors.New("pending session retirement does not match the requested generation and target") + } + return pending, nil + } + if err := removeIfExists(filepath.Join(directory, retirementAcknowledgementFilename)); err != nil { + return retirementControl{}, err + } + tokenBytes := make([]byte, 16) + if _, err := rand.Read(tokenBytes); err != nil { + return retirementControl{}, err + } + request := retirementControl{Token: hex.EncodeToString(tokenBytes), Generation: generation, Route: route, Bytes: target.Bytes, SHA256: target.SHA256} + if err := writeSessionControlFile(directory, retirementRequestFilename, request); err != nil { + return retirementControl{}, err + } + return request, nil +} + +func readRetirementRequest(store string, sessionID string) (retirementControl, bool, error) { + path := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID, retirementRequestFilename) + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return retirementControl{}, false, nil + } + if err != nil { + return retirementControl{}, false, err + } + var request retirementControl + if err := json.Unmarshal(data, &request); err != nil { + return retirementControl{}, false, fmt.Errorf("decode retirement request: %w", err) + } + if len(request.Token) != 32 || request.Generation == 0 || request.Route == "" || request.Bytes < 0 || len(request.SHA256) != 64 || request.Error != "" { + return retirementControl{}, false, errors.New("invalid retirement request") + } + return request, true, nil +} + +func writeRetirementAcknowledgement(store string, sessionID string, acknowledgement retirementControl) error { + if len(acknowledgement.Token) != 32 || acknowledgement.Generation == 0 || acknowledgement.Route == "" || acknowledgement.Bytes < 0 || len(acknowledgement.SHA256) != 64 { + return errors.New("complete retirement acknowledgement metadata is required") + } + directory := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + return writeSessionControlFile(directory, retirementAcknowledgementFilename, acknowledgement) +} + +func waitForRetirementAcknowledgement(ctx context.Context, store string, sessionID string, request retirementControl, timeout time.Duration) error { + if timeout <= 0 { + timeout = 15 * time.Second + } + path := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID, retirementAcknowledgementFilename) + deadline := time.Now().Add(timeout) + for { + data, err := os.ReadFile(path) + if err == nil { + var acknowledgement retirementControl + if json.Unmarshal(data, &acknowledgement) == nil && + acknowledgement.Token == request.Token && acknowledgement.Generation == request.Generation && + acknowledgement.Route == request.Route && acknowledgement.Bytes == request.Bytes && acknowledgement.SHA256 == request.SHA256 { + if acknowledgement.Error != "" { + return fmt.Errorf("retirement rejected: %s", acknowledgement.Error) + } + return nil + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if time.Now().After(deadline) { + return errors.New("timed out waiting for retirement acknowledgement") + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(25 * time.Millisecond): + } + } +} + +func clearRetirementControl(directory string) error { + var result error + for _, name := range []string{retirementRequestFilename, retirementAcknowledgementFilename} { + if err := removeIfExists(filepath.Join(filepath.Clean(directory), name)); err != nil { + result = errors.Join(result, err) + } + } + return result +} + +func removeIfExists(path string) error { + if err := os.Remove(filepath.Clean(path)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} + +func writeMountAcknowledgement(store string, sessionID string, generation uint64, route string) error { + if store == "" || !validSessionID(sessionID) || generation == 0 || route == "" { + return errors.New("complete mount acknowledgement metadata is required") + } + directory := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID) + return writeSessionControlFile(directory, "mounted.json", mountAcknowledgement{Generation: generation, Route: route}) +} + +func writeSessionControlFile(directory string, name string, value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + temporary, err := os.CreateTemp(directory, ".mounted-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return err + } + if _, err := temporary.Write(append(data, '\n')); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + return os.Rename(temporaryPath, filepath.Join(directory, name)) +} + +func waitForMountAcknowledgement(ctx context.Context, store string, sessionID string, generation uint64, route string, timeout time.Duration) error { + if timeout <= 0 { + timeout = 15 * time.Second + } + path := filepath.Join(filepath.Clean(store), "fs", "sessions", sessionID, "mounted.json") + deadline := time.Now().Add(timeout) + for { + data, err := os.ReadFile(path) + if err == nil { + var acknowledgement mountAcknowledgement + if json.Unmarshal(data, &acknowledgement) == nil && acknowledgement.Generation == generation && acknowledgement.Route == route { + return nil + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if time.Now().After(deadline) { + return errors.New("timed out waiting for the filesystem daemon") + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(25 * time.Millisecond): + } + } +} + +func retireCanonicalNativeSnapshot(store string, nativeRoot string, sessionID string, snapshotPath string, currentPath string, retiredState string) (string, error) { + snapshotPath = filepath.Clean(snapshotPath) + if snapshotPath == filepath.Clean(currentPath) { + return "", nil + } + var relative string + legacyRelative, legacyErr := relativeWithin(filepath.Clean(nativeRoot), snapshotPath) + hiddenRoot := filepath.Join(filepath.Clean(store), "fs", "snapshots", sessionID) + _, hiddenErr := relativeWithin(hiddenRoot, snapshotPath) + switch { + case legacyErr == nil: + relative = legacyRelative + case hiddenErr == nil && filepath.Base(snapshotPath) == "native.jsonl": + relative = filepath.Join("store-snapshot", "native.jsonl") + default: + return "", errors.New("canonical native snapshot is outside the retained snapshot roots") + } + target := filepath.Join(filepath.Clean(retiredState), "retained-native", relative) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return "", err + } + if err := os.Rename(snapshotPath, target); err != nil { + return "", err + } + oldSidecar := filepath.Join(filepath.Dir(snapshotPath), "._"+filepath.Base(snapshotPath)) + newSidecar := filepath.Join(filepath.Dir(target), "._"+filepath.Base(target)) + if _, err := os.Lstat(oldSidecar); err == nil { + if err := os.Rename(oldSidecar, newSidecar); err != nil { + _ = os.Rename(target, snapshotPath) + return "", err + } + } else if !errors.Is(err, os.ErrNotExist) { + _ = os.Rename(target, snapshotPath) + return "", err + } + if hiddenErr == nil { + _ = os.Remove(hiddenRoot) + } + return target, nil +} + +func validSessionID(sessionID string) bool { + return sessionID != "" && sessionID != "." && sessionID != ".." && !strings.ContainsAny(sessionID, "/\\\x00") +} + +func relativeWithin(root string, target string) (string, error) { + root = filepath.Clean(root) + target = filepath.Clean(target) + relative, err := filepath.Rel(root, target) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", errors.New("path is outside root") + } + return relative, nil +} + +func restoreCanonicalNativeSnapshot(snapshotPath string, retiredSnapshot string) error { + if retiredSnapshot == "" { + return nil + } + if err := os.MkdirAll(filepath.Dir(snapshotPath), 0o700); err != nil { + return err + } + if err := os.Rename(retiredSnapshot, snapshotPath); err != nil { + return err + } + retiredSidecar := filepath.Join(filepath.Dir(retiredSnapshot), "._"+filepath.Base(retiredSnapshot)) + originalSidecar := filepath.Join(filepath.Dir(snapshotPath), "._"+filepath.Base(snapshotPath)) + if _, err := os.Lstat(retiredSidecar); err == nil { + return os.Rename(retiredSidecar, originalSidecar) + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} + +func canonicalMountRoute(home string, mount string, route string) (string, error) { + relative, err := canonicalRelativeRoute(home, route) + if err != nil { + return "", err + } + return filepath.Join(filepath.Clean(mount), relative), nil +} + +func canonicalNativeRoute(home string, nativeRoot string, route string) (string, error) { + relative, err := canonicalRelativeRoute(home, route) + if err != nil { + return "", err + } + if !filepath.IsAbs(nativeRoot) { + return "", errors.New("canonical native root must be absolute") + } + return filepath.Join(filepath.Clean(nativeRoot), relative), nil +} + +func canonicalNamespaceRoute(home string, mount string, route string) (string, error) { + relative, homeErr := canonicalRelativeRoute(home, route) + if homeErr != nil { + var mountErr error + relative, mountErr = canonicalRelativeRoute(mount, route) + if mountErr != nil { + return "", homeErr + } + } + return "/" + filepath.ToSlash(relative), nil +} + +func canonicalSessionRoutes(home string, mount string, store string, states []vfs.SessionState, sessions []codex.Session) (map[string]string, error) { + managed := make(map[string]struct{}, len(states)) + for _, state := range states { + managed[state.SessionID] = struct{}{} + } + routes := make(map[string]string, len(states)) + for _, session := range sessions { + if _, exists := managed[session.ID]; !exists { + continue + } + if isGeneratedNativeFallbackPath(session.RolloutPath, store, session.ID) { + continue + } + route, err := canonicalNamespaceRoute(home, mount, session.RolloutPath) + if err != nil { + return nil, err + } + routes[session.ID] = route + } + return routes, nil +} + +func discoverCanonicalRoutes(home string, mount string, store string, states []vfs.SessionState, load func(string) ([]codex.Session, error)) (map[string]string, error) { + if len(states) == 0 { + return map[string]string{}, nil + } + sessions, err := load(home) + if err != nil { + return nil, err + } + return canonicalSessionRoutes(home, mount, store, states, sessions) +} + +func canonicalRelativeRoute(home string, route string) (string, error) { + if !filepath.IsAbs(home) || !filepath.IsAbs(route) { + return "", errors.New("canonical Codex and route paths must be absolute") + } + home = filepath.Clean(home) + relative, err := filepath.Rel(home, filepath.Clean(route)) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", errors.New("Codex route is outside its home directory") + } + first, remainder := relative, "" + if separator := strings.IndexByte(relative, byte(filepath.Separator)); separator >= 0 { + first, remainder = relative[:separator], relative[separator+1:] + } + if (first != "sessions" && first != "archived_sessions") || remainder == "" || !strings.HasSuffix(remainder, ".jsonl") { + return "", errors.New("Codex route is not inside sessions or archived_sessions") + } + return relative, nil +} + +func addServicePathFlags(command *cobra.Command, codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir *string) { + command.Flags().StringVar(codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().StringVar(mountPoint, "mount", "", "Mounted CodexFold filesystem path; defaults to /fold-fs") + command.Flags().StringVar(binaryPath, "binary", "", "Absolute CodexFold binary path; defaults to the current executable") + addServiceDefinitionFlags(command, plistPath) + command.Flags().StringVar(logDir, "log-dir", "", "Service log directory; defaults to /service/logs") +} + +func addServiceDefinitionFlags(command *cobra.Command, definitionPath *string) { + command.Flags().StringVar(definitionPath, "definition", "", "Native service definition path") + command.Flags().StringVar(definitionPath, "plist", "", "LaunchAgent plist path (macOS compatibility alias)") +} + +func resolveServicePaths(codexHome, storeDir, mountPoint, binaryPath, plistPath, logDir, label string) (string, string, string, string, string, string, error) { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return "", "", "", "", "", "", err + } + store := resolveFoldStore(home, storeDir) + mount := defaultMountPoint(home, mountPoint) + binary := binaryPath + if binary == "" { + binary, err = os.Executable() + if err != nil { + return "", "", "", "", "", "", err + } + } + binary, err = filepath.Abs(binary) + if err != nil { + return "", "", "", "", "", "", err + } + plist, err := resolveServiceDefinitionPathForLabel(plistPath, label) + if err != nil { + return "", "", "", "", "", "", err + } + logs := logDir + if logs == "" { + logs = filepath.Join(store, "service", "logs") + } + logs, err = filepath.Abs(logs) + if err != nil { + return "", "", "", "", "", "", err + } + return home, store, mount, binary, plist, logs, nil +} + +func resolvePlistPath(explicit string) (string, error) { + if explicit != "" { + return filepath.Abs(explicit) + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, "Library", "LaunchAgents", serviceLabel+".plist"), nil +} + +func nativeFSKitSupervisorDefinitionPath(definitionPath string) string { + definitionPath = filepath.Clean(definitionPath) + extension := filepath.Ext(definitionPath) + base := strings.TrimSuffix(filepath.Base(definitionPath), extension) + if extension == "" { + extension = ".plist" + } + return filepath.Join(filepath.Dir(definitionPath), base+".supervisor"+extension) +} + +func nativeFSKitSupervisorLabel(label string) string { + return label + ".supervisor" +} + +func resolveServiceDefinitionPath(explicit string) (string, error) { + if explicit != "" { + return filepath.Abs(explicit) + } + platform, err := service.CurrentPlatform() + if err != nil { + return "", err + } + switch platform { + case service.PlatformLaunchd: + return resolvePlistPath("") + case service.PlatformSystemd: + configHome := os.Getenv("XDG_CONFIG_HOME") + if !filepath.IsAbs(configHome) { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + configHome = filepath.Join(home, ".config") + } + unit, err := service.SystemdUnitName(serviceLabel) + if err != nil { + return "", err + } + return filepath.Join(configHome, "systemd", "user", unit), nil + case service.PlatformWindows: + programData := os.Getenv("ProgramData") + if programData == "" { + return "", errors.New("ProgramData is not set") + } + return filepath.Join(programData, "CodexFold", "service.json"), nil + default: + return "", errors.New("unknown service platform") + } +} + +func resolveServiceDefinitionPathForLabel(explicit, label string) (string, error) { + if explicit != "" || label == "" || label == serviceLabel { + return resolveServiceDefinitionPath(explicit) + } + for index, character := range label { + if (character >= 'a' && character <= 'z') || + (character >= 'A' && character <= 'Z') || + (character >= '0' && character <= '9') || character == '.' || character == '-' { + if index == 0 && character == '.' { + return "", errors.New("custom service label is not safe for a LaunchAgent path") + } + continue + } + return "", errors.New("custom service label is not safe for a LaunchAgent path") + } + if strings.HasSuffix(label, ".") { + return "", errors.New("custom service label is not safe for a LaunchAgent path") + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, "Library", "LaunchAgents", label+".plist"), nil +} diff --git a/internal/cli/fs_service_fskit.go b/internal/cli/fs_service_fskit.go new file mode 100644 index 0000000..3acc8be --- /dev/null +++ b/internal/cli/fs_service_fskit.go @@ -0,0 +1,10 @@ +package cli + +import "context" + +type fsKitAppTransaction interface { + AppGroupPath() string + Changed() bool + Rollback(context.Context) error + Commit() error +} diff --git a/internal/cli/fs_service_fskit_darwin.go b/internal/cli/fs_service_fskit_darwin.go new file mode 100644 index 0000000..f6860f2 --- /dev/null +++ b/internal/cli/fs_service_fskit_darwin.go @@ -0,0 +1,803 @@ +//go:build darwin + +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/samekind/codexfold/internal/service" + "golang.org/x/sys/unix" +) + +const launchServicesRegister = "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister" + +var runFSKitLaunchServicesCommand = func(ctx context.Context, args ...string) ([]byte, error) { + return exec.CommandContext(ctx, launchServicesRegister, args...).CombinedOutput() +} + +const ( + codexFoldFSKitModuleProcessName = "CodexFoldFSKitModule" + fsKitRegistrationWait = 15 * time.Second + fsKitModuleShutdownWait = 10 * time.Second +) + +type darwinFSKitAppTransaction struct { + target string + source string + stageRoot string + stagePath string + appGroupPath string + changed bool + hadTarget bool + contentsSwapped bool + appInstalled bool +} + +func prepareFSKitAppPlatform(ctx context.Context, source string, target string) (fsKitAppTransaction, error) { + if !filepath.IsAbs(target) { + return nil, errors.New("installed FSKit app path must be absolute") + } + target = filepath.Clean(target) + if source == "" { + appGroup, err := refreshFSKitApp(ctx, target) + if err != nil { + return nil, err + } + return &darwinFSKitAppTransaction{target: target, appGroupPath: appGroup}, nil + } + if !filepath.IsAbs(source) { + return nil, errors.New("FSKit app source path must be absolute") + } + source = filepath.Clean(source) + if source == target { + appGroup, err := refreshFSKitApp(ctx, target) + if err != nil { + return nil, err + } + return &darwinFSKitAppTransaction{target: target, appGroupPath: appGroup}, nil + } + if err := validateFSKitApp(ctx, source); err != nil { + return nil, fmt.Errorf("validate FSKit app source: %w", err) + } + sourceDigest, err := hashAppBundle(source) + if err != nil { + return nil, err + } + targetDigest, targetErr := hashAppBundle(target) + if targetErr == nil { + if targetDigest == sourceDigest { + if err := quiesceFSKitAppForUpdate(ctx); err != nil { + return nil, fmt.Errorf("quiesce existing FSKit app: %w", err) + } + appGroup, err := refreshFSKitApp(ctx, target) + if err != nil { + return nil, err + } + return &darwinFSKitAppTransaction{target: target, appGroupPath: appGroup}, nil + } + if err := requireNewerFSKitAppVersion(ctx, source, target); err != nil { + return nil, err + } + } else if !errors.Is(targetErr, os.ErrNotExist) { + return nil, targetErr + } + + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return nil, err + } + stageRoot, err := os.MkdirTemp(filepath.Dir(target), ".codexfold-fskit-stage-*") + if err != nil { + return nil, err + } + transaction := &darwinFSKitAppTransaction{target: target, source: source, stageRoot: stageRoot, changed: true} + stagePath := filepath.Join(stageRoot, filepath.Base(target)) + transaction.stagePath = stagePath + if output, err := exec.CommandContext(ctx, "/usr/bin/ditto", source, stagePath).CombinedOutput(); err != nil { + _ = transaction.Commit() + return nil, commandOutputError("stage FSKit app", output, err) + } + if err := validateFSKitApp(ctx, stagePath); err != nil { + _ = transaction.Commit() + return nil, fmt.Errorf("validate staged FSKit app: %w", err) + } + if stagedDigest, err := hashAppBundle(stagePath); err != nil || stagedDigest != sourceDigest { + _ = transaction.Commit() + if err != nil { + return nil, err + } + return nil, errors.New("staged FSKit app does not match the source bundle") + } + if targetErr == nil { + if err := quiesceFSKitAppForUpdate(ctx); err != nil { + _, restoreErr := refreshFSKitApp(ctx, target) + _ = transaction.Commit() + return nil, errors.Join(fmt.Errorf("quiesce existing FSKit app: %w", err), restoreErr) + } + } + if err := transaction.promoteStagedApp(); err != nil { + rollbackErr := transaction.Rollback(ctx) + return nil, errors.Join(err, rollbackErr) + } + installedDigest, err := hashAppBundle(target) + if err != nil { + rollbackErr := transaction.Rollback(ctx) + return nil, errors.Join(err, rollbackErr) + } + if installedDigest != sourceDigest { + rollbackErr := transaction.Rollback(ctx) + return nil, errors.Join(errors.New("installed FSKit app does not match the source bundle"), rollbackErr) + } + appGroup, err := refreshFSKitApp(ctx, target) + if err != nil { + rollbackErr := transaction.Rollback(ctx) + return nil, errors.Join(err, rollbackErr) + } + transaction.appGroupPath = appGroup + return transaction, nil +} + +func refreshFSKitApp(ctx context.Context, appPath string) (string, error) { + return validateAndEnableFSKitApp(ctx, appPath) +} + +func quiesceFSKitAppForUpdate(ctx context.Context) error { + return stopCodexFoldFSKitModuleProcesses(ctx) +} + +func unregisterStaleFSKitApps(ctx context.Context, appPath string) error { + targetModule, err := service.FSKitModulePath(appPath) + if err != nil { + return err + } + paths, err := registeredFSKitModulePaths(ctx) + if err != nil { + return err + } + for _, modulePath := range staleFSKitModulePaths(paths, targetModule) { + parentApp, ok := fsKitParentAppPath(modulePath) + if !ok { + return fmt.Errorf("FSKit module path has no parent app: %s", modulePath) + } + if err := unregisterFSKitAppRegistration(ctx, parentApp); err != nil { + return err + } + } + return nil +} + +// FSKit keeps a user-level activation state keyed by module identity. Removing +// a module through pluginkit can leave that state disabled until the next login. +// Update cleanup therefore removes only a temporary app registration, never the +// installed module itself. +func unregisterFSKitAppRegistration(ctx context.Context, appPath string) error { + output, err := runFSKitLaunchServicesCommand(ctx, "-u", appPath) + if err != nil { + return commandOutputError("unregister stale FSKit app registration", output, err) + } + return nil +} + +func registeredFSKitModulePaths(ctx context.Context) ([]string, error) { + output, err := exec.CommandContext( + ctx, + "/usr/bin/pluginkit", + "-m", "-A", "-D", "-v", "-i", service.FSKitModuleIdentifier, + ).CombinedOutput() + if err != nil { + return nil, commandOutputError("list FSKit module registrations", output, err) + } + return parseFSKitModulePaths(output), nil +} + +func parseFSKitModulePaths(output []byte) []string { + seen := make(map[string]struct{}) + var paths []string + for _, line := range strings.Split(string(output), "\n") { + separator := strings.LastIndexByte(line, '\t') + if separator < 0 { + continue + } + candidate := filepath.Clean(strings.TrimSpace(line[separator+1:])) + if !filepath.IsAbs(candidate) || filepath.Ext(candidate) != ".appex" { + continue + } + if _, exists := seen[candidate]; exists { + continue + } + seen[candidate] = struct{}{} + paths = append(paths, candidate) + } + return paths +} + +func normalizedFSKitModulePaths(paths []string) []string { + seen := make(map[string]struct{}, len(paths)) + result := make([]string, 0, len(paths)) + for _, path := range paths { + path = filepath.Clean(path) + if path == "." || path == "" { + continue + } + if _, exists := seen[path]; exists { + continue + } + seen[path] = struct{}{} + result = append(result, path) + } + sort.Strings(result) + return result +} + +func sameFSKitModulePaths(left []string, right []string) bool { + return strings.Join(normalizedFSKitModulePaths(left), "\x00") == strings.Join(normalizedFSKitModulePaths(right), "\x00") +} + +func staleFSKitModulePaths(paths []string, target string) []string { + target = filepath.Clean(target) + var stale []string + for _, path := range normalizedFSKitModulePaths(paths) { + if path != target { + stale = append(stale, path) + } + } + return stale +} + +func waitForFSKitModulePath(ctx context.Context, target string, timeout time.Duration) error { + target = filepath.Clean(target) + if timeout <= 0 { + timeout = fsKitRegistrationWait + } + deadline := time.NewTimer(timeout) + defer deadline.Stop() + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + var last []string + var lastErr error + for { + paths, err := registeredFSKitModulePaths(ctx) + if err == nil { + last = paths + lastErr = nil + for _, path := range paths { + if filepath.Clean(path) == target { + return nil + } + } + } else { + lastErr = err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline.C: + if lastErr != nil { + return fmt.Errorf("FSKit module registration query failed: %w", lastErr) + } + return fmt.Errorf("FSKit module registration did not include %s: got %v", target, normalizedFSKitModulePaths(last)) + case <-ticker.C: + } + } +} + +func waitForFSKitModulePaths(ctx context.Context, want []string, timeout time.Duration) error { + if timeout <= 0 { + timeout = fsKitRegistrationWait + } + deadline := time.NewTimer(timeout) + defer deadline.Stop() + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + var last []string + var lastErr error + for { + paths, err := registeredFSKitModulePaths(ctx) + if err == nil { + last = paths + lastErr = nil + if sameFSKitModulePaths(paths, want) { + return nil + } + } else { + lastErr = err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline.C: + if lastErr != nil { + return fmt.Errorf("FSKit module registration query failed: %w", lastErr) + } + return fmt.Errorf("FSKit module registration did not converge: got %v, want %v", normalizedFSKitModulePaths(last), normalizedFSKitModulePaths(want)) + case <-ticker.C: + } + } +} + +func fsKitParentAppPath(modulePath string) (string, bool) { + extensions := filepath.Dir(filepath.Clean(modulePath)) + if filepath.Base(extensions) != "Extensions" { + return "", false + } + contents := filepath.Dir(extensions) + if filepath.Base(contents) != "Contents" { + return "", false + } + app := filepath.Dir(contents) + if filepath.Ext(app) != ".app" { + return "", false + } + return app, true +} + +func (t *darwinFSKitAppTransaction) AppGroupPath() string { return t.appGroupPath } +func (t *darwinFSKitAppTransaction) Changed() bool { return t.changed } + +// promoteStagedApp preserves an existing app bundle root because macOS can +// attach launch authorization to that directory's inode. Swapping Contents is +// atomic on APFS and keeps the previous version in stagePath for rollback. +func (t *darwinFSKitAppTransaction) promoteStagedApp() error { + if t == nil || t.target == "" || t.stagePath == "" { + return errors.New("FSKit app transaction is incomplete") + } + if info, err := os.Stat(t.target); err == nil { + if !info.IsDir() { + return errors.New("installed FSKit app path is not a directory") + } + t.hadTarget = true + targetContents := filepath.Join(t.target, "Contents") + stagedContents := filepath.Join(t.stagePath, "Contents") + if info, err := os.Stat(targetContents); err != nil { + return fmt.Errorf("inspect installed FSKit app Contents: %w", err) + } else if !info.IsDir() { + return errors.New("installed FSKit app Contents is not a directory") + } + if info, err := os.Stat(stagedContents); err != nil { + return fmt.Errorf("inspect staged FSKit app Contents: %w", err) + } else if !info.IsDir() { + return errors.New("staged FSKit app Contents is not a directory") + } + if err := unix.RenamexNp(stagedContents, targetContents, unix.RENAME_SWAP); err != nil { + return fmt.Errorf("atomically replace FSKit app Contents: %w", err) + } + t.contentsSwapped = true + return syncDirectories(t.target, t.stagePath) + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if err := os.Rename(t.stagePath, t.target); err != nil { + return err + } + t.appInstalled = true + return syncDirectory(filepath.Dir(t.target)) +} + +func (t *darwinFSKitAppTransaction) rollbackStagedApp() error { + if t == nil { + return nil + } + if t.contentsSwapped { + targetContents := filepath.Join(t.target, "Contents") + stagedContents := filepath.Join(t.stagePath, "Contents") + if err := unix.RenamexNp(stagedContents, targetContents, unix.RENAME_SWAP); err != nil { + return fmt.Errorf("restore FSKit app Contents: %w", err) + } + t.contentsSwapped = false + return syncDirectories(t.target, t.stagePath) + } + if t.appInstalled { + if err := os.RemoveAll(t.target); err != nil { + return err + } + t.appInstalled = false + return syncDirectory(filepath.Dir(t.target)) + } + return nil +} + +func (t *darwinFSKitAppTransaction) Rollback(ctx context.Context) error { + if t == nil { + return nil + } + if !t.changed { + _, err := refreshFSKitApp(ctx, t.target) + return err + } + var result error + if err := quiesceFSKitAppForUpdate(ctx); err != nil { + return err + } + if err := t.rollbackStagedApp(); err != nil { + // Preserve the staged previous app for an operator-visible recovery rather + // than deleting the only rollback material after a failed restore. + return errors.Join(result, err) + } + if t.hadTarget { + if _, err := refreshFSKitApp(ctx, t.target); err != nil { + result = errors.Join(result, err) + } + } + t.changed = false + return errors.Join(result, t.Commit()) +} + +func (t *darwinFSKitAppTransaction) Commit() error { + if t == nil { + return nil + } + var result error + for _, path := range []string{t.stageRoot} { + if path == "" { + continue + } + if err := os.RemoveAll(path); err != nil && !errors.Is(err, os.ErrNotExist) { + result = errors.Join(result, err) + } + } + t.stageRoot = "" + t.stagePath = "" + return errors.Join(result, syncDirectory(filepath.Dir(t.target))) +} + +func validateAndEnableFSKitApp(ctx context.Context, appPath string) (string, error) { + if err := validateFSKitApp(ctx, appPath); err != nil { + return "", err + } + targetModule, err := service.FSKitModulePath(appPath) + if err != nil { + return "", err + } + if output, err := runFSKitLaunchServicesCommand(ctx, "-f", "-R", "-trusted", appPath); err != nil { + return "", commandOutputError("register FSKit app", output, err) + } + if err := waitForFSKitModulePath(ctx, targetModule, fsKitRegistrationWait); err != nil { + return "", err + } + if err := unregisterStaleFSKitApps(ctx, appPath); err != nil { + return "", err + } + if err := waitForFSKitModulePaths(ctx, []string{targetModule}, fsKitRegistrationWait); err != nil { + return "", fmt.Errorf("FSKit module registration is ambiguous: %w", err) + } + if _, err := ensureFSKitModuleEnabled(service.FSKitModuleIdentifier); err != nil { + return "", err + } + if output, err := exec.CommandContext(ctx, "/usr/bin/pluginkit", "-e", "use", "-p", "com.apple.fskit.fsmodule", "-i", service.FSKitModuleIdentifier).CombinedOutput(); err != nil { + return "", commandOutputError("enable FSKit extension election", output, err) + } + launcher, err := service.FSKitHostLauncherPath(appPath) + if err != nil { + return "", err + } + output, err := exec.CommandContext(ctx, launcher, "--app-group-path").CombinedOutput() + if err != nil { + return "", commandOutputError("resolve FSKit App Group path", output, err) + } + appGroup := filepath.Clean(strings.TrimSpace(string(output))) + if !filepath.IsAbs(appGroup) || filepath.Base(appGroup) != service.FSKitAppGroupIdentifier { + return "", fmt.Errorf("FSKit host returned invalid App Group path %q", appGroup) + } + return appGroup, nil +} + +func validateFSKitApp(ctx context.Context, appPath string) error { + launcher, err := service.FSKitHostLauncherPath(appPath) + if err != nil { + return err + } + module, err := service.FSKitModulePath(appPath) + if err != nil { + return err + } + if info, err := os.Stat(launcher); err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + if err != nil { + return err + } + return errors.New("FSKit host launcher is not executable") + } + if info, err := os.Stat(module); err != nil || !info.IsDir() { + if err != nil { + return err + } + return errors.New("FSKit module bundle is missing") + } + checks := []struct { + path string + key string + want string + }{ + {filepath.Join(appPath, "Contents", "Info.plist"), "CFBundleIdentifier", service.FSKitHostBundleIdentifier}, + {filepath.Join(module, "Contents", "Info.plist"), "CFBundleIdentifier", service.FSKitModuleIdentifier}, + } + for _, check := range checks { + output, err := exec.CommandContext(ctx, "/usr/bin/plutil", "-extract", check.key, "raw", check.path).CombinedOutput() + if err != nil { + return commandOutputError("read FSKit bundle identity", output, err) + } + if strings.TrimSpace(string(output)) != check.want { + return fmt.Errorf("FSKit bundle identifier=%q expected=%q", strings.TrimSpace(string(output)), check.want) + } + } + if output, err := exec.CommandContext(ctx, "/usr/bin/codesign", "--verify", "--deep", "--strict", "--verbose=2", appPath).CombinedOutput(); err != nil { + return commandOutputError("verify FSKit app signature", output, err) + } + entitlements, err := exec.CommandContext(ctx, "/usr/bin/codesign", "-d", "--entitlements", ":-", "--xml", module).CombinedOutput() + if err != nil { + return commandOutputError("read FSKit module entitlements", entitlements, err) + } + for _, required := range []string{"com.apple.developer.fskit.fsmodule", "com.apple.security.app-sandbox", "com.apple.security.application-groups", service.FSKitAppGroupIdentifier} { + if !strings.Contains(string(entitlements), required) { + return fmt.Errorf("FSKit module signature is missing %s", required) + } + } + profile := filepath.Join(module, "Contents", "embedded.provisionprofile") + profileData, err := exec.CommandContext(ctx, "/usr/bin/security", "cms", "-D", "-i", profile).CombinedOutput() + if err != nil { + return commandOutputError("read FSKit module provisioning profile", profileData, err) + } + if !strings.Contains(string(profileData), service.FSKitAppGroupIdentifier) { + return errors.New("FSKit module provisioning profile does not authorize the App Group") + } + return nil +} + +func requireNewerFSKitAppVersion(ctx context.Context, source string, target string) error { + sourceVersion, err := readFSKitAppVersion(ctx, source) + if err != nil { + return err + } + targetVersion, err := readFSKitAppVersion(ctx, target) + if err != nil { + return err + } + comparison, err := compareFSKitBundleVersions(sourceVersion, targetVersion) + if err != nil { + return err + } + if comparison <= 0 { + return fmt.Errorf("FSKit app candidate CFBundleVersion %s must exceed installed version %s when bundle contents differ", sourceVersion, targetVersion) + } + return nil +} + +func readFSKitAppVersion(ctx context.Context, appPath string) (string, error) { + infoPath := filepath.Join(appPath, "Contents", "Info.plist") + output, err := exec.CommandContext(ctx, "/usr/bin/plutil", "-extract", "CFBundleVersion", "raw", infoPath).CombinedOutput() + if err != nil { + return "", commandOutputError("read FSKit app version", output, err) + } + version := strings.TrimSpace(string(output)) + if _, err := parseFSKitBundleVersion(version); err != nil { + return "", err + } + return version, nil +} + +func compareFSKitBundleVersions(left string, right string) (int, error) { + leftParts, err := parseFSKitBundleVersion(left) + if err != nil { + return 0, err + } + rightParts, err := parseFSKitBundleVersion(right) + if err != nil { + return 0, err + } + for index := 0; index < max(len(leftParts), len(rightParts)); index++ { + var leftPart, rightPart uint64 + if index < len(leftParts) { + leftPart = leftParts[index] + } + if index < len(rightParts) { + rightPart = rightParts[index] + } + if leftPart < rightPart { + return -1, nil + } + if leftPart > rightPart { + return 1, nil + } + } + return 0, nil +} + +func parseFSKitBundleVersion(version string) ([]uint64, error) { + parts := strings.Split(strings.TrimSpace(version), ".") + if len(parts) == 0 || len(parts) > 3 { + return nil, fmt.Errorf("invalid FSKit CFBundleVersion %q", version) + } + parsed := make([]uint64, len(parts)) + for index, part := range parts { + if part == "" { + return nil, fmt.Errorf("invalid FSKit CFBundleVersion %q", version) + } + value, err := strconv.ParseUint(part, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid FSKit CFBundleVersion %q: %w", version, err) + } + parsed[index] = value + } + return parsed, nil +} + +func ensureFSKitModuleEnabled(moduleID string) (bool, error) { + home, err := os.UserHomeDir() + if err != nil { + return false, err + } + path := filepath.Join(home, "Library", "Group Containers", "group.com.apple.fskit.settings", "enabledModules.plist") + output, err := exec.Command("/usr/bin/plutil", "-convert", "json", "-o", "-", path).CombinedOutput() + if err != nil { + return false, commandOutputError("read enabled FSKit modules", output, err) + } + var modules []string + if err := json.Unmarshal(output, &modules); err != nil { + return false, err + } + for _, current := range modules { + if current == moduleID { + return false, nil + } + } + command := fmt.Sprintf("Add :%d string %s", len(modules), moduleID) + if output, err := exec.Command("/usr/libexec/PlistBuddy", "-c", command, path).CombinedOutput(); err != nil { + return false, commandOutputError("enable FSKit module", output, err) + } + if output, err := exec.Command("/usr/bin/plutil", "-lint", path).CombinedOutput(); err != nil { + return false, commandOutputError("validate enabled FSKit modules", output, err) + } + return true, nil +} + +func userProcessIDs(ctx context.Context, name string) ([]int, error) { + output, err := exec.CommandContext(ctx, "/usr/bin/pgrep", "-u", strconv.Itoa(os.Getuid()), "-x", name).Output() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { + return nil, nil + } + return nil, err + } + var result []int + for _, field := range strings.Fields(string(output)) { + pid, parseErr := strconv.Atoi(field) + if parseErr == nil && pid > 1 { + result = append(result, pid) + } + } + return result, nil +} + +func stopCodexFoldFSKitModuleProcesses(ctx context.Context) error { + pids, err := userProcessIDs(ctx, codexFoldFSKitModuleProcessName) + if err != nil { + return fmt.Errorf("inspect CodexFold FSKit module processes: %w", err) + } + for _, pid := range pids { + if err := unix.Kill(pid, unix.SIGTERM); err != nil && !errors.Is(err, unix.ESRCH) { + return fmt.Errorf("stop CodexFold FSKit module process %d: %w", pid, err) + } + } + if err := waitForNoCodexFoldFSKitModuleProcesses(ctx, fsKitModuleShutdownWait); err != nil { + for _, pid := range pids { + if killErr := unix.Kill(pid, unix.SIGKILL); killErr != nil && !errors.Is(killErr, unix.ESRCH) { + return errors.Join(err, fmt.Errorf("force-stop CodexFold FSKit module process %d: %w", pid, killErr)) + } + } + return waitForNoCodexFoldFSKitModuleProcesses(ctx, fsKitModuleShutdownWait) + } + return nil +} + +func waitForNoCodexFoldFSKitModuleProcesses(ctx context.Context, timeout time.Duration) error { + if timeout <= 0 { + timeout = fsKitModuleShutdownWait + } + deadline := time.NewTimer(timeout) + defer deadline.Stop() + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + pids, err := userProcessIDs(ctx, codexFoldFSKitModuleProcessName) + if err != nil { + return fmt.Errorf("inspect CodexFold FSKit module processes: %w", err) + } + if len(pids) == 0 { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline.C: + return fmt.Errorf("CodexFold FSKit module processes remain: %v", pids) + case <-ticker.C: + } + } +} + +func hashAppBundle(root string) (string, error) { + if !filepath.IsAbs(root) { + return "", errors.New("app bundle path must be absolute") + } + hash := sha256.New() + err := filepath.WalkDir(filepath.Clean(root), func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + info, err := entry.Info() + if err != nil { + return err + } + _, _ = io.WriteString(hash, relative) + _, _ = io.WriteString(hash, "\x00"+info.Mode().String()+"\x00") + if entry.Type()&os.ModeSymlink != 0 { + target, err := os.Readlink(path) + if err != nil { + return err + } + _, _ = io.WriteString(hash, target) + return nil + } + if !entry.Type().IsRegular() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + _, copyErr := io.Copy(hash, file) + closeErr := file.Close() + return errors.Join(copyErr, closeErr) + }) + if err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func syncDirectory(path string) error { + directory, err := os.Open(filepath.Clean(path)) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} + +func syncDirectories(paths ...string) error { + seen := make(map[string]struct{}, len(paths)) + var result error + for _, path := range paths { + path = filepath.Clean(path) + if _, ok := seen[path]; ok { + continue + } + seen[path] = struct{}{} + result = errors.Join(result, syncDirectory(path)) + } + return result +} + +func commandOutputError(action string, output []byte, err error) error { + detail := strings.TrimSpace(string(output)) + if detail == "" { + return fmt.Errorf("%s: %w", action, err) + } + return fmt.Errorf("%s: %w: %s", action, err, detail) +} diff --git a/internal/cli/fs_service_fskit_darwin_test.go b/internal/cli/fs_service_fskit_darwin_test.go new file mode 100644 index 0000000..84874b6 --- /dev/null +++ b/internal/cli/fs_service_fskit_darwin_test.go @@ -0,0 +1,223 @@ +//go:build darwin + +package cli + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "testing" + + "golang.org/x/sys/unix" +) + +func TestCompareFSKitBundleVersions(t *testing.T) { + tests := []struct { + left string + right string + want int + }{ + {left: "2", right: "1", want: 1}, + {left: "1.0.1", right: "1", want: 1}, + {left: "1", right: "1.0", want: 0}, + {left: "1.9", right: "2", want: -1}, + } + for _, test := range tests { + got, err := compareFSKitBundleVersions(test.left, test.right) + if err != nil { + t.Fatalf("compare %q and %q: %v", test.left, test.right, err) + } + if got != test.want { + t.Fatalf("compare %q and %q = %d, want %d", test.left, test.right, got, test.want) + } + } +} + +func TestCompareFSKitBundleVersionsRejectsInvalidInput(t *testing.T) { + for _, version := range []string{"", "1.2.3.4", "1.beta"} { + if _, err := compareFSKitBundleVersions(version, "1"); err == nil { + t.Fatalf("invalid version %q was accepted", version) + } + } +} + +func TestParseFSKitModulePathsIncludesDuplicateRegistrations(t *testing.T) { + output := []byte(` ++ vip.jstar.codexfold.fskitprofileprobe.module(0.1.1) OLD 2026-07-22 08:23:53 +0000 /private/tmp/old/CodexFoldFSKit.app/Contents/Extensions/CodexFoldFSKitModule.appex ++ vip.jstar.codexfold.fskitprofileprobe.module(0.1.1) CURRENT 2026-07-22 09:24:23 +0000 /Users/test/Applications/CodexFoldFSKit.app/Contents/Extensions/CodexFoldFSKitModule.appex ++ vip.jstar.codexfold.fskitprofileprobe.module(0.1.1) DUPLICATE 2026-07-22 09:24:24 +0000 /private/tmp/old/CodexFoldFSKit.app/Contents/Extensions/CodexFoldFSKitModule.appex + (3 plug-ins) +`) + want := []string{ + "/private/tmp/old/CodexFoldFSKit.app/Contents/Extensions/CodexFoldFSKitModule.appex", + "/Users/test/Applications/CodexFoldFSKit.app/Contents/Extensions/CodexFoldFSKitModule.appex", + } + if got := parseFSKitModulePaths(output); !reflect.DeepEqual(got, want) { + t.Fatalf("module paths = %#v, want %#v", got, want) + } +} + +func TestSameFSKitModulePathsIgnoresOrderAndDuplicateEntries(t *testing.T) { + left := []string{"/tmp/current.appex", "/tmp/old.appex", "/tmp/current.appex"} + right := []string{"/tmp/old.appex", "/tmp/current.appex"} + if !sameFSKitModulePaths(left, right) { + t.Fatalf("module path sets differ: left=%v right=%v", left, right) + } + if sameFSKitModulePaths(left, []string{"/tmp/current.appex"}) { + t.Fatal("different module path sets were treated as equal") + } +} + +func TestStaleFSKitModulePathsKeepsInstalledModule(t *testing.T) { + target := "/Users/test/Applications/CodexFoldFSKit.app/Contents/Extensions/CodexFoldFSKitModule.appex" + paths := []string{ + "/private/tmp/candidate/CodexFoldFSKit.app/Contents/Extensions/CodexFoldFSKitModule.appex", + target, + "/private/tmp/old/CodexFoldFSKit.app/Contents/Extensions/CodexFoldFSKitModule.appex", + target, + } + want := []string{ + "/private/tmp/candidate/CodexFoldFSKit.app/Contents/Extensions/CodexFoldFSKitModule.appex", + "/private/tmp/old/CodexFoldFSKit.app/Contents/Extensions/CodexFoldFSKitModule.appex", + } + if got := staleFSKitModulePaths(paths, target); !reflect.DeepEqual(got, want) { + t.Fatalf("stale module paths = %#v, want %#v", got, want) + } +} + +func TestUnregisterFSKitAppRegistrationUsesLaunchServicesOnly(t *testing.T) { + original := runFSKitLaunchServicesCommand + t.Cleanup(func() { runFSKitLaunchServicesCommand = original }) + var got []string + runFSKitLaunchServicesCommand = func(_ context.Context, args ...string) ([]byte, error) { + got = append([]string(nil), args...) + return nil, nil + } + app := "/private/tmp/candidate/CodexFoldFSKit.app" + if err := unregisterFSKitAppRegistration(context.Background(), app); err != nil { + t.Fatal(err) + } + if want := []string{"-u", app}; !reflect.DeepEqual(got, want) { + t.Fatalf("LaunchServices cleanup args = %#v, want %#v", got, want) + } +} + +func TestFSKitParentAppPath(t *testing.T) { + module := "/private/tmp/CodexFoldFSKit.app/Contents/Extensions/CodexFoldFSKitModule.appex" + if got, ok := fsKitParentAppPath(module); !ok || got != "/private/tmp/CodexFoldFSKit.app" { + t.Fatalf("parent app = %q ok=%t", got, ok) + } + if _, ok := fsKitParentAppPath("/private/tmp/CodexFoldFSKitModule.appex"); ok { + t.Fatal("module outside an app bundle unexpectedly had a parent app") + } +} + +func TestFSKitAppContentsSwapPreservesBundleRootAndRollsBack(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "CodexFoldFSKit.app") + stageRoot := filepath.Join(root, "stage") + stagePath := filepath.Join(stageRoot, filepath.Base(target)) + writeFSKitAppTestFile(t, filepath.Join(target, "Contents", "old.txt"), "old") + writeFSKitAppTestFile(t, filepath.Join(stagePath, "Contents", "new.txt"), "new") + + attribute := "com.codexfold.test-root" + value := []byte("preserve-this-root-xattr") + if err := unix.Setxattr(target, attribute, value, 0); err != nil { + t.Fatalf("set root xattr: %v", err) + } + before, err := os.Stat(target) + if err != nil { + t.Fatal(err) + } + + transaction := &darwinFSKitAppTransaction{ + target: target, stageRoot: stageRoot, stagePath: stagePath, changed: true, + } + if err := transaction.promoteStagedApp(); err != nil { + t.Fatalf("promote staged app: %v", err) + } + assertFSKitAppRootUnchanged(t, target, before, attribute, value) + assertFSKitAppTestFile(t, filepath.Join(target, "Contents", "new.txt"), "new") + assertFSKitAppTestFile(t, filepath.Join(stagePath, "Contents", "old.txt"), "old") + + if err := transaction.rollbackStagedApp(); err != nil { + t.Fatalf("rollback staged app: %v", err) + } + assertFSKitAppRootUnchanged(t, target, before, attribute, value) + assertFSKitAppTestFile(t, filepath.Join(target, "Contents", "old.txt"), "old") + if _, err := os.Stat(filepath.Join(target, "Contents", "new.txt")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("new contents remain after rollback: %v", err) + } + if err := transaction.Commit(); err != nil { + t.Fatalf("commit cleanup: %v", err) + } + if _, err := os.Stat(stageRoot); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("staging root remains after commit: %v", err) + } +} + +func TestFSKitAppContentsSwapFirstInstallRemovesOnlyCandidateOnRollback(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "CodexFoldFSKit.app") + stageRoot := filepath.Join(root, "stage") + stagePath := filepath.Join(stageRoot, filepath.Base(target)) + writeFSKitAppTestFile(t, filepath.Join(stagePath, "Contents", "new.txt"), "new") + transaction := &darwinFSKitAppTransaction{ + target: target, stageRoot: stageRoot, stagePath: stagePath, changed: true, + } + if err := transaction.promoteStagedApp(); err != nil { + t.Fatalf("first install: %v", err) + } + if !transaction.appInstalled || transaction.hadTarget { + t.Fatalf("first-install state = %#v", transaction) + } + assertFSKitAppTestFile(t, filepath.Join(target, "Contents", "new.txt"), "new") + if err := transaction.rollbackStagedApp(); err != nil { + t.Fatalf("rollback first install: %v", err) + } + if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("candidate app remains after rollback: %v", err) + } +} + +func writeFSKitAppTestFile(t *testing.T, path string, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +func assertFSKitAppTestFile(t *testing.T, path string, want string) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != want { + t.Fatalf("file %s = %q, want %q", path, got, want) + } +} + +func assertFSKitAppRootUnchanged(t *testing.T, path string, before os.FileInfo, attribute string, want []byte) { + t.Helper() + after, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(before, after) { + t.Fatalf("app bundle root inode changed: before=%#v after=%#v", before.Sys(), after.Sys()) + } + got := make([]byte, len(want)) + n, err := unix.Getxattr(path, attribute, got) + if err != nil { + t.Fatalf("read root xattr: %v", err) + } + if string(got[:n]) != string(want) { + t.Fatalf("root xattr = %q, want %q", got[:n], want) + } +} diff --git a/internal/cli/fs_service_fskit_other.go b/internal/cli/fs_service_fskit_other.go new file mode 100644 index 0000000..871cbae --- /dev/null +++ b/internal/cli/fs_service_fskit_other.go @@ -0,0 +1,12 @@ +//go:build !darwin + +package cli + +import ( + "context" + "errors" +) + +func prepareFSKitAppPlatform(context.Context, string, string) (fsKitAppTransaction, error) { + return nil, errors.New("native FSKit app installation is available only on macOS") +} diff --git a/internal/cli/fs_service_linux_integration_test.go b/internal/cli/fs_service_linux_integration_test.go new file mode 100644 index 0000000..873eb12 --- /dev/null +++ b/internal/cli/fs_service_linux_integration_test.go @@ -0,0 +1,170 @@ +//go:build linux && fuse && fuse3 && cgo + +package cli + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "os/signal" + "path/filepath" + "sync" + "syscall" + "testing" + "time" + + "github.com/samekind/codexfold/internal/mountfs" + "github.com/samekind/codexfold/internal/service" +) + +func TestRealLinuxFSServeRecoversAfterHostSIGKILL(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE3_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE3_TEST=1 to run the real Linux fs serve crash test") + } + if !mountfs.Available() { + t.Fatal("FUSE3 host is unavailable") + } + root := t.TempDir() + home := filepath.Join(root, "codex") + store := filepath.Join(root, "store") + mount := filepath.Join(root, "mount") + native := filepath.Join(root, "native") + for _, directory := range []string{home, store, native} { + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { + _ = exec.Command("fusermount3", "-uz", mount).Run() + _ = os.Chmod(mount, 0o500) + }) + + first, firstDone, firstOutput := startLinuxFSServeHelper(t, home, store, mount, native) + waitForLinuxFSServeMount(t, mount, firstDone, firstOutput) + firstPID := first.Process.Pid + if err := first.Process.Kill(); err != nil { + t.Fatal(err) + } + if err := <-firstDone; err == nil { + t.Fatal("SIGKILLed fs serve helper exited successfully") + } + + second, secondDone, secondOutput := startLinuxFSServeHelper(t, home, store, mount, native) + waitForLinuxFSServeMount(t, mount, secondDone, secondOutput) + if second.Process.Pid == firstPID { + t.Fatal("replacement fs serve reused the killed process ID") + } + if err := second.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatal(err) + } + select { + case err := <-secondDone: + if err != nil { + t.Fatalf("replacement fs serve shutdown: %v\n%s", err, secondOutput.String()) + } + case <-time.After(15 * time.Second): + _ = second.Process.Kill() + t.Fatal("replacement fs serve did not stop after SIGTERM") + } + waitForLinuxFSServeUnmount(t, mount) + info, err := os.Stat(mount) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o500 { + t.Fatalf("unmounted fs serve backing mode=%#o", info.Mode().Perm()) + } +} + +func TestRealLinuxFSServeCrashHelper(t *testing.T) { + if os.Getenv("CODEXFOLD_FS_SERVE_CRASH_HELPER") != "1" { + t.Skip("helper process") + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + root := NewRootCommand() + root.SetOut(os.Stdout) + root.SetErr(os.Stderr) + root.SetArgs([]string{ + "fs", "serve", "--apply", "--foreground=true", + "--codex-home", os.Getenv("CODEXFOLD_FS_SERVE_HOME"), + "--store", os.Getenv("CODEXFOLD_FS_SERVE_STORE"), + "--mount", os.Getenv("CODEXFOLD_FS_SERVE_MOUNT"), + "--canonical-namespace", "--native-root", os.Getenv("CODEXFOLD_FS_SERVE_NATIVE"), + }) + if err := root.ExecuteContext(ctx); err != nil && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} + +func startLinuxFSServeHelper(t *testing.T, home string, store string, mount string, native string) (*exec.Cmd, <-chan error, *lockedBuffer) { + t.Helper() + command := exec.Command(os.Args[0], "-test.run=^TestRealLinuxFSServeCrashHelper$", "-test.v") + command.Env = append(os.Environ(), + "CODEXFOLD_FS_SERVE_CRASH_HELPER=1", + "CODEXFOLD_FS_SERVE_HOME="+home, + "CODEXFOLD_FS_SERVE_STORE="+store, + "CODEXFOLD_FS_SERVE_MOUNT="+mount, + "CODEXFOLD_FS_SERVE_NATIVE="+native, + ) + output := &lockedBuffer{} + command.Stdout = output + command.Stderr = output + if err := command.Start(); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- command.Wait() }() + t.Cleanup(func() { _ = command.Process.Kill() }) + return command, done, output +} + +func waitForLinuxFSServeMount(t *testing.T, mount string, done <-chan error, output *lockedBuffer) { + t.Helper() + deadline := time.Now().Add(20 * time.Second) + var lastErr error + for time.Now().Before(deadline) { + if err := service.ProbeMount(mount); err == nil { + return + } else { + lastErr = err + } + select { + case err := <-done: + t.Fatalf("fs serve exited before mount health: %v\n%s", err, output.String()) + case <-time.After(50 * time.Millisecond): + } + } + t.Fatalf("fs serve mount did not become healthy: %v\n%s", lastErr, output.String()) +} + +func waitForLinuxFSServeUnmount(t *testing.T, mount string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if err := service.ProbeMount(mount); err != nil { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatal("fs serve mount remained healthy after shutdown") +} + +type lockedBuffer struct { + mu sync.Mutex + b bytes.Buffer +} + +func (b *lockedBuffer) Write(value []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.b.Write(value) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.b.String() +} diff --git a/internal/cli/fs_service_runtime_other.go b/internal/cli/fs_service_runtime_other.go new file mode 100644 index 0000000..42ba33a --- /dev/null +++ b/internal/cli/fs_service_runtime_other.go @@ -0,0 +1,7 @@ +//go:build !windows + +package cli + +import "github.com/spf13/cobra" + +func addPlatformServiceCommands(*cobra.Command) {} diff --git a/internal/cli/fs_service_runtime_windows.go b/internal/cli/fs_service_runtime_windows.go new file mode 100644 index 0000000..1dd4c8c --- /dev/null +++ b/internal/cli/fs_service_runtime_windows.go @@ -0,0 +1,147 @@ +//go:build windows + +package cli + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/samekind/codexfold/internal/mountfs" + "github.com/samekind/codexfold/internal/service" + "github.com/spf13/cobra" + "golang.org/x/sys/windows/svc" +) + +func addPlatformServiceCommands(parent *cobra.Command) { + parent.AddCommand(newFSServiceRunCommand()) +} + +func newFSServiceRunCommand() *cobra.Command { + var definitionPath string + command := &cobra.Command{ + Use: "run", + Short: "Run the Windows SCM service host", + Hidden: true, + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if !mountfs.Available() { + return errors.New("Windows service runtime requires a WinFsp-enabled build") + } + if !filepath.IsAbs(definitionPath) { + return errors.New("absolute Windows service definition path is required") + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return err + } + config, err := service.ParseWindowsConfig(definition) + if err != nil { + return err + } + if config.ServiceName != serviceLabel { + return errors.New("Windows service definition name does not match this binary") + } + isService, err := svc.IsWindowsService() + if err != nil { + return err + } + if !isService { + return errors.New("Windows service run must be started by the Service Control Manager") + } + stdout, stderr, closeLogs, err := openWindowsServiceLogs(config) + if err != nil { + return err + } + defer closeLogs() + handler := &windowsFSService{ + log: stderr, + run: func(ctx context.Context) error { + serve := newFSServeCommand() + serve.SetArgs(config.Arguments[2:]) + serve.SetOut(stdout) + serve.SetErr(stderr) + serve.SilenceErrors = true + serve.SilenceUsage = true + return serve.ExecuteContext(ctx) + }, + } + return svc.Run(config.ServiceName, handler) + }, + } + command.Flags().StringVar(&definitionPath, "definition", "", "Absolute Windows service definition path") + return command +} + +type windowsFSService struct { + run func(context.Context) error + log io.Writer +} + +func (s *windowsFSService) Execute(_ []string, requests <-chan svc.ChangeRequest, changes chan<- svc.Status) (bool, uint32) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + changes <- svc.Status{State: svc.StartPending, CheckPoint: 1, WaitHint: 15000} + go func() { done <- s.run(ctx) }() + running := svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown} + changes <- running + + for { + select { + case err := <-done: + if err != nil && !errors.Is(err, context.Canceled) { + _, _ = fmt.Fprintf(s.log, "filesystem service exited: %v\n", err) + return false, 1 + } + return false, 0 + case request := <-requests: + switch request.Cmd { + case svc.Interrogate: + changes <- running + case svc.Stop, svc.Shutdown: + changes <- svc.Status{State: svc.StopPending, CheckPoint: 1, WaitHint: 30000} + cancel() + select { + case err := <-done: + if err != nil && !errors.Is(err, context.Canceled) { + _, _ = fmt.Fprintf(s.log, "filesystem service shutdown failed: %v\n", err) + return false, 1 + } + return false, 0 + case <-time.After(30 * time.Second): + _, _ = fmt.Fprintln(s.log, "filesystem service shutdown timed out") + return false, 1 + } + } + } + } +} + +func openWindowsServiceLogs(config service.WindowsConfig) (io.Writer, io.Writer, func(), error) { + for _, path := range []string{config.StdoutPath, config.StderrPath} { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, nil, nil, err + } + } + stdout, err := os.OpenFile(config.StdoutPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return nil, nil, nil, err + } + stderr, err := os.OpenFile(config.StderrPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + _ = stdout.Close() + return nil, nil, nil, err + } + closeLogs := func() { + _ = stdout.Sync() + _ = stderr.Sync() + _ = stdout.Close() + _ = stderr.Close() + } + return stdout, stderr, closeLogs, nil +} diff --git a/internal/cli/fs_service_transaction_test.go b/internal/cli/fs_service_transaction_test.go new file mode 100644 index 0000000..1cab445 --- /dev/null +++ b/internal/cli/fs_service_transaction_test.go @@ -0,0 +1,209 @@ +package cli + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "runtime" + "testing" + + "github.com/samekind/codexfold/internal/service" +) + +type rollbackTestApp struct { + changed bool + rollback func() error +} + +func (a *rollbackTestApp) AppGroupPath() string { return "/tmp/group.vip.jstar.codexfold" } +func (a *rollbackTestApp) Changed() bool { return a.changed } +func (a *rollbackTestApp) Commit() error { return nil } +func (a *rollbackTestApp) Rollback(context.Context) error { + if a.rollback == nil { + return nil + } + return a.rollback() +} + +func TestRollbackFailedServiceInstallRestoresDefinitionAndAppBeforeRestart(t *testing.T) { + root := t.TempDir() + definition := filepath.Join(root, "com.codexfold.test.plist") + binary := filepath.Join(root, "codexfold") + candidate := filepath.Join(root, "candidate") + if err := os.WriteFile(definition, []byte("old-definition"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(binary, []byte("old-binary"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(candidate, []byte("new-binary"), 0o700); err != nil { + t.Fatal(err) + } + update, err := service.StageDefinitionUpdate(definition, []byte("new-definition")) + if err != nil { + t.Fatal(err) + } + if err := update.Promote(); err != nil { + t.Fatal(err) + } + binaryUpdate, err := service.StageBinaryUpdate(candidate, binary) + if err != nil { + t.Fatal(err) + } + if err := binaryUpdate.Promote(); err != nil { + t.Fatal(err) + } + + var order []string + appRolledBack := false + app := &rollbackTestApp{changed: true, rollback: func() error { + current, err := os.ReadFile(definition) + if err != nil { + return err + } + if string(current) != "old-definition" { + return errors.New("app rollback ran before the service definition was restored") + } + order = append(order, "app-rollback") + appRolledBack = true + return nil + }} + stop := func(context.Context, service.Platform, string) error { + order = append(order, "stop") + return errors.New("already stopped") + } + start := func(context.Context, service.Platform, string, string) error { + if !appRolledBack { + return errors.New("service restarted before app rollback") + } + current, err := os.ReadFile(definition) + if err != nil { + return err + } + if string(current) != "old-definition" { + return errors.New("service restarted before definition rollback") + } + current, err = os.ReadFile(binary) + if err != nil { + return err + } + if string(current) != "old-binary" { + return errors.New("service restarted before binary rollback") + } + order = append(order, "start") + return nil + } + + if err := rollbackFailedServiceInstall( + context.Background(), service.PlatformLaunchd, definition, filepath.Join(root, "mount"), + []*service.DefinitionUpdate{update}, app, binaryUpdate, true, stop, start, + ); err != nil { + t.Fatal(err) + } + if want := []string{"stop", "app-rollback", "start"}; !reflect.DeepEqual(order, want) { + t.Fatalf("rollback order = %v, want %v", order, want) + } + artifacts, err := filepath.Glob(filepath.Join(root, ".codexfold-definition-*")) + if err != nil { + t.Fatal(err) + } + if len(artifacts) != 0 { + t.Fatalf("definition rollback artifacts remain: %v", artifacts) + } +} + +func TestRollbackFailedFirstInstallDoesNotStartAService(t *testing.T) { + root := t.TempDir() + definition := filepath.Join(root, "com.codexfold.test.plist") + update, err := service.StageDefinitionUpdate(definition, []byte("new-definition")) + if err != nil { + t.Fatal(err) + } + if err := update.Promote(); err != nil { + t.Fatal(err) + } + started := false + if err := rollbackFailedServiceInstall( + context.Background(), service.PlatformLaunchd, definition, filepath.Join(root, "mount"), + []*service.DefinitionUpdate{update}, nil, nil, false, + func(context.Context, service.Platform, string) error { return nil }, + func(context.Context, service.Platform, string, string) error { started = true; return nil }, + ); err != nil { + t.Fatal(err) + } + if started { + t.Fatal("failed first install restarted a service that did not previously exist") + } + if _, err := os.Stat(definition); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("failed first install left definition behind: %v", err) + } +} + +func TestValidateLaunchdChildProcessRequiresLockOwnerToBelongToHost(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("launchd child ancestry is macOS-only") + } + lockPath := filepath.Join(t.TempDir(), "service.lock") + lock, err := service.AcquireProcessLock(lockPath) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + + status := validateLaunchdChildProcess(service.Status{DaemonRunning: true, DaemonPID: os.Getppid()}, lockPath, "test") + if !status.DaemonRunning || status.DaemonError != "" { + t.Fatalf("valid child process was rejected: %#v", status) + } + status = validateLaunchdChildProcess(service.Status{DaemonRunning: true, DaemonPID: os.Getppid() + 1}, lockPath, "test") + if status.DaemonRunning || status.DaemonError == "" { + t.Fatalf("foreign child process was accepted: %#v", status) + } +} + +func TestNativeFSKitProcessLocksReportHeldOwners(t *testing.T) { + root := t.TempDir() + paths := nativeFSKitProcessLockPaths{ + daemon: filepath.Join(root, "service.lock"), + supervisor: filepath.Join(root, "supervisor.lock"), + } + daemon, err := service.AcquireProcessLock(paths.daemon) + if err != nil { + t.Fatal(err) + } + defer daemon.Close() + supervisor, err := service.AcquireProcessLock(paths.supervisor) + if err != nil { + t.Fatal(err) + } + defer supervisor.Close() + + daemonStatus, err := service.InspectProcessLock(paths.daemon) + if err != nil { + t.Fatal(err) + } + supervisorStatus, err := service.InspectProcessLock(paths.supervisor) + if err != nil { + t.Fatal(err) + } + if !daemonStatus.Held || daemonStatus.PID != os.Getpid() { + t.Fatalf("daemon lock = %#v", daemonStatus) + } + if !supervisorStatus.Held || supervisorStatus.PID != os.Getpid() { + t.Fatalf("supervisor lock = %#v", supervisorStatus) + } +} + +func TestNativeFSKitServiceInactiveRequiresUnmountedMount(t *testing.T) { + status := service.Status{} + if nativeFSKitServiceInactive(status, service.ProcessLockStatus{}, service.ProcessLockStatus{}, true, nil) { + t.Fatal("an unhealthy but still-mounted filesystem was accepted as inactive") + } + if nativeFSKitServiceInactive(status, service.ProcessLockStatus{}, service.ProcessLockStatus{}, false, errors.New("mount state unknown")) { + t.Fatal("an unknown mount state was accepted as inactive") + } + if !nativeFSKitServiceInactive(status, service.ProcessLockStatus{}, service.ProcessLockStatus{}, false, nil) { + t.Fatal("fully stopped native FSKit service was not accepted as inactive") + } +} diff --git a/internal/cli/fs_supervisor.go b/internal/cli/fs_supervisor.go new file mode 100644 index 0000000..f0c18f0 --- /dev/null +++ b/internal/cli/fs_supervisor.go @@ -0,0 +1,74 @@ +package cli + +import ( + "errors" + "fmt" + "path/filepath" + "runtime" + "time" + + "github.com/samekind/codexfold/internal/service" + "github.com/spf13/cobra" +) + +type FSNativeSupervisorResult struct { + ResourcePath string `json:"resource_path"` + MountPoint string `json:"mount_point"` + Interval time.Duration `json:"interval"` + ProbeTimeout time.Duration `json:"probe_timeout"` + Recovery time.Duration `json:"recovery_timeout"` + DryRun bool `json:"dry_run"` +} + +func newFSNativeSupervisorCommand() *cobra.Command { + var resourcePath, mountPoint string + var interval, probeTimeout, recoveryTimeout time.Duration + var apply, jsonOutput bool + command := &cobra.Command{ + Use: "supervise", + Short: "Keep the native FSKit mount healthy and remount it after daemon or extension failure", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + if !filepath.IsAbs(resourcePath) || !filepath.IsAbs(mountPoint) { + return errors.New("absolute FSKit resource and mount paths are required") + } + if interval <= 0 || probeTimeout <= 0 || recoveryTimeout <= 0 { + return errors.New("supervisor timing values must be positive") + } + result := FSNativeSupervisorResult{ + ResourcePath: filepath.Clean(resourcePath), MountPoint: filepath.Clean(mountPoint), + Interval: interval, ProbeTimeout: probeTimeout, Recovery: recoveryTimeout, DryRun: !apply, + } + if !apply { + if jsonOutput { + return writeJSON(command, result) + } + _, err := fmt.Fprintf(command.OutOrStdout(), "dry_run=true resource=%s mount=%s interval=%s probe_timeout=%s recovery_timeout=%s\n", result.ResourcePath, result.MountPoint, result.Interval, result.ProbeTimeout, result.Recovery) + return err + } + if runtime.GOOS != "darwin" { + return errors.New("native FSKit supervision is available only on macOS") + } + processLock, err := service.AcquireProcessLock(filepath.Join(result.ResourcePath, service.NativeFSKitSupervisorLockName)) + if err != nil { + return err + } + defer processLock.Close() + return service.RunNativeFSKitSupervisor(command.Context(), service.NativeFSKitSupervisorOptions{ + ResourcePath: result.ResourcePath, MountPoint: result.MountPoint, + Interval: result.Interval, ProbeTimeout: result.ProbeTimeout, RecoveryTimeout: result.Recovery, + Event: func(message string) { + _, _ = fmt.Fprintf(command.ErrOrStderr(), "native-fskit supervisor: %s\n", message) + }, + }) + }, + } + command.Flags().StringVar(&resourcePath, "resource", "", "Absolute native FSKit resource descriptor path") + command.Flags().StringVar(&mountPoint, "mount", "", "Absolute native FSKit mount point") + command.Flags().DurationVar(&interval, "interval", time.Second, "Health reconciliation interval") + command.Flags().DurationVar(&probeTimeout, "probe-timeout", 2*time.Second, "Maximum duration of one mount health probe") + command.Flags().DurationVar(&recoveryTimeout, "recovery-timeout", 15*time.Second, "Maximum duration of one mount or unmount recovery") + command.Flags().BoolVar(&apply, "apply", false, "Run the native FSKit mount supervisor") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output for dry-run") + return command +} diff --git a/internal/cli/fs_supervisor_test.go b/internal/cli/fs_supervisor_test.go new file mode 100644 index 0000000..cb1160e --- /dev/null +++ b/internal/cli/fs_supervisor_test.go @@ -0,0 +1,32 @@ +package cli + +import ( + "bytes" + "encoding/json" + "path/filepath" + "testing" +) + +func TestFSNativeSupervisorDryRunReportsAbsoluteRuntimePaths(t *testing.T) { + root := t.TempDir() + command := NewRootCommand() + var output bytes.Buffer + command.SetOut(&output) + command.SetErr(&output) + command.SetArgs([]string{ + "fs", "supervise", + "--resource", filepath.Join(root, "resource.bin"), + "--mount", filepath.Join(root, "mount"), + "--json", + }) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + var result FSNativeSupervisorResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + t.Fatalf("decode supervisor dry-run: %v\n%s", err, output.String()) + } + if !result.DryRun || result.ResourcePath == "" || result.MountPoint == "" { + t.Fatalf("unexpected supervisor dry-run: %#v", result) + } +} diff --git a/internal/cli/fs_test.go b/internal/cli/fs_test.go new file mode 100644 index 0000000..ce039e2 --- /dev/null +++ b/internal/cli/fs_test.go @@ -0,0 +1,2981 @@ +package cli + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/compat" + "github.com/samekind/codexfold/internal/enroll" + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/fsctl" + "github.com/samekind/codexfold/internal/mountfs" + "github.com/samekind/codexfold/internal/pack" + "github.com/samekind/codexfold/internal/storage" + "github.com/samekind/codexfold/internal/vfs" +) + +type cliRejectingChecker struct { + Calls int + Projection storage.Projection +} + +func (c *cliRejectingChecker) Check(_ context.Context, projection storage.Projection) (storage.Assessment, error) { + c.Calls++ + c.Projection = projection + return storage.Assessment{}, storage.ErrBudgetExceeded +} + +func TestRootExposesPackAndFilesystemCommands(t *testing.T) { + root := NewRootCommand() + for _, commandPath := range [][]string{ + {"pack", "build"}, {"pack", "doctor"}, + {"fs", "status"}, {"fs", "doctor"}, {"fs", "validate-native"}, {"fs", "compatibility"}, {"fs", "compatibility-import"}, {"fs", "benchmark"}, + {"fs", "serve"}, {"fs", "migrate"}, {"fs", "rollback"}, {"fs", "compact"}, {"fs", "recover"}, + {"fs", "enroll", "plan"}, {"fs", "enroll", "apply"}, + {"fs", "namespace", "status"}, {"fs", "namespace", "activate"}, + {"fs", "namespace", "deactivate"}, {"fs", "namespace", "recover"}, + {"fs", "service", "install"}, {"fs", "service", "start"}, {"fs", "service", "stop"}, + {"fs", "service", "status"}, {"fs", "service", "update-binary"}, {"fs", "service", "update-preflight"}, + } { + if _, _, err := root.Find(commandPath); err != nil { + t.Fatalf("command %v should be exposed: %v", commandPath, err) + } + } +} + +func TestRetainCanonicalSnapshotBudgetRejectsBeforeCreatingSnapshot(t *testing.T) { + root := t.TempDir() + store := filepath.Join(root, "store") + sourcePath := filepath.Join(root, "source.jsonl") + if err := os.WriteFile(sourcePath, []byte("budgeted snapshot"), 0o600); err != nil { + t.Fatal(err) + } + source, err := hashPath(sourcePath) + if err != nil { + t.Fatal(err) + } + checker := &cliRejectingChecker{} + if _, err := retainCanonicalSnapshot(context.Background(), store, "session", source, checker); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("retainCanonicalSnapshot error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 || checker.Projection.Operation != "retain-migration-snapshot" || checker.Projection.AdditionalPersistentBytes != source.Bytes { + t.Fatalf("unexpected snapshot budget projection: %#v", checker) + } + if _, err := os.Stat(filepath.Join(store, "fs", "snapshots")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("snapshot directory exists after preflight rejection: %v", err) + } +} + +func TestFSCompatibilityImportPersistsOnlySanitizedContract(t *testing.T) { + home := t.TempDir() + store := filepath.Join(home, "fold-store") + trace := filepath.Join(home, "private-trace.log") + traceText := "12:00:00 open /Users/private/.codex/secret.jsonl codex.1\n12:00:01 read /Users/private/.codex/secret.jsonl codex.1\n" + if err := os.WriteFile(trace, []byte(traceText), 0o600); err != nil { + t.Fatal(err) + } + executeFS(t, []string{ + "fs", "compatibility-import", "--apply", "--codex-home", home, "--store", store, + "--trace", trace, "--client-kind", "cli", "--client-version", "1.2.3", + }) + contracts, err := compat.LoadAll(filepath.Join(store, "compatibility")) + if err != nil || len(contracts) != 1 || contracts[0].ClientVersion != "1.2.3" { + t.Fatalf("contracts = %#v err=%v", contracts, err) + } + data, err := os.ReadFile(filepath.Join(store, "compatibility", runtime.GOOS, "cli", "1.2.3.json")) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(data, []byte("/Users/private")) || bytes.Contains(data, []byte("secret.jsonl")) { + t.Fatalf("sanitized contract leaked trace content: %s", data) + } +} + +func TestOperationRecorderWritesOnlyTimeAndOperation(t *testing.T) { + tracePath := filepath.Join(t.TempDir(), "operations.log") + record, closer, err := newOperationRecorder(tracePath) + if err != nil { + t.Fatal(err) + } + record("open") + record("read") + if err := closer.Close(); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(tracePath) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(data, []byte("/")) || !bytes.Contains(data, []byte(" open\n")) || !bytes.Contains(data, []byte(" read\n")) { + t.Fatalf("operation trace = %q", data) + } +} + +func TestFSNamespaceActivateAndDeactivateCommandsPreserveNativeFiles(t *testing.T) { + root := t.TempDir() + home := filepath.Join(root, "home") + mount := filepath.Join(root, "mount") + nativeRoot := filepath.Join(home, "fold-native") + configPath := filepath.Join(home, "config.toml") + authPath := filepath.Join(home, "auth.json") + configBefore := []byte("model_provider = \"third-party\"\n") + authBefore := []byte("{\"access_token\":\"test\"}\n") + if err := os.MkdirAll(home, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(configPath, configBefore, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(authPath, authBefore, 0o600); err != nil { + t.Fatal(err) + } + for _, path := range []string{ + filepath.Join(home, "sessions", "active.jsonl"), + filepath.Join(home, "archived_sessions", "archived.jsonl"), + } { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(filepath.Base(path)), 0o600); err != nil { + t.Fatal(err) + } + } + for _, directory := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(mount, directory), 0o700); err != nil { + t.Fatal(err) + } + } + database, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`create table threads (id text primary key, rollout_path text not null)`); err != nil { + _ = database.Close() + t.Fatal(err) + } + if err := database.Close(); err != nil { + t.Fatal(err) + } + previousProbe := mountHealthProbe + t.Cleanup(func() { mountHealthProbe = previousProbe }) + mountHealthProbe = func(string) error { return nil } + executeFS(t, []string{ + "fs", "namespace", "activate", "--apply", + "--codex-home", home, "--mount", mount, "--native-root", nativeRoot, + }) + for _, directory := range []string{"sessions", "archived_sessions"} { + if target, err := os.Readlink(filepath.Join(home, directory)); err != nil || filepath.Clean(target) != filepath.Join(mount, directory) { + t.Fatalf("namespace link %s = %q err=%v", directory, target, err) + } + } + mountHealthProbe = func(string) error { return errors.New("not mounted") } + executeFS(t, []string{ + "fs", "namespace", "deactivate", "--apply", + "--codex-home", home, "--mount", mount, "--native-root", nativeRoot, + }) + for _, path := range []string{ + filepath.Join(home, "sessions", "active.jsonl"), + filepath.Join(home, "archived_sessions", "archived.jsonl"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("restored file %s: %v", path, err) + } + } + if got, err := os.ReadFile(configPath); err != nil || !bytes.Equal(got, configBefore) { + t.Fatalf("config.toml changed during namespace lifecycle: %q err=%v", got, err) + } + if got, err := os.ReadFile(authPath); err != nil || !bytes.Equal(got, authBefore) { + t.Fatalf("auth.json changed during namespace lifecycle: %q err=%v", got, err) + } +} + +func TestFSNamespaceDeactivateRejectsManagedSessions(t *testing.T) { + root := t.TempDir() + home := filepath.Join(root, "home") + mount := filepath.Join(root, "mount") + nativeRoot := filepath.Join(home, "fold-native") + store := filepath.Join(home, "fold-store") + for _, directory := range []string{ + filepath.Join(home, "sessions"), filepath.Join(home, "archived_sessions"), + filepath.Join(mount, "sessions"), filepath.Join(mount, "archived_sessions"), + filepath.Join(nativeRoot, "sessions"), filepath.Join(nativeRoot, "archived_sessions"), + } { + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + } + stateDirectory := filepath.Join(store, "fs", "sessions", "managed") + if err := os.MkdirAll(stateDirectory, 0o700); err != nil { + t.Fatal(err) + } + state := vfs.SessionState{ + Version: 1, SessionID: "managed", Generation: 1, + ManifestPath: filepath.Join(store, "manifests", "managed.json"), + BaseSHA256: "0000000000000000000000000000000000000000000000000000000000000000", + DeltaPath: filepath.Join(stateDirectory, "delta.jsonl"), + } + data, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(stateDirectory, "state.json"), data, 0o600); err != nil { + t.Fatal(err) + } + previousProbe := mountHealthProbe + t.Cleanup(func() { mountHealthProbe = previousProbe }) + mountHealthProbe = func(string) error { return errors.New("not mounted") } + command := NewRootCommand() + command.SetOut(&bytes.Buffer{}) + command.SetErr(&bytes.Buffer{}) + command.SetArgs([]string{ + "fs", "namespace", "deactivate", "--apply", + "--codex-home", home, "--store", store, "--mount", mount, "--native-root", nativeRoot, + }) + err = command.Execute() + if err == nil || err.Error() != "rollback all managed sessions before deactivating the namespace" { + t.Fatalf("deactivate error = %v", err) + } +} + +func TestCanonicalMountRouteMirrorsCodexSessionNamespace(t *testing.T) { + home := filepath.Join(string(filepath.Separator), "tmp", "codex-home") + mount := filepath.Join(string(filepath.Separator), "tmp", "codex-fold") + route := filepath.Join(home, "archived_sessions", "rollout-session.jsonl") + got, err := canonicalMountRoute(home, mount, route) + if err != nil || got != filepath.Join(mount, "archived_sessions", "rollout-session.jsonl") { + t.Fatalf("canonicalMountRoute = %q err=%v", got, err) + } + if _, err := canonicalMountRoute(home, mount, filepath.Join(home, "other", "rollout.jsonl")); err == nil { + t.Fatal("non-canonical Codex route should be rejected") + } +} + +func TestCanonicalSessionRoutesIgnoreUnmanagedSessionsOutsideCodexHome(t *testing.T) { + home := filepath.Join(string(filepath.Separator), "tmp", "codex-home") + mount := filepath.Join(home, "fold-fs") + states := []vfs.SessionState{{SessionID: "managed"}} + sessions := []codex.Session{ + {ID: "managed", RolloutPath: filepath.Join(home, "sessions", "2026", "07", "12", "rollout-managed.jsonl")}, + {ID: "unmanaged", RolloutPath: filepath.Join(string(filepath.Separator), "tmp", "native-fallback.jsonl")}, + } + routes, err := canonicalSessionRoutes(home, mount, filepath.Join(home, "fold-store"), states, sessions) + if err != nil { + t.Fatal(err) + } + if len(routes) != 1 || routes["managed"] != "/sessions/2026/07/12/rollout-managed.jsonl" { + t.Fatalf("canonical routes = %#v", routes) + } +} + +func TestCanonicalSessionRoutesSkipManagedNativeFallback(t *testing.T) { + home := t.TempDir() + mount := filepath.Join(home, "fold-fs") + store := filepath.Join(home, "fold-store") + state := vfs.SessionState{SessionID: "session"} + fallback := filepath.Join(store, "fs", "sessions", "session", "quarantine-current.jsonl") + routes, err := canonicalSessionRoutes(home, mount, store, []vfs.SessionState{state}, []codex.Session{{ID: "session", RolloutPath: fallback}}) + if err != nil { + t.Fatal(err) + } + if len(routes) != 0 { + t.Fatalf("native fallback leaked into canonical routes: %#v", routes) + } +} + +func TestCanonicalSessionRoutesAcceptDesktopMountAlias(t *testing.T) { + home := filepath.Join(string(filepath.Separator), "tmp", "codex-home") + mount := filepath.Join(home, "fold-fs") + states := []vfs.SessionState{{SessionID: "managed"}} + sessions := []codex.Session{{ + ID: "managed", + RolloutPath: filepath.Join(mount, "sessions", "2026", "07", "13", "rollout-managed.jsonl"), + }} + routes, err := canonicalSessionRoutes(home, mount, filepath.Join(home, "fold-store"), states, sessions) + if err != nil { + t.Fatal(err) + } + if len(routes) != 1 || routes["managed"] != "/sessions/2026/07/13/rollout-managed.jsonl" { + t.Fatalf("canonical routes from mount alias = %#v", routes) + } +} + +func TestDiscoverCanonicalRoutesSkipsCodexDatabaseWhenNoSessionsAreManaged(t *testing.T) { + called := false + routes, err := discoverCanonicalRoutes("/tmp/codex-home", "/tmp/codex-home/fold-fs", "/tmp/store", nil, func(string) ([]codex.Session, error) { + called = true + return nil, errors.New("database should not be opened") + }) + if err != nil || called || len(routes) != 0 { + t.Fatalf("empty canonical routes = %#v called=%t err=%v", routes, called, err) + } +} + +func TestCanonicalFSServeRequiresAbsoluteNativeRoot(t *testing.T) { + home := t.TempDir() + for _, nativeRoot := range []string{"", "relative-native-root"} { + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "serve", + "--canonical-namespace", + "--native-root", nativeRoot, + "--codex-home", home, + }) + if err := root.Execute(); err == nil { + t.Fatalf("native root %q should be rejected", nativeRoot) + } + } +} + +func TestFSServiceInstallIsDryRunByDefaultAndApplyRequiresFuseBuild(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + plistPath := filepath.Join(home, "LaunchAgents", "com.codexfold.fs.plist") + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "service", "install", "--codex-home", home, "--store", storeDir, "--plist", plistPath, "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("service install dry-run: %v", err) + } + if _, err := os.Stat(plistPath); !os.IsNotExist(err) { + t.Fatalf("dry-run wrote plist: %v", err) + } + if mountfs.Available() { + return + } + root = NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "service", "install", "--codex-home", home, "--store", storeDir, "--plist", plistPath, "--apply"}) + err := root.Execute() + if err == nil { + t.Fatal("default build should reject service installation without a FUSE host") + } +} + +func TestFSServiceInstallRendersNativeFSKitDaemonAndSupervisor(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("native FSKit services are macOS-only") + } + home, storeDir, _ := fsFixture(t, true) + definition := filepath.Join(home, "LaunchAgents", "com.codexfold.fs.plist") + fskitApp := filepath.Join(home, "Applications", "CodexFoldFSKit.app") + installedBinary := filepath.Join(home, "bin", "codexfold") + candidateBinary := filepath.Join(home, "build", "codexfold") + if err := os.MkdirAll(filepath.Dir(installedBinary), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(candidateBinary), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(installedBinary, []byte("installed"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(candidateBinary, []byte("candidate"), 0o700); err != nil { + t.Fatal(err) + } + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "service", "install", "--frontend", "native-fskit", + "--codex-home", home, "--store", storeDir, "--definition", definition, + "--fskit-app", fskitApp, "--binary", installedBinary, "--binary-source", candidateBinary, "--json", + }) + if err := root.Execute(); err != nil { + t.Fatalf("native FSKit service dry-run: %v", err) + } + var result FSServiceInstallResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + t.Fatal(err) + } + wantSupervisor := filepath.Join(filepath.Dir(definition), "com.codexfold.fs.supervisor.plist") + if !result.DryRun || result.Path != definition || result.SupervisorPath != wantSupervisor || result.SupervisorBytes == 0 { + t.Fatalf("native FSKit install result = %#v", result) + } + for _, path := range []string{definition, wantSupervisor} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("dry-run wrote %s: %v", path, err) + } + } + launcher := filepath.Join(fskitApp, "Contents", "MacOS", "CodexFoldFSKit") + if result.FSKitAppPath != fskitApp || result.FSKitLauncherPath != launcher || result.FSKitResourcePath == "" { + t.Fatalf("native FSKit resolved paths = %#v", result) + } + if result.BinarySourcePath != candidateBinary || result.BinaryCurrentSHA256 == "" || result.BinaryCandidateSHA256 == "" || !result.BinaryChanged { + t.Fatalf("native FSKit binary transaction = %#v", result) + } + if content, err := os.ReadFile(installedBinary); err != nil || string(content) != "installed" { + t.Fatalf("dry-run changed installed binary: content=%q err=%v", content, err) + } +} + +func TestFSServiceInstallCustomLabelUsesCustomDefinitionPath(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("custom LaunchAgent labels are macOS-only") + } + home, storeDir, _ := fsFixture(t, true) + label := "com.codexfold.native-service-e2e" + userHome, err := os.UserHomeDir() + if err != nil { + t.Fatal(err) + } + defaultDefinition := filepath.Join(userHome, "Library", "LaunchAgents", "com.codexfold.fs.plist") + customDefinition := filepath.Join(userHome, "Library", "LaunchAgents", label+".plist") + defaultBefore, defaultErr := os.ReadFile(defaultDefinition) + defaultExists := defaultErr == nil + if defaultErr != nil && !errors.Is(defaultErr, os.ErrNotExist) { + t.Fatal(defaultErr) + } + customBefore, customErr := os.ReadFile(customDefinition) + customExists := customErr == nil + if customErr != nil && !errors.Is(customErr, os.ErrNotExist) { + t.Fatal(customErr) + } + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "service", "install", "--frontend", "native-fskit", + "--label", label, "--codex-home", home, "--store", storeDir, "--json", + }) + if err := root.Execute(); err != nil { + t.Fatalf("custom native FSKit service dry-run: %v", err) + } + var result FSServiceInstallResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + t.Fatal(err) + } + wantSupervisor := filepath.Join(userHome, "Library", "LaunchAgents", label+".supervisor.plist") + if result.Path != customDefinition || result.SupervisorPath != wantSupervisor || !result.DryRun { + t.Fatalf("custom service install result = %#v", result) + } + defaultAfter, defaultErr := os.ReadFile(defaultDefinition) + if (defaultErr == nil) != defaultExists || (defaultExists && !bytes.Equal(defaultBefore, defaultAfter)) { + t.Fatalf("custom dry-run changed default definition %s", defaultDefinition) + } + customAfter, customErr := os.ReadFile(customDefinition) + if (customErr == nil) != customExists || (customExists && !bytes.Equal(customBefore, customAfter)) { + t.Fatalf("custom dry-run changed custom definition %s", customDefinition) + } + if _, err := os.Stat(wantSupervisor); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("custom dry-run wrote supervisor %s: %v", wantSupervisor, err) + } +} + +func TestFSServiceRestartIsExposedAsDryRun(t *testing.T) { + home := t.TempDir() + definition := filepath.Join(home, "com.codexfold.fs.plist") + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "service", "restart", "--definition", definition, "--json"}) + if err := root.Execute(); err != nil { + t.Fatal(err) + } + var result FSServiceActionResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + t.Fatal(err) + } + if result.Action != "restart" || !result.DryRun || result.Path != definition { + t.Fatalf("restart dry-run = %#v", result) + } +} + +func TestFSUpdatePreflightQuarantineRoutesLatestVisibleBytesNative(t *testing.T) { + allowFixtureMount(t) + home, storeDir, nativePath := fsFixture(t, true) + approvedCLI := approvedCLIContract(t, storeDir, "1.2.3") + mount := filepath.Join(home, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(mount, "session.jsonl") + original, _ := os.ReadFile(nativePath) + if err := os.WriteFile(target, original, 0o600); err != nil { + t.Fatal(err) + } + executeFS(t, []string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", mount, "--cli", approvedCLI, "--desktop-app", "none", "--apply"}) + state, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + managed, resolver, err := openManagedSession(context.Background(), storeDir, state) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + tail := []byte("{\"after_upgrade\":true}\n") + if _, err := writer.Append(context.Background(), tail); err != nil { + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + t.Fatal(err) + } + _ = writer.Close() + _ = resolver.Close() + unknownCLI := fakeCLI(t, "9.9.9") + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "service", "update-preflight", "--codex-home", home, "--store", storeDir, "--cli", unknownCLI, "--desktop-app", "none", "--apply-quarantine", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("update preflight: %v", err) + } + var result FSUpdatePreflightResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil || !result.Decision.Quarantine || result.Decision.RequiresNativeFallback || result.QuarantinedSessions != 1 { + t.Fatalf("unexpected quarantine result: %#v err=%v output=%s", result, err, output.String()) + } + sessions, err := codex.LoadSessions(home) + if err != nil { + t.Fatal(err) + } + quarantineBytes, err := os.ReadFile(sessions[0].RolloutPath) + if err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), original...), tail...) + if !bytes.Equal(quarantineBytes, want) { + t.Fatalf("quarantine route is stale: got=%q want=%q", quarantineBytes, want) + } + if _, err := managedState(storeDir, "session"); err == nil { + t.Fatal("quarantine left the session managed") + } +} + +func TestPackBuildAndDoctorCommandsUseFoldStore(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + var output bytes.Buffer + root := NewRootCommand() + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"pack", "build", "--codex-home", home, "--store", storeDir, "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("pack build: %v", err) + } + var build pack.BuildResult + if err := json.Unmarshal(output.Bytes(), &build); err != nil || build.ObjectCount == 0 { + t.Fatalf("unexpected pack build: %#v err=%v output=%s", build, err, output.String()) + } + + output.Reset() + root = NewRootCommand() + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"pack", "doctor", "--store", storeDir, "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("pack doctor: %v", err) + } + var doctor pack.DoctorResult + if err := json.Unmarshal(output.Bytes(), &doctor); err != nil || doctor.IssueCount != 0 { + t.Fatalf("unexpected pack doctor: %#v err=%v output=%s", doctor, err, output.String()) + } +} + +func TestFSMigrateIsDryRunByDefaultAndDoesNotChangeRoute(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", filepath.Join(home, "mount"), "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("fs migrate dry-run: %v", err) + } + var result FSMigrateResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil || !result.DryRun || result.Routed { + t.Fatalf("unexpected migrate result: %#v err=%v output=%s", result, err, output.String()) + } + sessions, err := codex.LoadSessions(home) + if err != nil || sessions[0].RolloutPath != nativePath { + t.Fatalf("dry-run changed route: sessions=%#v err=%v", sessions, err) + } +} + +func TestFSMigrateRejectsInvalidNativeRolloutBeforeCreatingManagedState(t *testing.T) { + for _, test := range []struct { + name string + data []byte + want string + }{ + {name: "invalid UTF-8", data: []byte{'{', '"', 'x', '"', ':', '"', 0xff, '"', '}', '\n'}, want: "not valid UTF-8"}, + {name: "invalid JSON", data: []byte("not-json\n"), want: "not valid JSON"}, + {name: "missing final newline", data: []byte("{\"record\":0}"), want: "missing its final newline"}, + } { + t.Run(test.name, func(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + if err := os.WriteFile(nativePath, test.data, 0o600); err != nil { + t.Fatal(err) + } + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", filepath.Join(home, "mount")}) + err := root.Execute() + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("migrate error = %v, want %q", err, test.want) + } + sessions, loadErr := codex.LoadSessions(home) + if loadErr != nil || len(sessions) != 1 || filepath.Clean(sessions[0].RolloutPath) != filepath.Clean(nativePath) { + t.Fatalf("invalid migration changed route: sessions=%#v err=%v", sessions, loadErr) + } + states, stateErr := vfs.DiscoverSessionStates(storeDir) + if stateErr != nil || len(states) != 0 { + t.Fatalf("invalid migration created managed state: states=%#v err=%v", states, stateErr) + } + }) + } +} + +func TestFSEnrollmentPlanRequiresTwoStableObservationsAndCanaryGate(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + allowEnrollmentWriterProbe(t) + oldProbe := mountHealthProbe + mountHealthProbe = func(string) error { return nil } + t.Cleanup(func() { mountHealthProbe = oldProbe }) + mount := filepath.Join(home, "mount") + nativeRoot := filepath.Join(home, "fold-native") + args := []string{ + "fs", "enroll", "plan", "--codex-home", home, "--store", storeDir, "--mount", mount, + "--canonical-namespace", "--native-root", nativeRoot, "--enrollment-canary", "--stable-for", "1ns", "--record-observations", "--json", + } + runPlan := func() enroll.Plan { + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs(args) + if err := root.Execute(); err != nil { + t.Fatalf("enrollment plan: %v", err) + } + var plan enroll.Plan + if err := json.Unmarshal(output.Bytes(), &plan); err != nil { + t.Fatalf("decode enrollment plan: %v\n%s", err, output.String()) + } + return plan + } + first := runPlan() + if len(first.Selected) != 0 { + t.Fatalf("first observation selected enrollment: %#v", first) + } + time.Sleep(time.Millisecond) + second := runPlan() + if len(second.Selected) != 1 || second.Selected[0].SessionID != "session" { + t.Fatalf("second stable observation was not selected: %#v", second) + } +} + +func TestFSEnrollmentApplyRunsFoldPackMigrateAndStopsBeforeRouteOnFailure(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + allowEnrollmentWriterProbe(t) + oldProbe := mountHealthProbe + mountHealthProbe = func(string) error { return nil } + t.Cleanup(func() { mountHealthProbe = oldProbe }) + observationPath := enrollmentObservationPath(storeDir) + info, err := os.Stat(nativePath) + if err != nil { + t.Fatal(err) + } + if err := enroll.SaveObservations(observationPath, enroll.Observations{"session": { + Path: nativePath, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), StableSinceUnixNano: time.Now().Add(-time.Hour).UnixNano(), + }}); err != nil { + t.Fatal(err) + } + oldRunner := runEnrollmentCommand + defer func() { runEnrollmentCommand = oldRunner }() + var calls [][]string + runEnrollmentCommand = func(_ context.Context, args []string) error { + calls = append(calls, append([]string(nil), args...)) + if len(calls) == 2 { + return errors.New("pack failed") + } + return nil + } + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "enroll", "apply", "--codex-home", home, "--store", storeDir, "--mount", filepath.Join(home, "mount"), + "--canonical-namespace", "--native-root", filepath.Join(home, "fold-native"), "--enrollment-canary", "--stable-for", "1ns", "--apply", + }) + if err := root.Execute(); err == nil || !strings.Contains(err.Error(), "pack failed") { + t.Fatalf("enrollment apply error = %v, want pack failure", err) + } + if len(calls) != 2 || len(calls[0]) < 2 || calls[0][0] != "fold" || calls[1][0] != "pack" { + t.Fatalf("enrollment command sequence = %#v", calls) + } + sessions, err := codex.LoadSessions(home) + if err != nil || len(sessions) != 1 || filepath.Clean(sessions[0].RolloutPath) != filepath.Clean(nativePath) { + t.Fatalf("failed enrollment changed route: sessions=%#v err=%v", sessions, err) + } + if data, err := os.ReadFile(nativePath); err != nil || len(data) == 0 { + t.Fatalf("failed enrollment changed source: bytes=%d err=%v", len(data), err) + } +} + +func TestFSEnrollmentApplyRejectsInvalidNativeRolloutBeforeCommands(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + allowEnrollmentWriterProbe(t) + allowFixtureMount(t) + invalid := []byte{'{', '"', 'x', '"', ':', '"', 0xff, '"', '}', '\n'} + if err := os.WriteFile(nativePath, invalid, 0o600); err != nil { + t.Fatal(err) + } + info, err := os.Stat(nativePath) + if err != nil { + t.Fatal(err) + } + if err := enroll.SaveObservations(enrollmentObservationPath(storeDir), enroll.Observations{"session": { + Path: nativePath, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), StableSinceUnixNano: time.Now().Add(-time.Hour).UnixNano(), + }}); err != nil { + t.Fatal(err) + } + oldRunner := runEnrollmentCommand + defer func() { runEnrollmentCommand = oldRunner }() + runEnrollmentCommand = func(context.Context, []string) error { + t.Fatal("invalid rollout reached an enrollment mutation command") + return nil + } + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "enroll", "apply", "--codex-home", home, "--store", storeDir, "--mount", filepath.Join(home, "mount"), + "--canonical-namespace", "--native-root", filepath.Join(home, "fold-native"), "--enrollment-canary", "--stable-for", "1ns", "--apply", + }) + if err := root.Execute(); err == nil || !strings.Contains(err.Error(), "not valid UTF-8") { + t.Fatalf("enrollment error = %v", err) + } + states, err := vfs.DiscoverSessionStates(storeDir) + if err != nil || len(states) != 0 { + t.Fatalf("invalid enrollment created managed state: states=%#v err=%v", states, err) + } +} + +func TestPeriodicEnrollmentLoopRunsSerialCyclesAndStopsWithContext(t *testing.T) { + oldRunner := runServiceEnrollmentCycle + defer func() { runServiceEnrollmentCycle = oldRunner }() + + started := make(chan struct{}, 2) + release := make(chan struct{}, 2) + flagErrors := make(chan error, 1) + runServiceEnrollmentCycle = func(ctx context.Context, flags enrollmentFlags) (FSEnrollmentApplyResult, error) { + if flags.batchSize != 3 || flags.stableFor != 2*time.Hour || !flags.canonicalNamespace { + select { + case flagErrors <- fmt.Errorf("unexpected enrollment flags: %#v", flags): + default: + } + } + started <- struct{}{} + select { + case <-ctx.Done(): + return FSEnrollmentApplyResult{}, ctx.Err() + case <-release: + return FSEnrollmentApplyResult{}, nil + } + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + runPeriodicEnrollment(ctx, enrollmentFlags{batchSize: 3, stableFor: 2 * time.Hour, canonicalNamespace: true}, time.Millisecond, nil) + close(done) + }() + + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("periodic enrollment did not run") + } + select { + case <-started: + t.Fatal("periodic enrollment overlapped a running cycle") + case <-time.After(10 * time.Millisecond): + } + release <- struct{}{} + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("periodic enrollment did not schedule the next cycle") + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("periodic enrollment did not stop with its context") + } + select { + case err := <-flagErrors: + t.Fatal(err) + default: + } +} + +func TestEnrollmentCycleInPreviewRecordsObservationsWithoutMutating(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + allowEnrollmentWriterProbe(t) + oldProbe := mountHealthProbe + mountHealthProbe = func(string) error { return nil } + t.Cleanup(func() { mountHealthProbe = oldProbe }) + oldRunner := runEnrollmentCommand + defer func() { runEnrollmentCommand = oldRunner }() + runEnrollmentCommand = func(context.Context, []string) error { + t.Fatal("preview enrollment cycle attempted a mutation") + return nil + } + + result, err := runEnrollmentCycle(context.Background(), enrollmentFlags{ + codexHome: home, storeDir: storeDir, mountPoint: filepath.Join(home, "mount"), + nativeRoot: filepath.Join(home, "fold-native"), canonicalNamespace: true, + stableFor: time.Nanosecond, batchSize: 1, + }) + if err != nil { + t.Fatalf("runEnrollmentCycle: %v", err) + } + if result.Apply.Applied != 0 || len(result.Plan.Selected) != 0 { + t.Fatalf("preview cycle selected or applied sessions: %#v", result) + } + observations, err := enroll.LoadObservations(enrollmentObservationPath(storeDir)) + if err != nil || len(observations) != 1 { + t.Fatalf("preview cycle did not persist observations: observations=%#v err=%v", observations, err) + } +} + +func allowEnrollmentWriterProbe(t *testing.T) { + t.Helper() + oldProbe := enrollmentWriterProbe + enrollmentWriterProbe = func(context.Context, []codex.Session) (map[string]bool, error) { + return map[string]bool{}, nil + } + t.Cleanup(func() { enrollmentWriterProbe = oldProbe }) +} + +func TestEnrollmentStorageHealthAllowsBootstrapButRejectsBrokenCommittedPack(t *testing.T) { + storeDir := t.TempDir() + if err := requireEnrollmentStorageHealth(context.Background(), storeDir); err != nil { + t.Fatalf("empty enrollment store should be a valid bootstrap state: %v", err) + } + if err := os.MkdirAll(filepath.Join(storeDir, "packs"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(storeDir, "packs", "CURRENT"), []byte("missing-generation\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := requireEnrollmentStorageHealth(context.Background(), storeDir); err == nil { + t.Fatal("a broken committed pack generation passed enrollment health") + } +} + +func TestFSMigrateApplyFailsClosedWithoutMountedTarget(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", filepath.Join(home, "missing-mount"), "--apply"}) + if err := root.Execute(); err == nil { + t.Fatal("fs migrate --apply should fail without a live mounted target") + } + sessions, _ := codex.LoadSessions(home) + if sessions[0].RolloutPath != nativePath { + t.Fatalf("failed apply changed route to %q", sessions[0].RolloutPath) + } +} + +func TestFSMigrateApplyRetiresManagedStateWhenMountedTargetNeverAppears(t *testing.T) { + allowFixtureMount(t) + home, storeDir, nativePath := fsFixture(t, true) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + mount := filepath.Join(home, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + original, err := os.ReadFile(nativePath) + if err != nil { + t.Fatal(err) + } + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "migrate", "session", "--codex-home", home, "--store", storeDir, + "--mount", mount, "--mount-wait", "50ms", "--cli", cliPath, + "--desktop-app", "none", "--apply", + }) + if err := root.Execute(); err == nil { + t.Fatal("fs migrate --apply should fail when the mounted target never appears") + } + sessions, err := codex.LoadSessions(home) + if err != nil || sessions[0].RolloutPath != nativePath { + t.Fatalf("failed apply changed route: sessions=%#v err=%v", sessions, err) + } + got, err := os.ReadFile(nativePath) + if err != nil || !bytes.Equal(got, original) { + t.Fatalf("failed apply changed source: got=%q err=%v", got, err) + } + states, err := vfs.DiscoverSessionStates(storeDir) + if err != nil || len(states) != 0 { + t.Fatalf("failed apply left managed state: states=%#v err=%v", states, err) + } +} + +func TestFSMigrateApplyRejectsPlainDirectoryThatOnlyLooksLikeMount(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + mount := filepath.Join(home, "plain-directory") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(nativePath) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(mount, "session.jsonl"), data, 0o600); err != nil { + t.Fatal(err) + } + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", mount, "--cli", cliPath, "--desktop-app", "none", "--apply"}) + if err := root.Execute(); err == nil { + t.Fatal("plain directory should not satisfy the FUSE mount health gate") + } + sessions, _ := codex.LoadSessions(home) + if sessions[0].RolloutPath != nativePath { + t.Fatalf("failed mount gate changed route to %q", sessions[0].RolloutPath) + } +} + +func TestCompatibilityCanaryRequiresIsolatedCanonicalHomeAndSkippedClients(t *testing.T) { + defaultHome := filepath.Join(t.TempDir(), ".codex") + isolatedHome := filepath.Join(t.TempDir(), "isolated") + isolatedStore := filepath.Join(isolatedHome, "fold-store") + skipped := compatibilityFlags{cliPath: "none", desktopPath: "none"} + if err := validateCompatibilityCanary(isolatedHome, defaultHome, isolatedStore, true, skipped); err != nil { + t.Fatalf("isolated canonical canary was rejected: %v", err) + } + for _, test := range []struct { + name string + home string + store string + canonical bool + flags compatibilityFlags + }{ + {name: "real home", home: defaultHome, store: filepath.Join(defaultHome, "fold-store"), canonical: true, flags: skipped}, + {name: "external store", home: isolatedHome, store: filepath.Join(t.TempDir(), "store"), canonical: true, flags: skipped}, + {name: "flat mount", home: isolatedHome, store: isolatedStore, canonical: false, flags: skipped}, + {name: "live cli", home: isolatedHome, store: isolatedStore, canonical: true, flags: compatibilityFlags{cliPath: "codex", desktopPath: "none"}}, + {name: "live desktop", home: isolatedHome, store: isolatedStore, canonical: true, flags: compatibilityFlags{cliPath: "none", desktopPath: "/Applications/ChatGPT.app"}}, + } { + t.Run(test.name, func(t *testing.T) { + if err := validateCompatibilityCanary(test.home, defaultHome, test.store, test.canonical, test.flags); err == nil { + t.Fatal("unsafe compatibility canary configuration was accepted") + } + }) + } +} + +func TestFSStatusDoesNotClaimTransparentReadiness(t *testing.T) { + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "status", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("fs status: %v", err) + } + if bytes.Contains(output.Bytes(), []byte("production-ready")) || bytes.Contains(output.Bytes(), []byte("platform-canary")) { + t.Fatalf("status overclaimed readiness: %s", output.String()) + } + var status fsctl.Status + if err := json.Unmarshal(output.Bytes(), &status); err != nil || status.Capability != fsctl.FSEnginePreview { + t.Fatalf("status did not report the verified engine preview: %#v err=%v", status, err) + } +} + +func TestFSCompatibilityApprovesOnlyExactInstalledClientContract(t *testing.T) { + _, storeDir, _ := fsFixture(t, true) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "compatibility", "--store", storeDir, "--cli", cliPath, "--desktop-app", "none", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("fs compatibility: %v", err) + } + var result FSCompatibilityResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil || !result.Evaluation.Approved || result.Evaluation.Quarantine { + t.Fatalf("unexpected compatibility result: %#v err=%v output=%s", result, err, output.String()) + } +} + +func TestFSMigrateApplyInitializesManagedStateAndRoutesVerifiedTarget(t *testing.T) { + allowFixtureMount(t) + home, storeDir, nativePath := fsFixture(t, true) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + mount := filepath.Join(home, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(mount, "session.jsonl") + data, err := os.ReadFile(nativePath) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, data, 0o600); err != nil { + t.Fatal(err) + } + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", mount, "--cli", cliPath, "--desktop-app", "none", "--apply"}) + if err := root.Execute(); err != nil { + t.Fatalf("fs migrate --apply: %v", err) + } + sessions, err := codex.LoadSessions(home) + if err != nil || sessions[0].RolloutPath != target { + t.Fatalf("route not updated: sessions=%#v err=%v", sessions, err) + } + states, err := vfs.DiscoverSessionStates(storeDir) + if err != nil || len(states) != 1 || states[0].NativeSnapshot.Path != nativePath { + t.Fatalf("managed state missing: %#v err=%v", states, err) + } +} + +func TestFSMigrateCanonicalKeepsCodexRouteAndHidesRetainedSnapshot(t *testing.T) { + allowFixtureMount(t) + home := t.TempDir() + storeDir := filepath.Join(home, "fold-store") + route := filepath.Join(home, "archived_sessions", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(route), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"type\":\"session_meta\"}\n{\"canonical\":true}\n") + if err := os.WriteFile(route, source, 0o600); err != nil { + t.Fatal(err) + } + writeStateFixture(t, home, route) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set archived = 1, id = 'session' where id = 'fixture'`); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + if _, err := fold.Fold(context.Background(), fold.Session{ID: "session", RolloutPath: route, Archived: true}, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { + t.Fatal(err) + } + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filepath.Base(route)) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(route, nativePath); err != nil { + t.Fatal(err) + } + mount := filepath.Join(home, "fold-fs") + target := filepath.Join(mount, "archived_sessions", filepath.Base(route)) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + acknowledged := make(chan error, 1) + go func() { + statePath := filepath.Join(storeDir, "fs", "sessions", "session", "state.json") + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + state, err := vfs.LoadSessionState(statePath) + if err == nil { + if err := writeMountAcknowledgement(storeDir, "session", state.Generation, "/archived_sessions/"+filepath.Base(route)); err != nil { + acknowledged <- err + return + } + time.Sleep(100 * time.Millisecond) + if _, err := os.Stat(nativePath); err != nil { + acknowledged <- errors.New("canonical source was hidden before mounted target verification") + return + } + if err := os.MkdirAll(filepath.Dir(target), 0o700); err == nil { + err = os.WriteFile(target, source, 0o600) + } + acknowledged <- err + return + } + time.Sleep(10 * time.Millisecond) + } + acknowledged <- errors.New("managed state was not created") + }() + executeFS(t, []string{ + "fs", "migrate", "session", "--apply", "--canonical-namespace", + "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, + "--cli", cliPath, "--desktop-app", "none", "--mount-wait", "500ms", + }) + if err := <-acknowledged; err != nil { + t.Fatal(err) + } + sessions, err := codex.LoadSessions(home) + if err != nil || len(sessions) != 1 || filepath.Clean(sessions[0].RolloutPath) != filepath.Clean(route) { + t.Fatalf("canonical migration changed Codex route: sessions=%#v err=%v", sessions, err) + } + states, err := vfs.DiscoverSessionStates(storeDir) + retainedPath := filepath.Join(storeDir, "fs", "snapshots", "session", "native.jsonl") + if err != nil || len(states) != 1 || filepath.Clean(states[0].NativeSnapshot.Path) != filepath.Clean(retainedPath) { + t.Fatalf("canonical native snapshot = %#v err=%v", states, err) + } + if _, err := os.Stat(nativePath); !os.IsNotExist(err) { + t.Fatalf("canonical source remained visible after migration: %v", err) + } + if got, err := os.ReadFile(retainedPath); err != nil || !bytes.Equal(got, source) { + t.Fatalf("hidden retained snapshot = %q err=%v", got, err) + } +} + +func TestRollbackCanonicalMigrationRestoresNativeBeforeRetiringManagedState(t *testing.T) { + root := t.TempDir() + store := filepath.Join(root, "store") + stateDirectory := filepath.Join(store, "fs", "sessions", "session") + if err := os.MkdirAll(stateDirectory, 0o700); err != nil { + t.Fatal(err) + } + sourcePath := filepath.Join(root, "native", "archived_sessions", "rollout.jsonl") + retainedPath := filepath.Join(store, "fs", "snapshots", "session", "native.jsonl") + content := []byte("{\"restored\":true}\n") + if err := os.MkdirAll(filepath.Dir(retainedPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(retainedPath, content, 0o600); err != nil { + t.Fatal(err) + } + cause := errors.New("force migration rollback") + if err := rollbackCanonicalMigration(store, "session", sourcePath, retainedPath, cause); !errors.Is(err, cause) { + t.Fatalf("rollback error = %v, want original cause", err) + } + if got, err := os.ReadFile(sourcePath); err != nil || !bytes.Equal(got, content) { + t.Fatalf("restored native source = %q err=%v", got, err) + } + if _, err := os.Stat(stateDirectory); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("managed state was not retired after native restoration: %v", err) + } + retired, err := filepath.Glob(filepath.Join(store, "fs", "retired", "session-*")) + if err != nil || len(retired) != 1 { + t.Fatalf("retired states = %v err=%v", retired, err) + } +} + +func TestRollbackCanonicalMigrationKeepsManagedStateWhenNativeRestoreConflicts(t *testing.T) { + root := t.TempDir() + store := filepath.Join(root, "store") + stateDirectory := filepath.Join(store, "fs", "sessions", "session") + if err := os.MkdirAll(stateDirectory, 0o700); err != nil { + t.Fatal(err) + } + sourcePath := filepath.Join(root, "native", "archived_sessions", "rollout.jsonl") + retainedPath := filepath.Join(store, "fs", "snapshots", "session", "native.jsonl") + if err := os.MkdirAll(filepath.Dir(sourcePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(retainedPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(sourcePath, []byte("{\"current\":true}\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(retainedPath, []byte("{\"retained\":true}\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := rollbackCanonicalMigration(store, "session", sourcePath, retainedPath, errors.New("force migration rollback")); err == nil { + t.Fatal("conflicting native restoration unexpectedly succeeded") + } + if _, err := os.Stat(stateDirectory); err != nil { + t.Fatalf("managed state was retired after native restore conflict: %v", err) + } + if _, err := os.Stat(retainedPath); err != nil { + t.Fatalf("retained snapshot was removed after native restore conflict: %v", err) + } + retired, err := filepath.Glob(filepath.Join(store, "fs", "retired", "session-*")) + if err != nil || len(retired) != 0 { + t.Fatalf("retired states after conflict = %v err=%v", retired, err) + } +} + +func TestFSRecoverRetiresInterruptedCanonicalMigrationWithUnchangedSource(t *testing.T) { + fixture := interruptedCanonicalMigrationFixture(t) + _ = fixture.resolver.Close() + + executeFS(t, []string{"fs", "recover", "session", "--apply", "--codex-home", fixture.home, "--store", fixture.store}) + if _, err := os.Stat(filepath.Join(fixture.store, "fs", "sessions", "session")); !os.IsNotExist(err) { + t.Fatalf("interrupted migration state remained: %v", err) + } + got, err := os.ReadFile(fixture.nativePath) + if err != nil || !bytes.Equal(got, fixture.source) { + t.Fatalf("recovery changed canonical source: got=%q err=%v", got, err) + } +} + +func TestFSRecoverLeavesLiveCanonicalMigrationManaged(t *testing.T) { + fixture := interruptedCanonicalMigrationFixture(t) + defer fixture.resolver.Close() + writer, err := fixture.managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + defer writer.Close() + + state := fixture.managed.State() + recovered, err := recoverInterruptedCanonicalMigration(fixture.home, fixture.store, fixture.nativeRoot, state) + if err != nil || recovered { + t.Fatalf("live migration recovery = %t, %v", recovered, err) + } + if _, err := managedState(fixture.store, "session"); err != nil { + t.Fatalf("live migration state was retired: %v", err) + } + if got, err := os.ReadFile(fixture.nativePath); err != nil || !bytes.Equal(got, fixture.source) { + t.Fatalf("live migration source changed: got=%q err=%v", got, err) + } + if got, err := os.ReadFile(state.NativeSnapshot.Path); err != nil || !bytes.Equal(got, fixture.source) { + t.Fatalf("live migration snapshot changed: got=%q err=%v", got, err) + } +} + +func TestFSRecoverLeavesPendingCanonicalRollbackManaged(t *testing.T) { + fixture := interruptedCanonicalMigrationFixture(t) + defer fixture.resolver.Close() + tail := []byte("{\"pending_rollback\":true}\n") + writer, err := fixture.managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + if _, err := writer.Append(context.Background(), tail); err != nil { + _ = writer.Close() + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + _ = writer.Close() + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + target, err := fixture.managed.MaterializeCurrent(context.Background(), fixture.nativePath, true) + if err != nil { + t.Fatal(err) + } + state := fixture.managed.State() + if _, err := createRetirementRequest(fixture.store, "session", state.Generation, "/archived_sessions/rollout-session.jsonl", target); err != nil { + t.Fatal(err) + } + recovered, err := recoverInterruptedCanonicalMigration(fixture.home, fixture.store, fixture.nativeRoot, state) + if err != nil || recovered { + t.Fatalf("pending rollback recovery = %t, %v", recovered, err) + } + if _, err := managedState(fixture.store, "session"); err != nil { + t.Fatalf("pending rollback state was retired: %v", err) + } + if err := clearRetirementControl(filepath.Join(fixture.store, "fs", "sessions", "session")); err != nil { + t.Fatal(err) + } + recovered, err = recoverInterruptedCanonicalMigration(fixture.home, fixture.store, fixture.nativeRoot, state) + if err != nil || recovered { + t.Fatalf("pre-request rollback recovery = %t, %v", recovered, err) + } + want := append(append([]byte(nil), fixture.source...), tail...) + if got, err := os.ReadFile(fixture.nativePath); err != nil || !bytes.Equal(got, want) { + t.Fatalf("pending rollback target changed: got=%q err=%v", got, err) + } +} + +func TestCreateRetirementRequestResumesOnlyExactPendingRequest(t *testing.T) { + storeDir := t.TempDir() + directory := filepath.Join(storeDir, "fs", "sessions", "session") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + target := vfs.NativeFile{Path: filepath.Join(t.TempDir(), "current.jsonl"), Bytes: 42, SHA256: strings.Repeat("a", 64)} + first, err := createRetirementRequest(storeDir, "session", 3, "/archived_sessions/rollout.jsonl", target) + if err != nil { + t.Fatal(err) + } + resumed, err := createRetirementRequest(storeDir, "session", 3, "/archived_sessions/rollout.jsonl", target) + if err != nil { + t.Fatalf("resume exact retirement request: %v", err) + } + if resumed != first { + t.Fatalf("resumed request changed token or metadata: first=%#v resumed=%#v", first, resumed) + } + target.SHA256 = strings.Repeat("b", 64) + if _, err := createRetirementRequest(storeDir, "session", 3, "/archived_sessions/rollout.jsonl", target); err == nil { + t.Fatal("mismatched pending retirement request should fail closed") + } +} + +type interruptedCanonicalFixture struct { + home string + store string + nativeRoot string + nativePath string + source []byte + managed *vfs.Session + resolver *pack.Resolver +} + +func interruptedCanonicalMigrationFixture(t *testing.T) interruptedCanonicalFixture { + t.Helper() + home := t.TempDir() + storeDir := filepath.Join(home, "fold-store") + route := filepath.Join(home, "archived_sessions", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(route), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"type\":\"session_meta\"}\n{\"interrupted_migration\":true}\n") + if err := os.WriteFile(route, source, 0o600); err != nil { + t.Fatal(err) + } + writeStateFixture(t, home, route) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set archived = 1, id = 'session' where id = 'fixture'`); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + if _, err := fold.Fold(context.Background(), fold.Session{ID: "session", RolloutPath: route, Archived: true}, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { + t.Fatal(err) + } + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filepath.Base(route)) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(route, nativePath); err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + t.Fatal(err) + } + retained, err := retainCanonicalSnapshot(context.Background(), storeDir, "session", native, nil) + if err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: retained, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + return interruptedCanonicalFixture{ + home: home, store: storeDir, nativeRoot: nativeRoot, nativePath: nativePath, + source: source, managed: managed, resolver: resolver, + } +} + +func TestFSMigrateCanonicalReservesWriterDuringCutover(t *testing.T) { + allowFixtureMount(t) + home := t.TempDir() + storeDir := filepath.Join(home, "fold-store") + route := filepath.Join(home, "archived_sessions", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(route), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"type\":\"session_meta\"}\n{\"canonical\":true}\n") + if err := os.WriteFile(route, source, 0o600); err != nil { + t.Fatal(err) + } + writeStateFixture(t, home, route) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set archived = 1, id = 'session' where id = 'fixture'`); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + if _, err := fold.Fold(context.Background(), fold.Session{ID: "session", RolloutPath: route, Archived: true}, fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { + t.Fatal(err) + } + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filepath.Base(route)) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(route, nativePath); err != nil { + t.Fatal(err) + } + mount := filepath.Join(home, "fold-fs") + target := filepath.Join(mount, "archived_sessions", filepath.Base(route)) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + writerAttempt := make(chan error, 1) + releaseWriter := make(chan struct{}) + go func() { + statePath := filepath.Join(storeDir, "fs", "sessions", "session", "state.json") + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + state, stateErr := vfs.LoadSessionState(statePath) + if stateErr == nil { + if ackErr := writeMountAcknowledgement(storeDir, "session", state.Generation, "/archived_sessions/"+filepath.Base(route)); ackErr != nil { + writerAttempt <- ackErr + return + } + managed, resolver, openErr := openManagedSession(context.Background(), storeDir, state) + if openErr != nil { + writerAttempt <- openErr + return + } + writer, writerErr := managed.OpenWriter() + if err := os.MkdirAll(filepath.Dir(target), 0o700); err == nil { + err = os.WriteFile(target, source, 0o600) + } + writerAttempt <- writerErr + if writer != nil { + <-releaseWriter + _ = writer.Close() + } + _ = resolver.Close() + return + } + time.Sleep(5 * time.Millisecond) + } + writerAttempt <- errors.New("managed state was not created") + }() + + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "migrate", "session", "--apply", "--canonical-namespace", + "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, + "--cli", cliPath, "--desktop-app", "none", "--mount-wait", "500ms", + }) + migrateErr := root.Execute() + writerErr := <-writerAttempt + close(releaseWriter) + if migrateErr != nil { + t.Fatalf("canonical migration failed: %v", migrateErr) + } + if !errors.Is(writerErr, vfs.ErrWriterBusy) { + t.Fatalf("concurrent writer error = %v, want %v", writerErr, vfs.ErrWriterBusy) + } + if _, err := managedState(storeDir, "session"); err != nil { + t.Fatalf("canonical migration did not retain managed state: %v", err) + } +} + +func TestFSRollbackUsesLatestVisibleBytesAfterVirtualAppend(t *testing.T) { + allowFixtureMount(t) + home, storeDir, nativePath := fsFixture(t, true) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + mount := filepath.Join(home, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(mount, "session.jsonl") + original, _ := os.ReadFile(nativePath) + if err := os.WriteFile(target, original, 0o600); err != nil { + t.Fatal(err) + } + executeFS(t, []string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", mount, "--cli", cliPath, "--desktop-app", "none", "--apply"}) + state, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + managed, resolver, err := openManagedSession(context.Background(), storeDir, state) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + tail := []byte("{\"appended\":true}\n") + if _, err := writer.Append(context.Background(), tail); err != nil { + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + t.Fatal(err) + } + _ = writer.Close() + _ = resolver.Close() + executeFS(t, []string{"fs", "rollback", "session", "--codex-home", home, "--store", storeDir, "--apply"}) + sessions, err := codex.LoadSessions(home) + if err != nil { + t.Fatal(err) + } + fallback, err := os.ReadFile(sessions[0].RolloutPath) + if err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), original...), tail...) + if !bytes.Equal(fallback, want) { + t.Fatalf("rollback used stale bytes: got=%q want=%q", fallback, want) + } + if _, err := managedState(storeDir, "session"); err == nil { + t.Fatal("rollback left the session managed") + } +} + +func TestFSRollbackRejectsActiveWriter(t *testing.T) { + home, storeDir, originalPath := fsFixture(t, true) + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + defer resolver.Close() + native, err := hashPath(originalPath) + if err != nil { + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: native, + }) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + defer writer.Close() + + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "rollback", "session", "--codex-home", home, "--store", storeDir, "--apply"}) + err = root.Execute() + if err == nil || !strings.Contains(err.Error(), "active writer") { + t.Fatalf("rollback error = %v, want active writer rejection", err) + } + if _, err := managedState(storeDir, "session"); err != nil { + t.Fatalf("active-writer rejection retired managed state: %v", err) + } + sessions, err := codex.LoadSessions(home) + if err != nil || len(sessions) != 1 || filepath.Clean(sessions[0].RolloutPath) != filepath.Clean(originalPath) { + t.Fatalf("active-writer rejection changed route: sessions=%#v err=%v", sessions, err) + } +} + +func TestFSRollbackCanonicalRetiresManagedStateAndKeepsRoute(t *testing.T) { + allowFixtureMount(t) + home, storeDir, originalPath := fsFixture(t, true) + original, err := os.ReadFile(originalPath) + if err != nil { + t.Fatal(err) + } + filename := "rollout-session.jsonl" + snapshotRoute := filepath.Join(home, "archived_sessions", filename) + route := filepath.Join(home, "sessions", "2026", "07", "12", filename) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set rollout_path = ? where id = 'session'`, route); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filename) + targetNativePath := filepath.Join(nativeRoot, "sessions", "2026", "07", "12", filename) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(originalPath, nativePath); err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: native, + }) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + tail := []byte("{\"canonical_rollback\":true}\n") + if _, err := writer.Append(context.Background(), tail); err != nil { + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + t.Fatal(err) + } + _ = writer.Close() + _ = resolver.Close() + mount := filepath.Join(home, "fold-fs") + mountedTarget := filepath.Join(mount, "sessions", "2026", "07", "12", filename) + if err := os.MkdirAll(filepath.Dir(mountedTarget), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(mountedTarget, original, 0o600); err != nil { + t.Fatal(err) + } + stateDirectory := filepath.Join(storeDir, "fs", "sessions", "session") + copyDone := emulateCanonicalRetirement(storeDir, "session", mountedTarget, targetNativePath) + executeFS(t, []string{ + "fs", "rollback", "session", "--apply", "--canonical-namespace", + "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, + }) + if err := <-copyDone; err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), original...), tail...) + if got, err := os.ReadFile(targetNativePath); err != nil || !bytes.Equal(got, want) { + t.Fatalf("canonical rollback bytes = %q err=%v", got, err) + } + if _, err := os.Stat(nativePath); !os.IsNotExist(err) { + t.Fatalf("retained snapshot remained visible at %s: %v", snapshotRoute, err) + } + sessions, err := codex.LoadSessions(home) + if err != nil || len(sessions) != 1 || filepath.Clean(sessions[0].RolloutPath) != filepath.Clean(route) { + t.Fatalf("canonical rollback changed route: sessions=%#v err=%v", sessions, err) + } + if _, err := os.Stat(stateDirectory); !os.IsNotExist(err) { + t.Fatalf("managed state remained after canonical rollback: %v", err) + } + retired, err := filepath.Glob(filepath.Join(storeDir, "fs", "retired", "session-*")) + if err != nil || len(retired) != 1 { + t.Fatalf("retired state = %#v err=%v", retired, err) + } + retained, err := filepath.Glob(filepath.Join(retired[0], "retained-native", "archived_sessions", filename)) + if err != nil || len(retained) != 1 { + t.Fatalf("retired native snapshot = %#v err=%v", retained, err) + } +} + +func TestFSRollbackCanonicalRetirementUsesRecoveredGeneration(t *testing.T) { + allowFixtureMount(t) + home, storeDir, originalPath := fsFixture(t, true) + original, err := os.ReadFile(originalPath) + if err != nil { + t.Fatal(err) + } + filename := "rollout-session.jsonl" + route := filepath.Join(home, "sessions", "2026", "07", "12", filename) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set rollout_path = ? where id = 'session'`, route); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filename) + targetNativePath := filepath.Join(nativeRoot, "sessions", "2026", "07", "12", filename) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(originalPath, nativePath); err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + recoveryStop := errors.New("stop after COW file publish") + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: native, + BeforeCOWPhase: func(phase string) error { + if phase == "after-file-publish" { + return recoveryStop + } + return nil + }, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if _, err := writer.WriteAt(context.Background(), []byte("X"), 0); !errors.Is(err, recoveryStop) { + _ = writer.Close() + _ = resolver.Close() + t.Fatalf("WriteAt error = %v, want %v", err, recoveryStop) + } + if err := writer.Close(); err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if err := resolver.Close(); err != nil { + t.Fatal(err) + } + staleState, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + if staleState.Generation != 1 { + t.Fatalf("pre-recovery generation = %d, want 1", staleState.Generation) + } + + mount := filepath.Join(home, "fold-fs") + mountedTarget := filepath.Join(mount, "sessions", "2026", "07", "12", filename) + if err := os.MkdirAll(filepath.Dir(mountedTarget), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(mountedTarget, original, 0o600); err != nil { + t.Fatal(err) + } + retirementDone := emulateCanonicalRetirementGeneration(t, storeDir, "session", mountedTarget, targetNativePath, 2) + + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "rollback", "session", "--apply", "--canonical-namespace", + "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, + "--mount-wait", "500ms", + }) + rollbackErr := root.Execute() + if err := <-retirementDone; err != nil { + t.Fatal(err) + } + if rollbackErr != nil { + t.Fatalf("rollback with recovered session state: %v", rollbackErr) + } + if _, err := os.Stat(filepath.Join(storeDir, "fs", "sessions", "session")); !os.IsNotExist(err) { + t.Fatalf("managed state remained after canonical rollback: %v", err) + } + if got, err := os.ReadFile(targetNativePath); err != nil || !bytes.Equal(got, original) { + t.Fatalf("canonical rollback bytes = %q err=%v", got, err) + } +} + +func TestCanonicalRetirementColdLoadRestoresManagedFallbackBeforeNativePreference(t *testing.T) { + home, storeDir, originalPath := fsFixture(t, true) + original, err := os.ReadFile(originalPath) + if err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + retainedPath := filepath.Join(storeDir, "fs", "snapshots", "session", "native.jsonl") + if err := os.MkdirAll(filepath.Dir(retainedPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(retainedPath, original, 0o600); err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + retained, err := hashPath(retainedPath) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: retained, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if err := resolver.Close(); err != nil { + t.Fatal(err) + } + + route := "/archived_sessions/rollout-session.jsonl" + nativeRoot := filepath.Join(home, "fold-native") + nativeTargetPath := filepath.Join(nativeRoot, "archived_sessions", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(nativeTargetPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(nativeTargetPath, original, 0o600); err != nil { + t.Fatal(err) + } + nativeTarget, err := hashPath(nativeTargetPath) + if err != nil { + t.Fatal(err) + } + if _, err := createRetirementRequest(storeDir, "session", managed.State().Generation, route, nativeTarget); err != nil { + t.Fatal(err) + } + + filesystem := mountfs.NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + known := make(map[string]uint64) + knownRoutes := make(map[string]string) + knownPacks := make(map[string]string) + currentPack, err := pack.CurrentGeneration(storeDir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = filesystem.CloseSessions() }) + opens := 0 + openState := func(state vfs.SessionState) (*vfs.Session, *pack.Resolver, error) { + opens++ + current, nextResolver, err := openManagedSession(context.Background(), storeDir, state) + return current, nextResolver, err + } + state, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + for attempt := 0; attempt < 2; attempt++ { + handled, err := syncCanonicalRetirement(storeDir, home, nativeRoot, filesystem, state, route, true, known, knownRoutes, knownPacks, currentPack, openState) + if err != nil || !handled { + t.Fatalf("sync retirement attempt %d: handled=%t err=%v", attempt, handled, err) + } + } + if opens != 1 || known["session"] != state.Generation || knownRoutes["session"] != route { + t.Fatalf("cold load state: opens=%d known=%#v routes=%#v", opens, known, knownRoutes) + } + acknowledgement, err := os.ReadFile(filepath.Join(storeDir, "fs", "sessions", "session", retirementAcknowledgementFilename)) + if err != nil || !bytes.Contains(acknowledgement, []byte(`"token"`)) { + t.Fatalf("retirement acknowledgement = %q err=%v", acknowledgement, err) + } + if got := readMountedFilesystemFile(t, filesystem, route, len(original)); !bytes.Equal(got, original) { + t.Fatalf("native-preferred bytes = %q", got) + } + if err := os.Remove(nativeTargetPath); err != nil { + t.Fatal(err) + } + if got := readMountedFilesystemFile(t, filesystem, route, len(original)); !bytes.Equal(got, original) { + t.Fatalf("managed fallback bytes = %q", got) + } +} + +func TestCanonicalRetirementDaemonRestartRejectsStaleNativeAcknowledgement(t *testing.T) { + home, storeDir, originalPath := fsFixture(t, true) + original, err := os.ReadFile(originalPath) + if err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + retainedPath := filepath.Join(storeDir, "fs", "snapshots", "session", "native.jsonl") + if err := os.MkdirAll(filepath.Dir(retainedPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(retainedPath, original, 0o600); err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + retained, err := hashPath(retainedPath) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: retained, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + tail := []byte("{\"pending_retirement\":true}\n") + writer, err := managed.OpenWriter() + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if _, err := writer.Append(context.Background(), tail); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + if err := writer.Close(); err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if err := resolver.Close(); err != nil { + t.Fatal(err) + } + current := append(append([]byte(nil), original...), tail...) + + route := "/archived_sessions/rollout-session.jsonl" + nativeRoot := filepath.Join(home, "fold-native") + nativeTargetPath := filepath.Join(nativeRoot, "archived_sessions", "rollout-session.jsonl") + if err := os.MkdirAll(filepath.Dir(nativeTargetPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(nativeTargetPath, current, 0o600); err != nil { + t.Fatal(err) + } + nativeTarget, err := hashPath(nativeTargetPath) + if err != nil { + t.Fatal(err) + } + state, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + retirement, err := createRetirementRequest(storeDir, "session", state.Generation, route, nativeTarget) + if err != nil { + t.Fatal(err) + } + + opens := 0 + openState := func(state vfs.SessionState) (*vfs.Session, *pack.Resolver, error) { + opens++ + current, nextResolver, err := openManagedSession(context.Background(), storeDir, state) + return current, nextResolver, err + } + + firstDaemon := mountfs.NewCanonical() + firstDaemon.SetNativeRoot(nativeRoot) + t.Cleanup(func() { _ = firstDaemon.CloseSessions() }) + firstKnown := make(map[string]uint64) + firstRoutes := make(map[string]string) + firstPacks := make(map[string]string) + currentPack, err := pack.CurrentGeneration(storeDir) + if err != nil { + t.Fatal(err) + } + handled, err := syncCanonicalRetirement(storeDir, home, nativeRoot, firstDaemon, state, route, true, firstKnown, firstRoutes, firstPacks, currentPack, openState) + if err != nil || !handled { + t.Fatalf("initial retirement sync: handled=%t err=%v", handled, err) + } + if got := readMountedFilesystemFile(t, firstDaemon, route, len(current)); !bytes.Equal(got, current) { + t.Fatalf("native-preferred bytes = %q, want %q", got, current) + } + + if err := os.Remove(nativeTargetPath); err != nil { + t.Fatal(err) + } + restartedDaemon := mountfs.NewCanonical() + restartedDaemon.SetNativeRoot(nativeRoot) + t.Cleanup(func() { _ = restartedDaemon.CloseSessions() }) + restartedKnown := make(map[string]uint64) + restartedRoutes := make(map[string]string) + restartedPacks := make(map[string]string) + handled, err = syncCanonicalRetirement(storeDir, home, nativeRoot, restartedDaemon, state, route, true, restartedKnown, restartedRoutes, restartedPacks, currentPack, openState) + if err != nil || !handled { + t.Fatalf("restart retirement sync: handled=%t err=%v", handled, err) + } + if opens != 2 { + t.Fatalf("daemon restarts opened managed state %d times, want 2", opens) + } + acknowledgement, err := os.ReadFile(filepath.Join(storeDir, "fs", "sessions", "session", retirementAcknowledgementFilename)) + if err != nil { + t.Fatal(err) + } + var acknowledged retirementControl + if err := json.Unmarshal(acknowledgement, &acknowledged); err != nil { + t.Fatal(err) + } + if acknowledged.Token != retirement.Token || acknowledged.Error == "" { + t.Fatalf("stale acknowledgement was not rejected: %#v", acknowledged) + } + if got := readMountedFilesystemFile(t, restartedDaemon, route, len(current)); !bytes.Equal(got, current) { + t.Fatalf("restart managed fallback bytes = %q, want %q", got, current) + } +} + +func TestFSRollbackCanonicalFailureWaitsForManagedRouteRestoration(t *testing.T) { + allowFixtureMount(t) + home, storeDir, originalPath := fsFixture(t, true) + original, err := os.ReadFile(originalPath) + if err != nil { + t.Fatal(err) + } + filename := "rollout-session.jsonl" + route := filepath.Join(home, "sessions", "2026", "07", "12", filename) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set rollout_path = ? where id = 'session'`, route); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filename) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(originalPath, nativePath); err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: native, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + tail := []byte("{\"rollback_failure\":true}\n") + if _, err := writer.Append(context.Background(), tail); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + _ = writer.Close() + _ = resolver.Close() + want := append(append([]byte(nil), original...), tail...) + + mount := filepath.Join(home, "fold-fs") + mountedTarget := filepath.Join(mount, "sessions", "2026", "07", "12", filename) + if err := os.MkdirAll(filepath.Dir(mountedTarget), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(mountedTarget, original, 0o600); err != nil { + t.Fatal(err) + } + stateDirectory := filepath.Join(storeDir, "fs", "sessions", "session") + retirementRequest := filepath.Join(stateDirectory, "retire.request.json") + canonicalRoute := "/sessions/2026/07/12/" + filename + restored := make(chan error, 1) + go func() { + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(stateDirectory); err != nil { + restored <- fmt.Errorf("managed state moved before retirement request: %w", err) + return + } + request, err := os.ReadFile(retirementRequest) + if err == nil { + var rejection retirementControl + if err := json.Unmarshal(request, &rejection); err != nil { + restored <- err + return + } + rejection.Error = "native rollback target is unavailable or changed" + if err := writeRetirementAcknowledgement(storeDir, "session", rejection); err != nil { + restored <- err + return + } + break + } + if !errors.Is(err, os.ErrNotExist) { + restored <- err + return + } + time.Sleep(5 * time.Millisecond) + } + for time.Now().Before(deadline) { + if _, err := os.Stat(stateDirectory); err != nil { + restored <- fmt.Errorf("managed state moved during retirement cancellation: %w", err) + return + } + if _, err := os.Stat(retirementRequest); errors.Is(err, os.ErrNotExist) { + state, stateErr := managedState(storeDir, "session") + if stateErr != nil { + restored <- stateErr + return + } + if state.Generation < 2 { + time.Sleep(5 * time.Millisecond) + continue + } + if err := os.WriteFile(mountedTarget, want, 0o600); err != nil { + restored <- err + return + } + restored <- writeMountAcknowledgement(storeDir, "session", state.Generation, canonicalRoute) + return + } + time.Sleep(5 * time.Millisecond) + } + restored <- errors.New("managed route was not restored") + }() + + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "rollback", "session", "--apply", "--canonical-namespace", + "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, + "--mount-wait", "200ms", + }) + err = root.Execute() + if err == nil || !strings.Contains(err.Error(), "retirement rejected") { + t.Fatalf("rollback error = %v, want retirement rejection", err) + } + select { + case restoreErr := <-restored: + if restoreErr != nil { + t.Fatal(restoreErr) + } + default: + t.Fatal("rollback returned before the managed route became readable again") + } + state, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + if state.Generation != 2 { + t.Fatalf("restored generation = %d, want 2", state.Generation) + } +} + +func TestFSRollbackCanonicalRetiresHiddenSnapshot(t *testing.T) { + allowFixtureMount(t) + home, storeDir, originalPath := fsFixture(t, true) + original, err := os.ReadFile(originalPath) + if err != nil { + t.Fatal(err) + } + filename := "rollout-session.jsonl" + route := filepath.Join(home, "sessions", "2026", "07", "12", filename) + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`update threads set rollout_path = ? where id = 'session'`, route); err != nil { + _ = db.Close() + t.Fatal(err) + } + _ = db.Close() + nativeRoot := filepath.Join(home, "fold-native") + nativePath := filepath.Join(nativeRoot, "archived_sessions", filename) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(originalPath, nativePath); err != nil { + t.Fatal(err) + } + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + hiddenPath := filepath.Join(storeDir, "fs", "snapshots", "session", "native.jsonl") + if err := os.MkdirAll(filepath.Dir(hiddenPath), 0o700); err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if err := os.Rename(nativePath, hiddenPath); err != nil { + _ = resolver.Close() + t.Fatal(err) + } + native.Path = hiddenPath + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: native, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + tail := []byte("{\"hidden_snapshot_rollback\":true}\n") + if _, err := writer.Append(context.Background(), tail); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + _ = writer.Close() + _ = resolver.Close() + mount := filepath.Join(home, "fold-fs") + mountedTarget := filepath.Join(mount, "sessions", "2026", "07", "12", filename) + targetNativePath := filepath.Join(nativeRoot, "sessions", "2026", "07", "12", filename) + if err := os.MkdirAll(filepath.Dir(mountedTarget), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(mountedTarget, original, 0o600); err != nil { + t.Fatal(err) + } + copyDone := emulateCanonicalRetirement(storeDir, "session", mountedTarget, targetNativePath) + // The mounted target is only used as the FUSE visibility probe. The + // canonical rollback writes the latest bytes to the retained native route. + executeFS(t, []string{ + "fs", "rollback", "session", "--apply", "--canonical-namespace", + "--codex-home", home, "--store", storeDir, "--mount", mount, "--native-root", nativeRoot, + }) + if err := <-copyDone; err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), original...), tail...) + retired, err := filepath.Glob(filepath.Join(storeDir, "fs", "retired", "session-*", "retained-native", "store-snapshot", "native.jsonl")) + if err != nil || len(retired) != 1 { + t.Fatalf("hidden snapshot retirement = %#v err=%v", retired, err) + } + if got, err := os.ReadFile(retired[0]); err != nil || !bytes.Equal(got, original) { + t.Fatalf("retired hidden snapshot bytes = %q err=%v", got, err) + } + if got, err := os.ReadFile(targetNativePath); err != nil || !bytes.Equal(got, want) { + t.Fatalf("canonical rollback bytes = %q err=%v", got, err) + } + if _, err := os.Stat(hiddenPath); !os.IsNotExist(err) { + t.Fatalf("hidden snapshot remained after retirement: %v", err) + } + if _, err := os.Stat(nativePath); !os.IsNotExist(err) { + t.Fatalf("legacy native snapshot unexpectedly restored: %v", err) + } +} + +func TestFSUpdatePreflightPreservesNewerNativeFallbackAfterRollback(t *testing.T) { + allowFixtureMount(t) + home, storeDir, nativePath := fsFixture(t, true) + cliPath := approvedCLIContract(t, storeDir, "1.2.3") + mount := filepath.Join(home, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(mount, "session.jsonl") + original, err := os.ReadFile(nativePath) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, original, 0o600); err != nil { + t.Fatal(err) + } + executeFS(t, []string{"fs", "migrate", "session", "--codex-home", home, "--store", storeDir, "--mount", mount, "--cli", cliPath, "--desktop-app", "none", "--apply"}) + + state, err := managedState(storeDir, "session") + if err != nil { + t.Fatal(err) + } + managed, resolver, err := openManagedSession(context.Background(), storeDir, state) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + managedTail := []byte("{\"managed_tail\":true}\n") + if _, err := writer.Append(context.Background(), managedTail); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + _ = writer.Close() + _ = resolver.Close() + + executeFS(t, []string{"fs", "rollback", "session", "--codex-home", home, "--store", storeDir, "--apply"}) + sessions, err := codex.LoadSessions(home) + if err != nil { + t.Fatal(err) + } + fallbackPath := sessions[0].RolloutPath + if filepath.Base(fallbackPath) != "fallback-current.jsonl" { + t.Fatalf("rollback did not use the generated fallback: %s", fallbackPath) + } + + nativeTail := []byte("{\"native_tail\":true}\n") + fallback, err := os.OpenFile(fallbackPath, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if _, err := fallback.Write(nativeTail); err != nil { + _ = fallback.Close() + t.Fatal(err) + } + if err := fallback.Sync(); err != nil { + _ = fallback.Close() + t.Fatal(err) + } + if err := fallback.Close(); err != nil { + t.Fatal(err) + } + want := append(append(append([]byte(nil), original...), managedTail...), nativeTail...) + beforeRoute := fallbackPath + + unknownCLI := fakeCLI(t, "9.9.9") + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "service", "update-preflight", "--codex-home", home, "--store", storeDir, "--cli", unknownCLI, "--desktop-app", "none", "--apply-quarantine", "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("update preflight: %v", err) + } + var result FSUpdatePreflightResult + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + t.Fatalf("decode preflight result: %v output=%s", err, output.String()) + } + if !result.Decision.Quarantine || result.Decision.RequiresNativeFallback || result.QuarantinedSessions != 0 { + t.Fatalf("unexpected fallback preflight result: %#v output=%s", result, output.String()) + } + sessions, err = codex.LoadSessions(home) + if err != nil { + t.Fatal(err) + } + if sessions[0].RolloutPath != beforeRoute { + t.Fatalf("preflight replaced newer native fallback: got=%s want=%s", sessions[0].RolloutPath, beforeRoute) + } + got, err := os.ReadFile(fallbackPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("preflight changed newer native fallback: got=%q want=%q", got, want) + } +} + +func TestFSCompactCommitsNewExactGeneration(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, Reader: resolver, NativeSnapshot: native}) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + if _, err := writer.Append(context.Background(), []byte("{\"tail\":2}\n")); err != nil { + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + t.Fatal(err) + } + _ = writer.Close() + before := filepath.Join(home, "before.jsonl") + expected, err := managed.MaterializeCurrent(context.Background(), before, false) + if err != nil { + t.Fatal(err) + } + _ = resolver.Close() + executeFS(t, []string{"fs", "compact", "session", "--codex-home", home, "--store", storeDir, "--apply"}) + state, err := managedState(storeDir, "session") + if err != nil || state.Generation != 2 || state.BaseSHA256 != expected.SHA256 { + t.Fatalf("unexpected compacted state: %#v err=%v", state, err) + } + reopened, nextResolver, err := openManagedSession(context.Background(), storeDir, state) + if err != nil { + t.Fatal(err) + } + after, err := reopened.MaterializeCurrent(context.Background(), filepath.Join(home, "after.jsonl"), false) + _ = nextResolver.Close() + if err != nil || after.Bytes != expected.Bytes || after.SHA256 != expected.SHA256 { + t.Fatalf("compacted bytes changed: before=%#v after=%#v err=%v", expected, after, err) + } +} + +func TestFSReadOnlyCommandsRunWithoutClaimingMountHealth(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + for _, args := range [][]string{ + {"fs", "doctor", "--codex-home", home, "--store", storeDir, "--json"}, + {"fs", "benchmark", "session", "--codex-home", home, "--store", storeDir, "--random-reads", "10", "--json"}, + {"fs", "serve", "--codex-home", home, "--store", storeDir, "--json"}, + } { + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs(args) + if err := root.Execute(); err != nil { + t.Fatalf("%v: %v", args, err) + } + if bytes.Contains(output.Bytes(), []byte("production-ready")) || bytes.Contains(output.Bytes(), []byte("platform-canary")) { + t.Fatalf("%v overclaimed readiness: %s", args, output.String()) + } + } + if !mountfs.Available() { + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fs", "serve", "--codex-home", home, "--store", storeDir, "--mount", filepath.Join(home, "mount"), "--apply"}) + if err := root.Execute(); err == nil { + t.Fatal("default build should not claim the FUSE prerequisite is available") + } + } + status, _ := fsctl.NewStatus(fsctl.StorageEngine, runtime.GOOS) + if status.Capability != fsctl.StorageEngine { + t.Fatalf("unexpected capability: %#v", status) + } +} + +func TestFSBenchmarkUsesManagedVisibleBytesAfterAppendAndCompact(t *testing.T) { + home, storeDir, nativePath := fsFixture(t, true) + manifest, err := fold.LoadManifest(storeDir, "session") + if err != nil { + t.Fatal(err) + } + resolver, err := pack.Open(storeDir, pack.OpenOptions{}) + if err != nil { + t.Fatal(err) + } + native, err := hashPath(nativePath) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: storeDir, ManifestPath: fold.ManifestPath(storeDir, "session"), Manifest: manifest, + Reader: resolver, NativeSnapshot: native, + }) + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if _, err := writer.Append(context.Background(), []byte("{\"benchmark\":\"append\"}\n")); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + if err := writer.Sync(); err != nil { + _ = writer.Close() + _ = resolver.Close() + t.Fatal(err) + } + if err := writer.Close(); err != nil { + _ = resolver.Close() + t.Fatal(err) + } + if err := resolver.Close(); err != nil { + t.Fatal(err) + } + + benchmark := func() fsctl.BenchmarkReport { + t.Helper() + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "benchmark", "session", "--codex-home", home, "--store", storeDir, + "--sequential-block-bytes", "16", "--random-block-bytes", "8", "--random-reads", "10", + "--bypass-os-cache", "--json", + }) + if err := root.Execute(); err != nil { + t.Fatalf("benchmark: %v", err) + } + var report fsctl.BenchmarkReport + if err := json.Unmarshal(output.Bytes(), &report); err != nil { + t.Fatalf("decode benchmark report: %v\n%s", err, output.String()) + } + if !report.OSCacheBypassRequested { + t.Fatal("benchmark did not preserve --bypass-os-cache") + } + if report.Native.Bytes != report.Virtual.Bytes { + t.Fatalf("benchmark byte counts differ: native=%d virtual=%d", report.Native.Bytes, report.Virtual.Bytes) + } + return report + } + + appended := benchmark() + if appended.Native.Bytes <= manifest.Source.Bytes { + t.Fatalf("benchmark ignored active delta: native=%d base=%d", appended.Native.Bytes, manifest.Source.Bytes) + } + + executeFS(t, []string{"fs", "compact", "session", "--codex-home", home, "--store", storeDir, "--apply", "--json"}) + compacted := benchmark() + if compacted.Native.Bytes != appended.Native.Bytes || compacted.Virtual.Bytes != appended.Virtual.Bytes { + t.Fatalf("benchmark changed visible size across compact: appended=%+v compacted=%+v", appended, compacted) + } +} + +func TestFSDoctorUsesExplicitServiceDefinition(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + definition := filepath.Join(t.TempDir(), "isolated-service-definition") + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{ + "fs", "doctor", "--codex-home", home, "--store", storeDir, + "--definition", definition, "--json", + }) + if err := root.Execute(); err != nil { + t.Fatalf("fs doctor: %v", err) + } + var report fsctl.DoctorReport + if err := json.Unmarshal(output.Bytes(), &report); err != nil { + t.Fatalf("decode fs doctor: %v\n%s", err, output.String()) + } + for _, issue := range report.Issues { + if issue.Component != fsctl.ComponentDaemon { + continue + } + if !strings.Contains(issue.Message, definition) { + t.Fatalf("daemon issue did not use explicit definition %q: %#v", definition, issue) + } + return + } + t.Fatalf("fs doctor did not report the missing explicit definition: %#v", report) +} + +func TestFSStatusAndDoctorExposePhysicalStorageAccounting(t *testing.T) { + home, storeDir, _ := fsFixture(t, true) + for _, args := range [][]string{ + {"fs", "status", "--codex-home", home, "--store", storeDir, "--json"}, + {"fs", "doctor", "--codex-home", home, "--store", storeDir, "--json"}, + } { + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs(args) + if err := root.Execute(); err != nil { + t.Fatalf("%v: %v", args, err) + } + var payload struct { + Storage struct { + LogicalSessionBytes int64 `json:"logical_session_bytes"` + TotalPhysicalBytes int64 `json:"total_physical_bytes"` + } `json:"storage"` + StorageLimits storage.Limits `json:"storage_limits"` + AvailableBytes int64 `json:"available_bytes"` + } + if err := json.Unmarshal(output.Bytes(), &payload); err != nil { + t.Fatalf("decode %v: %v\n%s", args, err, output.String()) + } + if payload.Storage.LogicalSessionBytes <= 0 || payload.Storage.TotalPhysicalBytes <= 0 || payload.StorageLimits.MaxPhysicalBytes <= 0 || payload.AvailableBytes <= 0 { + t.Fatalf("incomplete storage accounting for %v: %#v", args, payload) + } + } +} + +func TestStartupStorageGCRunsOnlyAfterHealthyStoreVerification(t *testing.T) { + _, storeDir, _ := fsFixture(t, true) + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + before := countPackGenerationDirectories(t, storeDir) + result, ran, err := startupStorageGC(context.Background(), storeDir) + if err != nil { + t.Fatalf("startupStorageGC: %v", err) + } + if !ran || before != 3 || result.RemovedCount != 1 || countPackGenerationDirectories(t, storeDir) != 2 { + t.Fatalf("healthy startup GC result: before=%d ran=%t result=%#v after=%d", before, ran, result, countPackGenerationDirectories(t, storeDir)) + } + + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + current, err := os.ReadFile(filepath.Join(storeDir, "packs", "CURRENT")) + if err != nil { + t.Fatal(err) + } + index, err := os.ReadFile(filepath.Join(storeDir, "packs", strings.TrimSpace(string(current)), "index.json")) + if err != nil { + t.Fatal(err) + } + var decoded struct { + Objects []struct { + Blocks []struct { + Pack string `json:"pack"` + } `json:"blocks"` + } `json:"objects"` + } + if err := json.Unmarshal(index, &decoded); err != nil || len(decoded.Objects) == 0 || len(decoded.Objects[0].Blocks) == 0 { + t.Fatalf("decode current pack index: %#v err=%v", decoded, err) + } + packPath := filepath.Join(storeDir, "packs", strings.TrimSpace(string(current)), decoded.Objects[0].Blocks[0].Pack) + if err := os.WriteFile(packPath, []byte("corrupt"), 0o600); err != nil { + t.Fatal(err) + } + before = countPackGenerationDirectories(t, storeDir) + _, ran, err = startupStorageGC(context.Background(), storeDir) + if err != nil { + t.Fatalf("unhealthy startupStorageGC: %v", err) + } + if ran || countPackGenerationDirectories(t, storeDir) != before { + t.Fatalf("unhealthy store was mutated: ran=%t before=%d after=%d", ran, before, countPackGenerationDirectories(t, storeDir)) + } +} + +func TestStartStorageMaintenanceDoesNotBlockAvailabilityOrCancelService(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + entered := make(chan struct{}) + release := make(chan struct{}) + released := false + defer func() { + if !released { + close(release) + } + }() + var diagnostics bytes.Buffer + started := make(chan (<-chan struct{}), 1) + go func() { + started <- startStorageMaintenance(ctx, &diagnostics, "store", func(context.Context, string) (storage.StorageGCResult, bool, error) { + close(entered) + <-release + return storage.StorageGCResult{}, true, errors.New("maintenance failed") + }) + }() + + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("storage maintenance did not start") + } + var done <-chan struct{} + select { + case done = <-started: + case <-time.After(100 * time.Millisecond): + t.Fatal("storage maintenance blocked service availability") + } + select { + case <-ctx.Done(): + t.Fatal("storage maintenance canceled the service") + default: + } + + close(release) + released = true + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("storage maintenance did not finish") + } + if !strings.Contains(diagnostics.String(), "storage maintenance failed: maintenance failed") { + t.Fatalf("diagnostics = %q", diagnostics.String()) + } +} + +func TestRuntimeMemoryReclaimableUsesOnlyUnreleasedIdleHeap(t *testing.T) { + if runtimeMemoryReclaimable(runtime.MemStats{HeapIdle: 128 << 20, HeapReleased: 80 << 20}, 64<<20) { + t.Fatal("reclaimed below-threshold idle heap") + } + if !runtimeMemoryReclaimable(runtime.MemStats{HeapIdle: 160 << 20, HeapReleased: 80 << 20}, 64<<20) { + t.Fatal("did not reclaim above-threshold idle heap") + } + if runtimeMemoryReclaimable(runtime.MemStats{HeapIdle: 64 << 20, HeapReleased: 96 << 20}, 1) { + t.Fatal("underflowed released heap accounting") + } +} + +func countPackGenerationDirectories(t *testing.T, store string) int { + t.Helper() + entries, err := os.ReadDir(filepath.Join(store, "packs")) + if err != nil { + t.Fatal(err) + } + count := 0 + for _, entry := range entries { + if entry.IsDir() && !strings.HasPrefix(entry.Name(), ".") { + count++ + } + } + return count +} + +func fsFixture(t *testing.T, archived bool) (string, string, string) { + t.Helper() + home := t.TempDir() + storeDir := filepath.Join(home, "fold-store") + nativePath := filepath.Join(home, "session.jsonl") + source := []byte("{\"type\":\"session_meta\"}\n{\"value\":\"repeated-field-value\"}\n") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatalf("write rollout: %v", err) + } + writeStateFixture(t, home, nativePath) + if archived { + dbPath := filepath.Join(home, "state_5.sqlite") + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open state: %v", err) + } + _, err = db.Exec(`update threads set archived = 1 where id = 'fixture'; update threads set id = 'session' where id = 'fixture'`) + _ = db.Close() + if err != nil { + t.Fatalf("archive fixture: %v", err) + } + } + session := codex.Session{ID: "session", RolloutPath: nativePath, Archived: archived} + if _, err := fold.Fold(context.Background(), toFoldSession(session), fold.FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8}); err != nil { + t.Fatalf("fold fixture: %v", err) + } + if _, err := pack.Build(context.Background(), storeDir, pack.BuildOptions{}); err != nil { + t.Fatalf("pack fixture: %v", err) + } + return home, storeDir, nativePath +} + +func approvedCLIContract(t *testing.T, storeDir string, version string) string { + t.Helper() + cliPath := fakeCLI(t, version) + _, err := compat.Save(filepath.Join(storeDir, "compatibility"), compat.Contract{ + Version: compat.ContractVersion, Platform: runtime.GOOS, ClientKind: "cli", ClientVersion: version, + Operations: []compat.Operation{{Name: "read", Count: 1}}, + TraceSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }) + if err != nil { + t.Fatal(err) + } + return cliPath +} + +func fakeCLI(t *testing.T, version string) string { + t.Helper() + cliPath := filepath.Join(t.TempDir(), "codex") + script := "#!/bin/sh\necho 'codex-cli " + version + "'\n" + if err := os.WriteFile(cliPath, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + return cliPath +} + +func executeFS(t *testing.T, args []string) { + t.Helper() + root := NewRootCommand() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetArgs(args) + if err := root.Execute(); err != nil { + t.Fatalf("execute %v: %v", args, err) + } +} + +func emulateCanonicalRetirement(storeDir string, sessionID string, mountedTarget string, nativeTarget string) <-chan error { + done := make(chan error, 1) + go func() { + stateDirectory := filepath.Join(storeDir, "fs", "sessions", sessionID) + requestPath := filepath.Join(stateDirectory, retirementRequestFilename) + acknowledgementPath := filepath.Join(stateDirectory, retirementAcknowledgementFilename) + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(stateDirectory); err != nil { + done <- fmt.Errorf("managed state moved before retirement request: %w", err) + return + } + request, err := os.ReadFile(requestPath) + if err == nil { + data, readErr := os.ReadFile(nativeTarget) + if readErr == nil { + readErr = os.WriteFile(mountedTarget, data, 0o600) + } + if readErr == nil { + readErr = os.WriteFile(acknowledgementPath, request, 0o600) + } + done <- readErr + return + } + if !errors.Is(err, os.ErrNotExist) { + done <- err + return + } + time.Sleep(5 * time.Millisecond) + } + done <- errors.New("retirement request was not created") + }() + return done +} + +func emulateCanonicalRetirementGeneration(t *testing.T, storeDir string, sessionID string, mountedTarget string, nativeTarget string, expectedGeneration uint64) <-chan error { + t.Helper() + done := make(chan error, 1) + go func() { + stateDirectory := filepath.Join(storeDir, "fs", "sessions", sessionID) + requestPath := filepath.Join(stateDirectory, retirementRequestFilename) + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(requestPath) + if err == nil { + var request retirementControl + if err := json.Unmarshal(data, &request); err != nil { + done <- err + return + } + if request.Generation != expectedGeneration { + rejection := request + rejection.Error = fmt.Sprintf("retirement generation = %d, want recovered %d", request.Generation, expectedGeneration) + if err := writeRetirementAcknowledgement(storeDir, sessionID, rejection); err != nil { + done <- err + return + } + for time.Now().Before(deadline) { + if _, err := os.Stat(requestPath); !errors.Is(err, os.ErrNotExist) { + time.Sleep(5 * time.Millisecond) + continue + } + state, err := managedState(storeDir, sessionID) + if err != nil || state.Generation <= expectedGeneration { + time.Sleep(5 * time.Millisecond) + continue + } + native, err := os.ReadFile(nativeTarget) + if err == nil { + err = os.WriteFile(mountedTarget, native, 0o600) + } + if err == nil { + err = writeMountAcknowledgement(storeDir, sessionID, state.Generation, request.Route) + } + done <- err + return + } + done <- errors.New("managed route was not republished after stale retirement request") + return + } + native, err := os.ReadFile(nativeTarget) + if err == nil { + err = os.WriteFile(mountedTarget, native, 0o600) + } + if err == nil { + err = writeRetirementAcknowledgement(storeDir, sessionID, request) + } + done <- err + return + } + if !errors.Is(err, os.ErrNotExist) { + done <- err + return + } + time.Sleep(5 * time.Millisecond) + } + done <- errors.New("retirement request was not created") + }() + return done +} + +func readMountedFilesystemFile(t *testing.T, filesystem *mountfs.Filesystem, route string, size int) []byte { + t.Helper() + handle, errno := filesystem.Open(route, os.O_RDONLY) + if errno != 0 { + t.Fatalf("open mounted route %s: %v", route, errno) + } + defer filesystem.Release(handle) + data := make([]byte, size) + n, errno := filesystem.Read(handle, data, 0) + if errno != 0 { + t.Fatalf("read mounted route %s: %v", route, errno) + } + return data[:n] +} + +func allowFixtureMount(t *testing.T) { + t.Helper() + previous := mountHealthProbe + mountHealthProbe = func(string) error { return nil } + t.Cleanup(func() { mountHealthProbe = previous }) +} diff --git a/internal/cli/pack.go b/internal/cli/pack.go new file mode 100644 index 0000000..0149405 --- /dev/null +++ b/internal/cli/pack.go @@ -0,0 +1,79 @@ +package cli + +import ( + "fmt" + + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/pack" + "github.com/spf13/cobra" +) + +func newPackCommand() *cobra.Command { + command := &cobra.Command{Use: "pack", Short: "Build and verify packed object generations"} + command.AddCommand(newPackBuildCommand()) + command.AddCommand(newPackDoctorCommand()) + return command +} + +func newPackBuildCommand() *cobra.Command { + var codexHome string + var storeDir string + var options pack.BuildOptions + var jsonOutput bool + command := &cobra.Command{ + Use: "build", + Short: "Build a verified immutable pack generation", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + result, err := pack.Build(command.Context(), resolveFoldStore(home, storeDir), options) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "generation=%s objects=%d blocks=%d packs=%d raw=%s stored=%s\n", result.Generation, result.ObjectCount, result.BlockCount, result.PackCount, formatBytes(result.RawBytes), formatBytes(result.StoredBytes)) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().Int64Var(&options.BlockBytes, "block-bytes", 0, "Uncompressed bytes per independently compressed block") + command.Flags().Int64Var(&options.PackBytes, "pack-bytes", 0, "Maximum stored bytes per pack file") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} + +func newPackDoctorCommand() *cobra.Command { + var codexHome string + var storeDir string + var jsonOutput bool + command := &cobra.Command{ + Use: "doctor", + Short: "Verify the active packed object generation", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + home, err := codex.ResolveHome(codexHome) + if err != nil { + return err + } + result, err := pack.Doctor(command.Context(), resolveFoldStore(home, storeDir)) + if err != nil { + return err + } + if jsonOutput { + return writeJSON(command, result) + } + _, err = fmt.Fprintf(command.OutOrStdout(), "generation=%s objects=%d verified=%d issues=%d\n", result.Generation, result.ObjectCount, result.VerifiedCount, result.IssueCount) + return err + }, + } + command.Flags().StringVar(&codexHome, "codex-home", "", "Codex home directory; defaults to CODEX_HOME or ~/.codex") + command.Flags().StringVar(&storeDir, "store", "", "Fold store directory; defaults to /fold-store") + command.Flags().BoolVar(&jsonOutput, "json", false, "Emit JSON output") + return command +} diff --git a/internal/cli/remove_contained.go b/internal/cli/remove_contained.go index 22f03b4..ada6821 100644 --- a/internal/cli/remove_contained.go +++ b/internal/cli/remove_contained.go @@ -3,8 +3,8 @@ package cli import ( "fmt" - "github.com/jstar0/codexfold/internal/codex" - "github.com/jstar0/codexfold/internal/prune" + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/prune" "github.com/spf13/cobra" ) diff --git a/internal/cli/root.go b/internal/cli/root.go index 0a86c2e..5ad9beb 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -7,8 +7,8 @@ import ( "runtime/debug" "strings" - "github.com/jstar0/codexfold/internal/codex" - "github.com/jstar0/codexfold/internal/scan" + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/scan" "github.com/spf13/cobra" ) @@ -23,6 +23,8 @@ func NewRootCommand() *cobra.Command { Version: resolvedVersion(), } root.AddCommand(newScanCommand()) + root.AddCommand(newForkFamilyCommand()) + root.AddCommand(newArchiveCommand()) root.AddCommand(newContainsCommand()) root.AddCommand(newRemoveContainedCommand()) root.AddCommand(newFoldCommand()) @@ -30,6 +32,8 @@ func NewRootCommand() *cobra.Command { root.AddCommand(newUnfoldCommand("materialize")) root.AddCommand(newDoctorCommand()) root.AddCommand(newGCCommand()) + root.AddCommand(newPackCommand()) + root.AddCommand(newFSCommand()) return root } diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 3106231..b166374 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -8,19 +8,88 @@ import ( "path/filepath" "testing" - "github.com/jstar0/codexfold/internal/scan" + "github.com/samekind/codexfold/internal/family" + "github.com/samekind/codexfold/internal/scan" _ "modernc.org/sqlite" ) func TestRootExposesScanCommand(t *testing.T) { root := NewRootCommand() - for _, name := range []string{"scan", "contains", "remove-contained", "fold", "unfold", "materialize", "doctor", "gc"} { + for _, name := range []string{"scan", "fork-family", "archive", "contains", "remove-contained", "fold", "unfold", "materialize", "doctor", "gc"} { if _, _, err := root.Find([]string{name}); err != nil { t.Fatalf("%s command should be exposed: %v", name, err) } } } +func TestForkFamilyShowAndCompareUseExplicitSessions(t *testing.T) { + home := t.TempDir() + leftPath := filepath.Join(home, "left.jsonl") + rightPath := filepath.Join(home, "right.jsonl") + if err := os.WriteFile(leftPath, []byte("{\"type\":\"session_meta\",\"id\":\"left\"}\n{\"v\":1}\n{\"left\":true}\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(rightPath, []byte("{\"type\":\"session_meta\",\"id\":\"right\"}\n{\"v\":1}\n{\"right\":true}\n"), 0o600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(` + create table threads ( + id text primary key, title text, cwd text, rollout_path text, + model_provider text, model text, updated_at integer, + archived integer, git_branch text + ); + create table thread_spawn_edges ( + parent_thread_id text not null, + child_thread_id text not null primary key, + status text not null + ); + `); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`insert into threads values ('left', 'Left', '/workspace', ?, 'provider', 'model', 2, 0, 'main')`, leftPath); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`insert into threads values ('right', 'Right', '/workspace', ?, 'provider', 'model', 1, 1, 'main')`, rightPath); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`insert into thread_spawn_edges values ('left', 'right', 'closed')`); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + + root := NewRootCommand() + var output bytes.Buffer + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fork-family", "show", "left", "--codex-home", home, "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("fork-family show: %v", err) + } + var report family.Report + if err := json.Unmarshal(output.Bytes(), &report); err != nil || len(report.Members) != 2 || len(report.Edges) != 1 { + t.Fatalf("family show = %#v err=%v output=%s", report, err, output.String()) + } + + output.Reset() + root = NewRootCommand() + root.SetOut(&output) + root.SetErr(&bytes.Buffer{}) + root.SetArgs([]string{"fork-family", "compare", "left", "right", "--codex-home", home, "--json"}) + if err := root.Execute(); err != nil { + t.Fatalf("fork-family compare: %v", err) + } + var comparison family.Comparison + if err := json.Unmarshal(output.Bytes(), &comparison); err != nil || comparison.Relation != family.RelationIndependentTails || comparison.GraphRelation != family.GraphAncestor { + t.Fatalf("family comparison = %#v err=%v output=%s", comparison, err, output.String()) + } +} + func TestRootUsesExplicitBuildVersion(t *testing.T) { previous := Version Version = "v-test" diff --git a/internal/codex/edges.go b/internal/codex/edges.go new file mode 100644 index 0000000..4d22023 --- /dev/null +++ b/internal/codex/edges.go @@ -0,0 +1,55 @@ +package codex + +import ( + "database/sql" + "fmt" + "path/filepath" + + _ "modernc.org/sqlite" +) + +type SpawnEdge struct { + ParentID string `json:"parent_id"` + ChildID string `json:"child_id"` + Status string `json:"status"` +} + +func LoadSpawnEdges(home string) ([]SpawnEdge, error) { + dbPath := filepath.Join(home, "state_5.sqlite") + db, err := sql.Open("sqlite", sqliteReadOnlyDSN(dbPath)) + if err != nil { + return nil, fmt.Errorf("open Codex spawn-edge database: %w", err) + } + defer func() { _ = db.Close() }() + if _, err := db.Exec(`pragma busy_timeout = 5000`); err != nil { + return nil, fmt.Errorf("configure Codex spawn-edge database: %w", err) + } + var exists int + if err := db.QueryRow(`select count(*) from sqlite_master where type = 'table' and name = 'thread_spawn_edges'`).Scan(&exists); err != nil { + return nil, fmt.Errorf("inspect Codex spawn-edge table: %w", err) + } + if exists == 0 { + return []SpawnEdge{}, nil + } + rows, err := db.Query(` + select parent_thread_id, child_thread_id, status + from thread_spawn_edges + order by parent_thread_id, child_thread_id, status + `) + if err != nil { + return nil, fmt.Errorf("query Codex spawn edges: %w", err) + } + defer func() { _ = rows.Close() }() + edges := make([]SpawnEdge, 0) + for rows.Next() { + var edge SpawnEdge + if err := rows.Scan(&edge.ParentID, &edge.ChildID, &edge.Status); err != nil { + return nil, fmt.Errorf("scan Codex spawn edge: %w", err) + } + edges = append(edges, edge) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate Codex spawn edges: %w", err) + } + return edges, nil +} diff --git a/internal/codex/edges_test.go b/internal/codex/edges_test.go new file mode 100644 index 0000000..46fb1f3 --- /dev/null +++ b/internal/codex/edges_test.go @@ -0,0 +1,54 @@ +package codex + +import ( + "database/sql" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func TestLoadSpawnEdgesReturnsCurrentGraphAndAllowsMissingTable(t *testing.T) { + home := t.TempDir() + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(` + create table thread_spawn_edges ( + parent_thread_id text not null, + child_thread_id text not null primary key, + status text not null + ); + insert into thread_spawn_edges values ('parent', 'child-b', 'closed'); + insert into thread_spawn_edges values ('parent', 'child-a', 'open'); + `); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + edges, err := LoadSpawnEdges(home) + if err != nil { + t.Fatal(err) + } + if len(edges) != 2 || edges[0].ChildID != "child-a" || edges[1].Status != "closed" { + t.Fatalf("spawn edges = %#v", edges) + } + + emptyHome := t.TempDir() + empty, err := sql.Open("sqlite", filepath.Join(emptyHome, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := empty.Exec(`create table threads (id text primary key);`); err != nil { + t.Fatal(err) + } + if err := empty.Close(); err != nil { + t.Fatal(err) + } + edges, err = LoadSpawnEdges(emptyHome) + if err != nil || len(edges) != 0 { + t.Fatalf("missing edge table = %#v err=%v", edges, err) + } +} diff --git a/internal/codex/routes.go b/internal/codex/routes.go new file mode 100644 index 0000000..d6abc32 --- /dev/null +++ b/internal/codex/routes.go @@ -0,0 +1,118 @@ +package codex + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + _ "modernc.org/sqlite" +) + +type RouteTarget struct { + Path string `json:"path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` +} + +type RouteOptions struct { + CodexHome string + SessionID string + ExpectedPath string + Target RouteTarget +} + +type RouteResult struct { + SessionID string `json:"session_id"` + PreviousPath string `json:"previous_path"` + CurrentPath string `json:"current_path"` +} + +func RouteSession(ctx context.Context, options RouteOptions) (RouteResult, error) { + if options.CodexHome == "" || options.SessionID == "" || options.ExpectedPath == "" || options.Target.Path == "" || options.Target.Bytes < 0 || len(options.Target.SHA256) != 64 { + return RouteResult{}, errors.New("complete route options and verified target metadata are required") + } + if err := verifyRouteTarget(options.Target); err != nil { + return RouteResult{}, err + } + db, err := sql.Open("sqlite", filepath.Join(options.CodexHome, "state_5.sqlite")) + if err != nil { + return RouteResult{}, fmt.Errorf("open Codex route database: %w", err) + } + defer db.Close() + conn, err := db.Conn(ctx) + if err != nil { + return RouteResult{}, err + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, `pragma busy_timeout = 10000`); err != nil { + return RouteResult{}, err + } + if _, err := conn.ExecContext(ctx, `begin immediate`); err != nil { + return RouteResult{}, fmt.Errorf("begin immediate Codex route transaction: %w", err) + } + committed := false + defer func() { + if !committed { + _, _ = conn.ExecContext(context.Background(), `rollback`) + } + }() + var current string + if err := conn.QueryRowContext(ctx, `select rollout_path from threads where id = ?`, options.SessionID).Scan(¤t); err != nil { + return RouteResult{}, fmt.Errorf("read current Codex route: %w", err) + } + if filepath.Clean(current) != filepath.Clean(options.ExpectedPath) { + return RouteResult{}, fmt.Errorf("Codex route changed: current=%s expected=%s", current, options.ExpectedPath) + } + if err := verifyRouteTarget(options.Target); err != nil { + return RouteResult{}, fmt.Errorf("revalidate route target inside transaction: %w", err) + } + result, err := conn.ExecContext(ctx, `update threads set rollout_path = ? where id = ? and rollout_path = ?`, options.Target.Path, options.SessionID, current) + if err != nil { + return RouteResult{}, fmt.Errorf("update Codex route: %w", err) + } + rows, err := result.RowsAffected() + if err != nil { + return RouteResult{}, err + } + if rows != 1 { + return RouteResult{}, fmt.Errorf("Codex route update affected %d rows", rows) + } + if _, err := conn.ExecContext(ctx, `commit`); err != nil { + return RouteResult{}, fmt.Errorf("commit Codex route: %w", err) + } + committed = true + var confirmed string + if err := conn.QueryRowContext(ctx, `select rollout_path from threads where id = ?`, options.SessionID).Scan(&confirmed); err != nil { + return RouteResult{}, fmt.Errorf("confirm Codex route: %w", err) + } + if filepath.Clean(confirmed) != filepath.Clean(options.Target.Path) { + return RouteResult{}, errors.New("committed Codex route did not persist") + } + return RouteResult{SessionID: options.SessionID, PreviousPath: current, CurrentPath: confirmed}, nil +} + +func verifyRouteTarget(target RouteTarget) error { + file, err := os.Open(target.Path) + if err != nil { + return fmt.Errorf("open route target: %w", err) + } + hasher := sha256.New() + bytesRead, copyErr := io.Copy(hasher, file) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + if bytesRead != target.Bytes || hex.EncodeToString(hasher.Sum(nil)) != target.SHA256 { + return errors.New("route target does not match current-byte metadata") + } + return nil +} diff --git a/internal/codex/routes_test.go b/internal/codex/routes_test.go new file mode 100644 index 0000000..f269802 --- /dev/null +++ b/internal/codex/routes_test.go @@ -0,0 +1,118 @@ +package codex + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "os" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func TestRouteSessionOptimisticallyUpdatesExpectedPath(t *testing.T) { + home, nativePath := routeFixture(t) + virtualPath := filepath.Join(home, "mounted", "session.jsonl") + if err := os.MkdirAll(filepath.Dir(virtualPath), 0o700); err != nil { + t.Fatalf("create mount fixture: %v", err) + } + data := []byte("current virtual bytes\n") + if err := os.WriteFile(virtualPath, data, 0o600); err != nil { + t.Fatalf("write target: %v", err) + } + result, err := RouteSession(context.Background(), RouteOptions{CodexHome: home, SessionID: "session", ExpectedPath: nativePath, Target: RouteTarget{Path: virtualPath, Bytes: int64(len(data)), SHA256: routeDigest(data)}}) + if err != nil { + t.Fatalf("RouteSession returned error: %v", err) + } + if result.PreviousPath != nativePath || result.CurrentPath != virtualPath { + t.Fatalf("unexpected route result: %#v", result) + } + if got := queryRoute(t, home); got != virtualPath { + t.Fatalf("database route = %q, want %q", got, virtualPath) + } +} + +func TestRouteSessionRejectsConcurrentOrStaleExpectedPath(t *testing.T) { + home, nativePath := routeFixture(t) + changedPath := filepath.Join(home, "changed.jsonl") + if err := updateRoute(t, home, changedPath); err != nil { + t.Fatalf("change route: %v", err) + } + target := filepath.Join(home, "target.jsonl") + data := []byte("target") + _ = os.WriteFile(target, data, 0o600) + if _, err := RouteSession(context.Background(), RouteOptions{CodexHome: home, SessionID: "session", ExpectedPath: nativePath, Target: RouteTarget{Path: target, Bytes: int64(len(data)), SHA256: routeDigest(data)}}); err == nil { + t.Fatal("RouteSession should reject a stale expected path") + } + if got := queryRoute(t, home); got != changedPath { + t.Fatalf("rejected transaction changed route to %q", got) + } +} + +func TestRouteSessionRejectsStaleOrCorruptFallbackBytes(t *testing.T) { + home, nativePath := routeFixture(t) + target := filepath.Join(home, "fallback.jsonl") + if err := os.WriteFile(target, []byte("old snapshot"), 0o600); err != nil { + t.Fatalf("write fallback: %v", err) + } + current := []byte("latest bytes") + if _, err := RouteSession(context.Background(), RouteOptions{CodexHome: home, SessionID: "session", ExpectedPath: nativePath, Target: RouteTarget{Path: target, Bytes: int64(len(current)), SHA256: routeDigest(current)}}); err == nil { + t.Fatal("RouteSession should reject a target not equal to current-byte metadata") + } + if got := queryRoute(t, home); got != nativePath { + t.Fatalf("corrupt fallback changed route to %q", got) + } +} + +func routeFixture(t *testing.T) (string, string) { + t.Helper() + home := t.TempDir() + nativePath := filepath.Join(home, "native.jsonl") + if err := os.WriteFile(nativePath, []byte("native\n"), 0o600); err != nil { + t.Fatalf("write native: %v", err) + } + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatalf("open state: %v", err) + } + _, err = db.Exec(`create table threads (id text primary key, rollout_path text not null); insert into threads values ('session', ?)`, nativePath) + if closeErr := db.Close(); err == nil { + err = closeErr + } + if err != nil { + t.Fatalf("create state: %v", err) + } + return home, nativePath +} + +func queryRoute(t *testing.T, home string) string { + t.Helper() + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatalf("open state: %v", err) + } + defer db.Close() + var path string + if err := db.QueryRow(`select rollout_path from threads where id = 'session'`).Scan(&path); err != nil { + t.Fatalf("query route: %v", err) + } + return path +} + +func updateRoute(t *testing.T, home, path string) error { + t.Helper() + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + return err + } + defer db.Close() + _, err = db.Exec(`update threads set rollout_path = ? where id = 'session'`, path) + return err +} + +func routeDigest(data []byte) string { + digest := sha256.Sum256(data) + return hex.EncodeToString(digest[:]) +} diff --git a/internal/compat/compat_test.go b/internal/compat/compat_test.go new file mode 100644 index 0000000..14364e6 --- /dev/null +++ b/internal/compat/compat_test.go @@ -0,0 +1,159 @@ +package compat + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestParseFSUsageProducesSanitizedOperationContract(t *testing.T) { + trace := strings.Join([]string{ + "12:00:00.000 open F=3 (R__________X___) /Users/example/.codex/sessions/private.jsonl codex.123", + "12:00:00.001 read F=3 B=4096 /Users/example/.codex/sessions/private.jsonl codex.123", + "12:00:00.002 fcntl F=3 codex.123", + "12:00:00.003 fsync F=3 /Users/example/.codex/sessions/private.jsonl codex.123", + "12:00:00.004 read F=3 B=4096 /Users/example/.codex/sessions/private.jsonl codex.123", + }, "\n") + contract, err := ParseFSUsage(strings.NewReader(trace), ContractOptions{Platform: "darwin", ClientKind: "cli", ClientVersion: "0.1.0"}) + if err != nil { + t.Fatalf("ParseFSUsage returned error: %v", err) + } + if contract.TraceSHA256 == "" || len(contract.Operations) != 4 { + t.Fatalf("unexpected contract: %#v", contract) + } + encoded, err := json.Marshal(contract) + if err != nil { + t.Fatalf("marshal contract: %v", err) + } + if bytes.Contains(encoded, []byte("/Users/example")) || bytes.Contains(encoded, []byte("private.jsonl")) { + t.Fatalf("contract leaked trace paths: %s", encoded) + } + if contract.Operations[1].Name != "read" || contract.Operations[1].Count != 2 { + t.Fatalf("operation aggregation differs: %#v", contract.Operations) + } + if got := contract.Operations[0].Signatures; len(got) != 1 || got[0].Value != "(R__________X___)" { + t.Fatalf("open signatures should contain only stable flags: %#v", got) + } + if got := contract.Operations[1].Signatures; len(got) != 0 { + t.Fatalf("read signatures leaked volatile descriptor or byte counts: %#v", got) + } + if got := contract.Operations[2].Signatures; len(got) != 1 || got[0].Value != "" { + t.Fatalf("fcntl signatures should preserve the stable command: %#v", got) + } +} + +func TestParseFSUsageRecognizesSanitizedFuseAdapterOperations(t *testing.T) { + trace := "1 getattr\n2 readdir\n3 open\n4 read\n5 release\n6 rename\n7 fsync\n" + contract, err := ParseFSUsage(strings.NewReader(trace), ContractOptions{Platform: "darwin", ClientKind: "cli", ClientVersion: "1.2.3"}) + if err != nil { + t.Fatal(err) + } + if len(contract.Operations) != 7 { + t.Fatalf("adapter operations = %#v", contract.Operations) + } +} + +func TestParseFSUsageRecognizesNativeFSKitOperationsWithoutDoubleCountingIO(t *testing.T) { + trace := strings.Join([]string{ + "1 operation=getattr request=1 status=0 payload=18", + "2 operation=read request=2 status=0 payload=20", + "3 io=read handle=1 offset=0 bytes=4096", + "4 operation=write request=3 status=0 payload=64", + "5 io=write handle=1 offset=4096 bytes=64", + "6 operation=sync request=4 status=0 payload=0", + "7 operation=statfs request=5 status=0 payload=0", + "8 operation=release request=6 status=0 payload=8", + "9 operation=hello request=1 status=0 payload=36", + "10 operation=namespace_version request=7 status=0 payload=0", + }, "\n") + contract, err := ParseFSUsage(strings.NewReader(trace), ContractOptions{Platform: "darwin", ClientKind: "desktop", ClientVersion: "26.1"}) + if err != nil { + t.Fatal(err) + } + want := []Operation{ + {Name: "getattr", Count: 1}, + {Name: "read", Count: 1}, + {Name: "write", Count: 1}, + {Name: "fsync", Count: 1}, + {Name: "statfs", Count: 1}, + {Name: "release", Count: 1}, + } + if !reflect.DeepEqual(contract.Operations, want) { + t.Fatalf("native FSKit operations = %#v, want %#v", contract.Operations, want) + } +} + +func TestParseFSUsageCanonicalizesDarwinKernelAliases(t *testing.T) { + trace := strings.Join([]string{ + "12:00:00.000 RdData[S] D=1 B=0x1000 /tmp/session.jsonl codex.1", + "12:00:00.001 WrData[A] D=1 B=0x1000 /tmp/session.jsonl codex.1", + "12:00:00.002 statfs64 /tmp/session.jsonl codex.1", + "12:00:00.003 fstatfs64 F=4 codex.1", + "12:00:00.004 fstatat64 /tmp/session.jsonl codex.1", + "12:00:00.005 getdirentries64 F=4 codex.1", + "12:00:00.006 open_dprotected F=4 /tmp/session.jsonl codex.1", + }, "\n") + contract, err := ParseFSUsage(strings.NewReader(trace), ContractOptions{Platform: "darwin", ClientKind: "desktop", ClientVersion: "1.2.3"}) + if err != nil { + t.Fatal(err) + } + want := []Operation{ + {Name: "read", Count: 1}, + {Name: "write", Count: 1}, + {Name: "statfs", Count: 2}, + {Name: "fstat", Count: 1}, + {Name: "readdir", Count: 1}, + {Name: "open", Count: 1}, + } + if !reflect.DeepEqual(contract.Operations, want) { + t.Fatalf("Darwin operations = %#v, want %#v", contract.Operations, want) + } +} + +func TestEvaluateQuarantinesUnknownClientVersion(t *testing.T) { + contracts := []Contract{{Version: ContractVersion, Platform: "darwin", ClientKind: "cli", ClientVersion: "1.0.0", TraceSHA256: strings.Repeat("a", 64)}} + approved := Evaluate([]ClientVersion{{Platform: "darwin", Kind: "cli", Version: "1.0.0"}}, contracts) + if approved.Quarantine || !approved.Approved { + t.Fatalf("known version should be approved: %#v", approved) + } + unknown := Evaluate([]ClientVersion{{Platform: "darwin", Kind: "cli", Version: "1.1.0"}}, contracts) + if !unknown.Quarantine || unknown.Approved || len(unknown.Unknown) != 1 { + t.Fatalf("unknown version should quarantine: %#v", unknown) + } +} + +func TestSaveAndLoadContractRoundTrip(t *testing.T) { + root := t.TempDir() + contract := Contract{Version: ContractVersion, Platform: "darwin", ClientKind: "desktop", ClientVersion: "26.1", TraceSHA256: strings.Repeat("b", 64), Operations: []Operation{{Name: "open", Count: 1}}} + path, err := Save(root, contract) + if err != nil { + t.Fatalf("Save returned error: %v", err) + } + loaded, err := Load(path) + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + if loaded.ClientVersion != contract.ClientVersion || loaded.TraceSHA256 != contract.TraceSHA256 { + t.Fatalf("loaded contract differs: %#v", loaded) + } +} + +func TestDetectCLIVersionParsesCommandOutput(t *testing.T) { + root := t.TempDir() + command := filepath.Join(root, "codex-test") + if err := os.WriteFile(command, []byte("#!/bin/sh\necho 'codex-cli 9.8.7'\n"), 0o700); err != nil { + t.Fatalf("write fake CLI: %v", err) + } + version, err := DetectCLIVersion(context.Background(), command) + if err != nil { + t.Fatalf("DetectCLIVersion returned error: %v", err) + } + if version.Platform == "" || version.Kind != "cli" || version.Version != "9.8.7" { + t.Fatalf("unexpected CLI version: %#v", version) + } +} diff --git a/internal/compat/contract.go b/internal/compat/contract.go new file mode 100644 index 0000000..39c248a --- /dev/null +++ b/internal/compat/contract.go @@ -0,0 +1,183 @@ +package compat + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +const ContractVersion = 1 + +type Operation struct { + Name string `json:"name"` + Count int `json:"count"` + Signatures []Signature `json:"signatures,omitempty"` +} + +type Signature struct { + Value string `json:"value"` + Count int `json:"count"` +} + +type Contract struct { + Version int `json:"version"` + Platform string `json:"platform"` + ClientKind string `json:"client_kind"` + ClientVersion string `json:"client_version"` + Operations []Operation `json:"operations"` + TraceSHA256 string `json:"trace_sha256"` +} + +type ClientVersion struct { + Platform string `json:"platform,omitempty"` + Kind string `json:"kind"` + Version string `json:"version"` +} + +type Evaluation struct { + Approved bool `json:"approved"` + Quarantine bool `json:"quarantine"` + Unknown []ClientVersion `json:"unknown,omitempty"` +} + +func Evaluate(installed []ClientVersion, contracts []Contract) Evaluation { + known := make(map[string]struct{}, len(contracts)) + for _, contract := range contracts { + if validateContract(contract) == nil { + known[contract.Platform+"\x00"+contract.ClientKind+"\x00"+contract.ClientVersion] = struct{}{} + } + } + result := Evaluation{Approved: true} + for _, client := range installed { + key := client.Platform + "\x00" + client.Kind + "\x00" + client.Version + if _, ok := known[key]; !ok || client.Kind == "" || client.Version == "" { + result.Unknown = append(result.Unknown, client) + } + } + if len(result.Unknown) != 0 { + result.Approved = false + result.Quarantine = true + } + return result +} + +func Save(root string, contract Contract) (string, error) { + if err := validateContract(contract); err != nil { + return "", err + } + directory := filepath.Join(root, safeName(contract.Platform), safeName(contract.ClientKind)) + if err := os.MkdirAll(directory, 0o700); err != nil { + return "", err + } + path := filepath.Join(directory, safeName(contract.ClientVersion)+".json") + data, err := json.MarshalIndent(contract, "", " ") + if err != nil { + return "", err + } + data = append(data, '\n') + temporary, err := os.CreateTemp(directory, ".contract-*.tmp") + if err != nil { + return "", err + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return "", err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return "", err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return "", err + } + if err := temporary.Close(); err != nil { + return "", err + } + if err := os.Rename(temporaryPath, path); err != nil { + return "", err + } + return path, nil +} + +func Load(path string) (Contract, error) { + data, err := os.ReadFile(path) + if err != nil { + return Contract{}, err + } + var contract Contract + if err := json.Unmarshal(data, &contract); err != nil { + return Contract{}, err + } + if err := validateContract(contract); err != nil { + return Contract{}, err + } + return contract, nil +} + +func LoadAll(root string) ([]Contract, error) { + var contracts []Contract + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + return nil + } + contract, err := Load(path) + if err != nil { + return err + } + contracts = append(contracts, contract) + return nil + }) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + sort.Slice(contracts, func(i, j int) bool { + if contracts[i].Platform != contracts[j].Platform { + return contracts[i].Platform < contracts[j].Platform + } + if contracts[i].ClientKind != contracts[j].ClientKind { + return contracts[i].ClientKind < contracts[j].ClientKind + } + return contracts[i].ClientVersion < contracts[j].ClientVersion + }) + return contracts, err +} + +func validateContract(contract Contract) error { + if contract.Version != ContractVersion || contract.Platform == "" || contract.ClientKind == "" || contract.ClientVersion == "" || len(contract.TraceSHA256) != 64 { + return errors.New("invalid compatibility contract metadata") + } + for _, operation := range contract.Operations { + if operation.Name == "" || operation.Count <= 0 { + return errors.New("invalid compatibility operation") + } + for _, signature := range operation.Signatures { + if signature.Value == "" || signature.Count <= 0 || strings.ContainsAny(signature.Value, "/\\") { + return errors.New("invalid compatibility operation signature") + } + } + } + return nil +} + +func safeName(value string) string { + value = strings.Map(func(character rune) rune { + if character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' || strings.ContainsRune("._-", character) { + return character + } + return '_' + }, value) + if value == "" || value == "." || value == ".." { + return fmt.Sprintf("value-%x", []byte(value)) + } + return value +} diff --git a/internal/compat/fsusage.go b/internal/compat/fsusage.go new file mode 100644 index 0000000..792d9e4 --- /dev/null +++ b/internal/compat/fsusage.go @@ -0,0 +1,116 @@ +package compat + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "regexp" + "sort" + "strings" +) + +type ContractOptions struct { + Platform string + ClientKind string + ClientVersion string +} + +var operationPattern = regexp.MustCompile(`(?i)\b(open|openat|close|release|read|pread|readv|write|pwrite|writev|flush|fsync|fdatasync|stat|stat64|lstat|lstat64|fstat|fstat64|statfs|getattr|readdir|access|chmod|chown|utimens|mmap|truncate|ftruncate|create|mknod|mkdir|rmdir|link|symlink|readlink|rename|renameat|unlink|unlinkat|flock|fcntl|clonefile|getattrlist|setxattr|getxattr|listxattr|removexattr)\b`) +var signaturePattern = regexp.MustCompile(`\([A-Z_]{4,32}\)|<[A-Z0-9_=+-]+>`) + +func ParseFSUsage(reader io.Reader, options ContractOptions) (Contract, error) { + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 4096), 4<<20) + hasher := sha256.New() + counts := make(map[string]int) + signatures := make(map[string]map[string]int) + var names []string + for scanner.Scan() { + line := scanner.Bytes() + _, _ = hasher.Write(line) + _, _ = hasher.Write([]byte{'\n'}) + name := fsUsageOperationName(line) + if name == "" { + continue + } + if counts[name] == 0 { + names = append(names, name) + } + counts[name]++ + if signatures[name] == nil { + signatures[name] = make(map[string]int) + } + for _, signature := range signaturePattern.FindAllString(string(line), -1) { + signatures[name][signature]++ + } + } + if err := scanner.Err(); err != nil { + return Contract{}, err + } + if len(counts) == 0 { + return Contract{}, errors.New("trace contains no recognized filesystem operations") + } + contract := Contract{Version: ContractVersion, Platform: options.Platform, ClientKind: options.ClientKind, ClientVersion: options.ClientVersion, TraceSHA256: hex.EncodeToString(hasher.Sum(nil))} + for _, name := range names { + operation := Operation{Name: name, Count: counts[name]} + values := make([]string, 0, len(signatures[name])) + for value := range signatures[name] { + values = append(values, value) + } + sort.Strings(values) + for _, value := range values { + operation.Signatures = append(operation.Signatures, Signature{Value: value, Count: signatures[name][value]}) + } + contract.Operations = append(contract.Operations, operation) + } + if err := validateContract(contract); err != nil { + return Contract{}, err + } + return contract, nil +} + +func fsUsageOperationName(line []byte) string { + fields := bytes.Fields(line) + if len(fields) > 1 { + token := strings.ToLower(string(fields[1])) + if strings.HasPrefix(token, "operation=") { + return nativeFSKitOperationName(strings.TrimPrefix(token, "operation=")) + } + if strings.HasPrefix(token, "io=") { + return "" + } + switch { + case token == "rddata" || strings.HasPrefix(token, "rddata["): + return "read" + case token == "wrdata" || strings.HasPrefix(token, "wrdata["): + return "write" + case token == "getdirentries64": + return "readdir" + case token == "statfs64" || token == "fstatfs64": + return "statfs" + case token == "fstatat64": + return "fstat" + case token == "open_dprotected": + return "open" + } + } + match := operationPattern.FindSubmatch(line) + if len(match) == 0 { + return "" + } + return strings.ToLower(string(match[1])) +} + +func nativeFSKitOperationName(name string) string { + if name == "sync" { + return "fsync" + } + match := operationPattern.FindStringSubmatch(name) + if len(match) == 0 || match[0] != name { + return "" + } + return strings.ToLower(match[1]) +} diff --git a/internal/compat/version.go b/internal/compat/version.go new file mode 100644 index 0000000..8832204 --- /dev/null +++ b/internal/compat/version.go @@ -0,0 +1,24 @@ +package compat + +import ( + "context" + "errors" + "os/exec" + "runtime" + "strings" +) + +func DetectCLIVersion(ctx context.Context, binary string) (ClientVersion, error) { + if binary == "" { + return ClientVersion{}, errors.New("Codex CLI path is required") + } + output, err := exec.CommandContext(ctx, binary, "--version").CombinedOutput() + if err != nil { + return ClientVersion{}, err + } + fields := strings.Fields(strings.TrimSpace(string(output))) + if len(fields) < 2 { + return ClientVersion{}, errors.New("Codex CLI returned an unrecognized version") + } + return ClientVersion{Platform: runtime.GOOS, Kind: "cli", Version: fields[len(fields)-1]}, nil +} diff --git a/internal/compat/version_darwin.go b/internal/compat/version_darwin.go new file mode 100644 index 0000000..b1c670c --- /dev/null +++ b/internal/compat/version_darwin.go @@ -0,0 +1,26 @@ +//go:build darwin + +package compat + +import ( + "context" + "errors" + "os/exec" + "strings" +) + +func DetectDesktopVersion(ctx context.Context, appPath string) (ClientVersion, error) { + if appPath == "" { + return ClientVersion{}, errors.New("Codex application path is required") + } + plist := appPath + "/Contents/Info.plist" + short, err := exec.CommandContext(ctx, "/usr/bin/plutil", "-extract", "CFBundleShortVersionString", "raw", plist).Output() + if err != nil { + return ClientVersion{}, err + } + build, err := exec.CommandContext(ctx, "/usr/bin/plutil", "-extract", "CFBundleVersion", "raw", plist).Output() + if err != nil { + return ClientVersion{}, err + } + return ClientVersion{Platform: "darwin", Kind: "desktop", Version: strings.TrimSpace(string(short)) + "+" + strings.TrimSpace(string(build))}, nil +} diff --git a/internal/compat/version_other.go b/internal/compat/version_other.go new file mode 100644 index 0000000..efd36db --- /dev/null +++ b/internal/compat/version_other.go @@ -0,0 +1,12 @@ +//go:build !darwin + +package compat + +import ( + "context" + "errors" +) + +func DetectDesktopVersion(context.Context, string) (ClientVersion, error) { + return ClientVersion{}, errors.New("desktop version detection is not implemented on this platform") +} diff --git a/internal/enroll/apply.go b/internal/enroll/apply.go new file mode 100644 index 0000000..a77650b --- /dev/null +++ b/internal/enroll/apply.go @@ -0,0 +1,55 @@ +package enroll + +import ( + "context" + "errors" + "fmt" + "os" +) + +type ApplyOptions struct { + Limit int + IsManaged func(context.Context, string) (bool, error) + Apply func(context.Context, Decision) error +} + +type ApplyResult struct { + Selected int `json:"selected"` + Applied int `json:"applied"` + SkippedChanged int `json:"skipped_changed"` + SkippedManaged int `json:"skipped_managed"` +} + +func Apply(ctx context.Context, plan Plan, options ApplyOptions) (ApplyResult, error) { + if options.IsManaged == nil || options.Apply == nil { + return ApplyResult{}, errors.New("enrollment managed-state and apply callbacks are required") + } + limit := options.Limit + if limit <= 0 || limit > len(plan.Selected) { + limit = len(plan.Selected) + } + result := ApplyResult{Selected: min(limit, len(plan.Selected))} + for _, decision := range plan.Selected[:limit] { + if err := ctx.Err(); err != nil { + return result, err + } + managed, err := options.IsManaged(ctx, decision.SessionID) + if err != nil { + return result, err + } + if managed { + result.SkippedManaged++ + continue + } + info, err := os.Lstat(decision.RolloutPath) + if err != nil || !info.Mode().IsRegular() || info.Size() != decision.Fingerprint.Size || info.ModTime().UnixNano() != decision.Fingerprint.ModTimeUnixNano { + result.SkippedChanged++ + continue + } + if err := options.Apply(ctx, decision); err != nil { + return result, fmt.Errorf("apply enrollment for %s: %w", decision.SessionID, err) + } + result.Applied++ + } + return result, nil +} diff --git a/internal/enroll/observations.go b/internal/enroll/observations.go new file mode 100644 index 0000000..bf79bec --- /dev/null +++ b/internal/enroll/observations.go @@ -0,0 +1,88 @@ +package enroll + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +type observationFile struct { + Version int `json:"version"` + Observations Observations `json:"observations"` +} + +func LoadObservations(path string) (Observations, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return make(Observations), nil + } + if err != nil { + return nil, err + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var stored observationFile + if err := decoder.Decode(&stored); err != nil { + return nil, fmt.Errorf("decode enrollment observations: %w", err) + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + if err == nil { + err = errors.New("multiple JSON values") + } + return nil, fmt.Errorf("decode enrollment observations: %w", err) + } + if stored.Version != 1 { + return nil, fmt.Errorf("unsupported enrollment observation version %d", stored.Version) + } + if stored.Observations == nil { + stored.Observations = make(Observations) + } + return stored.Observations, nil +} + +func SaveObservations(path string, observations Observations) error { + if !filepath.IsAbs(path) { + return errors.New("enrollment observation path must be absolute") + } + if observations == nil { + observations = make(Observations) + } + data, err := json.MarshalIndent(observationFile{Version: 1, Observations: observations}, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + directory := filepath.Dir(path) + if err := os.MkdirAll(directory, 0o700); err != nil { + return err + } + temporary, err := os.CreateTemp(directory, ".observations-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return err + } + return syncObservationDirectory(directory) +} diff --git a/internal/enroll/observations_sync_unix.go b/internal/enroll/observations_sync_unix.go new file mode 100644 index 0000000..81e3d0d --- /dev/null +++ b/internal/enroll/observations_sync_unix.go @@ -0,0 +1,14 @@ +//go:build !windows + +package enroll + +import "os" + +func syncObservationDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} diff --git a/internal/enroll/observations_sync_windows.go b/internal/enroll/observations_sync_windows.go new file mode 100644 index 0000000..24670f7 --- /dev/null +++ b/internal/enroll/observations_sync_windows.go @@ -0,0 +1,5 @@ +//go:build windows + +package enroll + +func syncObservationDirectory(string) error { return nil } diff --git a/internal/enroll/observations_test.go b/internal/enroll/observations_test.go new file mode 100644 index 0000000..0385bb1 --- /dev/null +++ b/internal/enroll/observations_test.go @@ -0,0 +1,35 @@ +package enroll + +import ( + "os" + "path/filepath" + "testing" +) + +func TestObservationStoreRoundTripsAndRejectsUnknownVersion(t *testing.T) { + path := filepath.Join(t.TempDir(), "enrollment", "observations.json") + want := Observations{"session": {Path: "/tmp/session.jsonl", Size: 12, ModTimeUnixNano: 34, StableSinceUnixNano: 56}} + if err := SaveObservations(path, want); err != nil { + t.Fatalf("SaveObservations: %v", err) + } + got, err := LoadObservations(path) + if err != nil { + t.Fatalf("LoadObservations: %v", err) + } + if got["session"] != want["session"] { + t.Fatalf("observations = %#v, want %#v", got, want) + } + if err := os.WriteFile(path, []byte(`{"version":2,"observations":{}}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadObservations(path); err == nil { + t.Fatal("unknown observation version should fail") + } +} + +func TestLoadObservationsReturnsEmptyWhenMissing(t *testing.T) { + got, err := LoadObservations(filepath.Join(t.TempDir(), "missing.json")) + if err != nil || len(got) != 0 { + t.Fatalf("missing observations = %#v err=%v", got, err) + } +} diff --git a/internal/enroll/planner.go b/internal/enroll/planner.go new file mode 100644 index 0000000..525b415 --- /dev/null +++ b/internal/enroll/planner.go @@ -0,0 +1,243 @@ +package enroll + +import ( + "context" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/storage" +) + +type Reason string + +const ( + ReasonAlreadyManaged Reason = "already-managed" + ReasonNotArchived Reason = "not-archived" + ReasonInvalidPath Reason = "invalid-rollout-path" + ReasonStabilityPending Reason = "stability-observation-pending" + ReasonFileChanged Reason = "rollout-changed" + ReasonWriterActive Reason = "writer-active" + ReasonDoctorUnhealthy Reason = "doctor-unhealthy" + ReasonCompatibility Reason = "client-compatibility-unapproved" + ReasonMountUnhealthy Reason = "mount-unhealthy" + ReasonNamespaceDisabled Reason = "canonical-namespace-disabled" + ReasonPromotionStage Reason = "promotion-stage-blocked" + ReasonInsufficientBudget Reason = "insufficient-storage-budget" + ReasonBatchLimit Reason = "batch-limit" +) + +type Policy struct { + StableFor time.Duration `json:"stable_for"` + BatchSize int `json:"batch_size"` + ArchivedOnly bool `json:"archived_only"` +} + +type Gates struct { + DoctorHealthy bool `json:"doctor_healthy"` + CompatibilityApproved bool `json:"compatibility_approved"` + MountHealthy bool `json:"mount_healthy"` + CanonicalNamespace bool `json:"canonical_namespace"` + EnrollmentAllowed bool `json:"enrollment_allowed"` +} + +type Observation struct { + Path string `json:"path"` + Size int64 `json:"size"` + ModTimeUnixNano int64 `json:"mod_time_unix_nano"` + StableSinceUnixNano int64 `json:"stable_since_unix_nano"` +} + +type Observations map[string]Observation + +type Fingerprint struct { + Size int64 `json:"size"` + ModTimeUnixNano int64 `json:"mod_time_unix_nano"` +} + +type WriterProbe func(context.Context, codex.Session) (bool, error) + +type Input struct { + Sessions []codex.Session + Managed map[string]struct{} + Previous Observations + Now time.Time + Policy Policy + Gates Gates + WriterActive WriterProbe + Budget storage.Checker +} + +type Decision struct { + SessionID string `json:"session_id"` + RolloutPath string `json:"rollout_path"` + Archived bool `json:"archived"` + Eligible bool `json:"eligible"` + Selected bool `json:"selected"` + Reasons []Reason `json:"reasons,omitempty"` + Fingerprint Fingerprint `json:"fingerprint"` +} + +type Plan struct { + GeneratedAt string `json:"generated_at"` + Decisions []Decision `json:"decisions"` + Selected []Decision `json:"selected"` + Observations Observations `json:"observations"` +} + +func Build(ctx context.Context, input Input) (Plan, error) { + if input.Now.IsZero() { + input.Now = time.Now() + } + if input.Policy.StableFor <= 0 { + input.Policy.StableFor = time.Hour + } + if input.Policy.BatchSize <= 0 { + input.Policy.BatchSize = 1 + } + if input.Previous == nil { + input.Previous = make(Observations) + } + plan := Plan{GeneratedAt: input.Now.UTC().Format(time.RFC3339Nano), Observations: make(Observations)} + sessions := append([]codex.Session(nil), input.Sessions...) + sort.Slice(sessions, func(i, j int) bool { + if sessions[i].UpdatedAt == sessions[j].UpdatedAt { + return sessions[i].ID < sessions[j].ID + } + return sessions[i].UpdatedAt < sessions[j].UpdatedAt + }) + var selectedPersistentBytes int64 + for _, session := range sessions { + if err := ctx.Err(); err != nil { + return Plan{}, err + } + decision := Decision{SessionID: session.ID, RolloutPath: filepath.Clean(session.RolloutPath), Archived: session.Archived} + if _, managed := input.Managed[session.ID]; managed { + decision.Reasons = append(decision.Reasons, ReasonAlreadyManaged) + plan.Decisions = append(plan.Decisions, decision) + continue + } + if !safeSessionID(session.ID) || !filepath.IsAbs(session.RolloutPath) { + decision.Reasons = append(decision.Reasons, ReasonInvalidPath) + plan.Decisions = append(plan.Decisions, decision) + continue + } + info, err := os.Lstat(session.RolloutPath) + if err != nil || !info.Mode().IsRegular() { + decision.Reasons = append(decision.Reasons, ReasonInvalidPath) + plan.Decisions = append(plan.Decisions, decision) + continue + } + decision.Fingerprint = Fingerprint{Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano()} + previous, observed := input.Previous[session.ID] + unchanged := observed && filepath.Clean(previous.Path) == decision.RolloutPath && previous.Size == info.Size() && previous.ModTimeUnixNano == info.ModTime().UnixNano() + observation := Observation{Path: decision.RolloutPath, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), StableSinceUnixNano: input.Now.UnixNano()} + if unchanged { + observation.StableSinceUnixNano = previous.StableSinceUnixNano + } + plan.Observations[session.ID] = observation + + if input.Policy.ArchivedOnly && !session.Archived { + decision.Reasons = append(decision.Reasons, ReasonNotArchived) + } + if !input.Gates.DoctorHealthy { + decision.Reasons = append(decision.Reasons, ReasonDoctorUnhealthy) + } + if !input.Gates.CompatibilityApproved { + decision.Reasons = append(decision.Reasons, ReasonCompatibility) + } + if !input.Gates.MountHealthy { + decision.Reasons = append(decision.Reasons, ReasonMountUnhealthy) + } + if !input.Gates.CanonicalNamespace { + decision.Reasons = append(decision.Reasons, ReasonNamespaceDisabled) + } + if !input.Gates.EnrollmentAllowed { + decision.Reasons = append(decision.Reasons, ReasonPromotionStage) + } + if input.WriterActive != nil { + active, err := input.WriterActive(ctx, session) + if err != nil { + return Plan{}, fmt.Errorf("probe writer for %s: %w", session.ID, err) + } + if active { + decision.Reasons = append(decision.Reasons, ReasonWriterActive) + } + } + switch { + case !observed: + decision.Reasons = append(decision.Reasons, ReasonStabilityPending) + case !unchanged: + decision.Reasons = append(decision.Reasons, ReasonFileChanged) + case input.Now.Sub(time.Unix(0, observation.StableSinceUnixNano)) < input.Policy.StableFor: + decision.Reasons = append(decision.Reasons, ReasonStabilityPending) + case input.Now.Sub(info.ModTime()) < input.Policy.StableFor: + decision.Reasons = append(decision.Reasons, ReasonStabilityPending) + } + if len(decision.Reasons) != 0 { + plan.Decisions = append(plan.Decisions, decision) + continue + } + projectedPersistent, err := enrollmentPersistentBytes(info.Size()) + if err != nil { + return Plan{}, err + } + cumulative, overflow := addBytes(selectedPersistentBytes, projectedPersistent) + if overflow { + return Plan{}, errors.New("enrollment batch byte estimate overflow") + } + if input.Budget == nil { + decision.Reasons = append(decision.Reasons, ReasonInsufficientBudget) + } else if _, err := input.Budget.Check(ctx, storage.Projection{ + Operation: "enroll:" + session.ID, AdditionalPersistentBytes: cumulative, TemporaryBytes: info.Size(), + }); err != nil { + if !errors.Is(err, storage.ErrBudgetExceeded) { + return Plan{}, err + } + decision.Reasons = append(decision.Reasons, ReasonInsufficientBudget) + } + if len(decision.Reasons) == 0 && len(plan.Selected) >= input.Policy.BatchSize { + decision.Reasons = append(decision.Reasons, ReasonBatchLimit) + } + if len(decision.Reasons) == 0 { + decision.Eligible = true + decision.Selected = true + selectedPersistentBytes = cumulative + plan.Selected = append(plan.Selected, decision) + } + plan.Decisions = append(plan.Decisions, decision) + } + return plan, nil +} + +func safeSessionID(sessionID string) bool { + return sessionID != "" && sessionID != "." && sessionID != ".." && !strings.ContainsAny(sessionID, "/\\\x00") +} + +func enrollmentPersistentBytes(rawBytes int64) (int64, error) { + if rawBytes < 0 { + return 0, errors.New("enrollment byte estimate cannot be negative") + } + overhead := rawBytes/16 + 1<<20 + if rawBytes > math.MaxInt64-overhead { + return 0, errors.New("enrollment byte estimate overflow") + } + estimated := rawBytes + overhead + if estimated > math.MaxInt64/3 { + return 0, errors.New("enrollment byte estimate overflow") + } + return estimated * 3, nil +} + +func addBytes(left int64, right int64) (int64, bool) { + if right > math.MaxInt64-left { + return 0, true + } + return left + right, false +} diff --git a/internal/enroll/planner_test.go b/internal/enroll/planner_test.go new file mode 100644 index 0000000..f69d3ad --- /dev/null +++ b/internal/enroll/planner_test.go @@ -0,0 +1,243 @@ +package enroll + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/storage" +) + +func TestPlannerRequiresStableArchivedSessionAndAllGlobalGates(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "session.jsonl") + if err := os.WriteFile(path, []byte("stable-session\n"), 0o600); err != nil { + t.Fatal(err) + } + now := time.Unix(10_000, 0) + if err := os.Chtimes(path, now.Add(-2*time.Hour), now.Add(-2*time.Hour)); err != nil { + t.Fatal(err) + } + session := codex.Session{ID: "session", RolloutPath: path, Archived: true, UpdatedAt: now.Add(-2 * time.Hour).Unix()} + input := Input{ + Sessions: []codex.Session{session}, Now: now, + Policy: Policy{StableFor: time.Hour, BatchSize: 1, ArchivedOnly: true}, + Gates: Gates{DoctorHealthy: true, CompatibilityApproved: true, MountHealthy: true, CanonicalNamespace: true, EnrollmentAllowed: true}, + Budget: allowingBudget{}, + } + first, err := Build(context.Background(), input) + if err != nil { + t.Fatal(err) + } + assertDecisionReason(t, first, "session", ReasonStabilityPending) + if len(first.Selected) != 0 { + t.Fatalf("first observation selected a session: %#v", first) + } + + input.Now = now.Add(2 * time.Hour) + input.Previous = first.Observations + second, err := Build(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if len(second.Selected) != 1 || second.Selected[0].SessionID != "session" { + t.Fatalf("stable archived session was not selected: %#v", second) + } + + for name, test := range map[string]struct { + mutate func(*Input) + reason Reason + }{ + "doctor": {mutate: func(input *Input) { input.Gates.DoctorHealthy = false }, reason: ReasonDoctorUnhealthy}, + "client": {mutate: func(input *Input) { input.Gates.CompatibilityApproved = false }, reason: ReasonCompatibility}, + "mount": {mutate: func(input *Input) { input.Gates.MountHealthy = false }, reason: ReasonMountUnhealthy}, + "namespace": {mutate: func(input *Input) { input.Gates.CanonicalNamespace = false }, reason: ReasonNamespaceDisabled}, + "stage": {mutate: func(input *Input) { input.Gates.EnrollmentAllowed = false }, reason: ReasonPromotionStage}, + } { + t.Run(name, func(t *testing.T) { + candidate := input + test.mutate(&candidate) + plan, err := Build(context.Background(), candidate) + if err != nil { + t.Fatal(err) + } + assertDecisionReason(t, plan, "session", test.reason) + }) + } +} + +func TestPlannerSeparatesActiveChangingManagedWriterBudgetAndBatchCases(t *testing.T) { + root := t.TempDir() + now := time.Unix(20_000, 0) + makeSession := func(id string, archived bool) codex.Session { + path := filepath.Join(root, id+".jsonl") + if err := os.WriteFile(path, []byte(id+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, now.Add(-2*time.Hour), now.Add(-2*time.Hour)); err != nil { + t.Fatal(err) + } + return codex.Session{ID: id, RolloutPath: path, Archived: archived, UpdatedAt: now.Add(-2 * time.Hour).Unix()} + } + active := makeSession("active", false) + changing := makeSession("changing", true) + managed := makeSession("managed", true) + writer := makeSession("writer", true) + firstBatch := makeSession("batch-a", true) + secondBatch := makeSession("batch-b", true) + budgeted := makeSession("budgeted", true) + previous := make(Observations) + for _, session := range []codex.Session{changing, managed, writer, firstBatch, secondBatch, budgeted} { + info, err := os.Stat(session.RolloutPath) + if err != nil { + t.Fatal(err) + } + previous[session.ID] = Observation{Path: session.RolloutPath, Size: info.Size(), ModTimeUnixNano: info.ModTime().UnixNano(), StableSinceUnixNano: now.Add(-2 * time.Hour).UnixNano()} + } + if err := os.WriteFile(changing.RolloutPath, []byte("changed\n"), 0o600); err != nil { + t.Fatal(err) + } + input := Input{ + Sessions: []codex.Session{active, changing, managed, writer, firstBatch, secondBatch, budgeted}, + Managed: map[string]struct{}{"managed": {}}, Previous: previous, Now: now, + Policy: Policy{StableFor: time.Hour, BatchSize: 1, ArchivedOnly: true}, + Gates: Gates{DoctorHealthy: true, CompatibilityApproved: true, MountHealthy: true, CanonicalNamespace: true, EnrollmentAllowed: true}, + WriterActive: func(_ context.Context, session codex.Session) (bool, error) { return session.ID == "writer", nil }, + Budget: rejectingSessionBudget{sessionID: "budgeted"}, + } + plan, err := Build(context.Background(), input) + if err != nil { + t.Fatal(err) + } + assertDecisionReason(t, plan, "active", ReasonNotArchived) + assertDecisionReason(t, plan, "changing", ReasonFileChanged) + assertDecisionReason(t, plan, "managed", ReasonAlreadyManaged) + assertDecisionReason(t, plan, "writer", ReasonWriterActive) + assertDecisionReason(t, plan, "batch-b", ReasonBatchLimit) + assertDecisionReason(t, plan, "budgeted", ReasonInsufficientBudget) + if len(plan.Selected) != 1 || plan.Selected[0].SessionID != "batch-a" { + t.Fatalf("bounded selection = %#v", plan.Selected) + } +} + +func TestPlannerDiscoversExistingNewAndForkedSessionsAcrossCycles(t *testing.T) { + root := t.TempDir() + now := time.Unix(30_000, 0) + makeSession := func(id string) codex.Session { + path := filepath.Join(root, id+".jsonl") + if err := os.WriteFile(path, []byte(id+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, now.Add(-2*time.Hour), now.Add(-2*time.Hour)); err != nil { + t.Fatal(err) + } + return codex.Session{ID: id, RolloutPath: path, Archived: true, UpdatedAt: now.Add(-2 * time.Hour).Unix()} + } + existing := makeSession("existing") + input := Input{ + Sessions: []codex.Session{existing}, Now: now, + Policy: Policy{StableFor: time.Hour, BatchSize: 3, ArchivedOnly: true}, + Gates: Gates{DoctorHealthy: true, CompatibilityApproved: true, MountHealthy: true, CanonicalNamespace: true, EnrollmentAllowed: true}, + Budget: allowingBudget{}, + } + first, err := Build(context.Background(), input) + if err != nil { + t.Fatal(err) + } + newSession := makeSession("new") + fork := makeSession("fork") + input.Sessions = []codex.Session{existing, newSession, fork} + input.Previous = first.Observations + input.Now = now.Add(2 * time.Hour) + second, err := Build(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if len(second.Selected) != 1 || second.Selected[0].SessionID != "existing" { + t.Fatalf("existing session was not selected while new discoveries observed: %#v", second) + } + assertDecisionReason(t, second, "new", ReasonStabilityPending) + assertDecisionReason(t, second, "fork", ReasonStabilityPending) + + input.Managed = map[string]struct{}{"existing": {}} + input.Previous = second.Observations + input.Now = now.Add(4 * time.Hour) + third, err := Build(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if len(third.Selected) != 2 || third.Selected[0].SessionID != "fork" || third.Selected[1].SessionID != "new" { + t.Fatalf("new and forked sessions were not selected after becoming stable: %#v", third.Selected) + } +} + +func TestApplyRevalidatesFingerprintAndSkipsAlreadyManagedSessions(t *testing.T) { + root := t.TempDir() + first := filepath.Join(root, "first.jsonl") + second := filepath.Join(root, "second.jsonl") + if err := os.WriteFile(first, []byte("first\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(second, []byte("second\n"), 0o600); err != nil { + t.Fatal(err) + } + firstInfo, _ := os.Stat(first) + secondInfo, _ := os.Stat(second) + plan := Plan{Selected: []Decision{ + {SessionID: "first", RolloutPath: first, Selected: true, Fingerprint: Fingerprint{Size: firstInfo.Size(), ModTimeUnixNano: firstInfo.ModTime().UnixNano()}}, + {SessionID: "second", RolloutPath: second, Selected: true, Fingerprint: Fingerprint{Size: secondInfo.Size(), ModTimeUnixNano: secondInfo.ModTime().UnixNano()}}, + }} + if err := os.WriteFile(first, []byte("first changed\n"), 0o600); err != nil { + t.Fatal(err) + } + applied := make([]string, 0) + result, err := Apply(context.Background(), plan, ApplyOptions{ + IsManaged: func(_ context.Context, sessionID string) (bool, error) { return sessionID == "second", nil }, + Apply: func(_ context.Context, decision Decision) error { + applied = append(applied, decision.SessionID) + return nil + }, + }) + if err != nil { + t.Fatal(err) + } + if len(applied) != 0 || result.Applied != 0 || result.SkippedChanged != 1 || result.SkippedManaged != 1 { + t.Fatalf("apply revalidation result = %#v applied=%v", result, applied) + } +} + +type allowingBudget struct{} + +func (allowingBudget) Check(_ context.Context, projection storage.Projection) (storage.Assessment, error) { + return storage.Assessment{Budget: storage.BudgetReport{Operation: projection.Operation, Allowed: true}}, nil +} + +type rejectingSessionBudget struct { + sessionID string +} + +func (b rejectingSessionBudget) Check(_ context.Context, projection storage.Projection) (storage.Assessment, error) { + if projection.Operation == "enroll:"+b.sessionID { + return storage.Assessment{}, storage.ErrBudgetExceeded + } + return storage.Assessment{Budget: storage.BudgetReport{Operation: projection.Operation, Allowed: true}}, nil +} + +func assertDecisionReason(t *testing.T, plan Plan, sessionID string, reason Reason) { + t.Helper() + for _, decision := range plan.Decisions { + if decision.SessionID != sessionID { + continue + } + for _, found := range decision.Reasons { + if found == reason { + return + } + } + t.Fatalf("decision %s reasons = %v, want %s", sessionID, decision.Reasons, reason) + } + t.Fatalf("decision not found: %s", sessionID) +} diff --git a/internal/family/family.go b/internal/family/family.go new file mode 100644 index 0000000..ed9ef44 --- /dev/null +++ b/internal/family/family.go @@ -0,0 +1,507 @@ +package family + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "sort" + + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/contain" +) + +type GraphRelation string + +const ( + GraphSeed GraphRelation = "seed" + GraphAncestor GraphRelation = "ancestor" + GraphDescendant GraphRelation = "descendant" + GraphCollateral GraphRelation = "collateral" + GraphNone GraphRelation = "none" +) + +type Relation string + +const ( + RelationIdentical Relation = "identical-applicable-records" + RelationLeftContained Relation = "left-contained-in-right" + RelationRightContained Relation = "right-contained-in-left" + RelationIndependentTails Relation = "shared-prefix-independent-tails" + RelationSharedRecords Relation = "shared-exact-records" + RelationUnknown Relation = "unknown" +) + +type Member struct { + ID string `json:"id"` + Title string `json:"title"` + CWD string `json:"cwd"` + RolloutPath string `json:"rollout_path"` + Archived bool `json:"archived"` + GitBranch string `json:"git_branch,omitempty"` + RelationToSeed GraphRelation `json:"relation_to_seed"` +} + +type Report struct { + SeedID string `json:"seed_id"` + Members []Member `json:"members"` + Edges []codex.SpawnEdge `json:"edges"` + MissingSessionIDs []string `json:"missing_session_ids,omitempty"` +} + +type Comparison struct { + LeftID string `json:"left_id"` + RightID string `json:"right_id"` + LeftArchived bool `json:"left_archived"` + RightArchived bool `json:"right_archived"` + GraphRelation GraphRelation `json:"graph_relation"` + Relation Relation `json:"relation"` + VerifiedExact bool `json:"verified_exact"` + LeftRecords int64 `json:"left_records"` + RightRecords int64 `json:"right_records"` + SharedPrefixRecords int64 `json:"shared_prefix_records"` + SharedRecords int64 `json:"shared_records"` + LeftContainedInRight bool `json:"left_contained_in_right"` + RightContainedInLeft bool `json:"right_contained_in_left"` +} + +type sourceSnapshot struct { + FileInfo os.FileInfo +} + +var beforeComparisonSourceValidation = func() {} + +func Build(seedID string, sessions []codex.Session, edges []codex.SpawnEdge) (Report, error) { + byID := make(map[string]codex.Session, len(sessions)) + for _, session := range sessions { + byID[session.ID] = session + } + if _, exists := byID[seedID]; !exists { + return Report{}, fmt.Errorf("session not found: %s", seedID) + } + adjacent := make(map[string][]string) + for _, edge := range edges { + adjacent[edge.ParentID] = append(adjacent[edge.ParentID], edge.ChildID) + adjacent[edge.ChildID] = append(adjacent[edge.ChildID], edge.ParentID) + } + component := map[string]struct{}{seedID: {}} + queue := []string{seedID} + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + for _, next := range adjacent[current] { + if _, seen := component[next]; seen { + continue + } + component[next] = struct{}{} + queue = append(queue, next) + } + } + report := Report{SeedID: seedID} + for sessionID := range component { + session, exists := byID[sessionID] + if !exists { + report.MissingSessionIDs = append(report.MissingSessionIDs, sessionID) + continue + } + report.Members = append(report.Members, Member{ + ID: session.ID, Title: session.Title, CWD: session.CWD, RolloutPath: session.RolloutPath, + Archived: session.Archived, GitBranch: session.GitBranch, + RelationToSeed: graphRelation(session.ID, seedID, edges), + }) + } + for _, edge := range edges { + if _, parent := component[edge.ParentID]; !parent { + continue + } + if _, child := component[edge.ChildID]; child { + report.Edges = append(report.Edges, edge) + } + } + sort.Slice(report.Members, func(i, j int) bool { return report.Members[i].ID < report.Members[j].ID }) + sort.Slice(report.Edges, func(i, j int) bool { + if report.Edges[i].ParentID != report.Edges[j].ParentID { + return report.Edges[i].ParentID < report.Edges[j].ParentID + } + if report.Edges[i].ChildID != report.Edges[j].ChildID { + return report.Edges[i].ChildID < report.Edges[j].ChildID + } + return report.Edges[i].Status < report.Edges[j].Status + }) + sort.Strings(report.MissingSessionIDs) + return report, nil +} + +func Compare(ctx context.Context, left codex.Session, right codex.Session, edges []codex.SpawnEdge) (Comparison, error) { + if left.ID == "" || right.ID == "" || left.ID == right.ID || left.RolloutPath == "" || right.RolloutPath == "" { + return Comparison{}, errors.New("distinct sessions with rollout paths are required") + } + leftFile, err := os.Open(left.RolloutPath) + if err != nil { + return Comparison{}, fmt.Errorf("open left rollout: %w", err) + } + defer func() { _ = leftFile.Close() }() + rightFile, err := os.Open(right.RolloutPath) + if err != nil { + return Comparison{}, fmt.Errorf("open right rollout: %w", err) + } + defer func() { _ = rightFile.Close() }() + leftSnapshot, err := snapshotFile(leftFile) + if err != nil { + return Comparison{}, fmt.Errorf("stat left rollout: %w", err) + } + rightSnapshot, err := snapshotFile(rightFile) + if err != nil { + return Comparison{}, fmt.Errorf("stat right rollout: %w", err) + } + leftRecords, err := scanFile(ctx, leftFile) + if err != nil { + return Comparison{}, fmt.Errorf("scan left rollout: %w", err) + } + rightRecords, err := scanFile(ctx, rightFile) + if err != nil { + return Comparison{}, fmt.Errorf("scan right rollout: %w", err) + } + if len(leftRecords) == 0 || len(rightRecords) == 0 { + return Comparison{}, errors.New("both rollouts must contain comparable records") + } + result := Comparison{ + LeftID: left.ID, RightID: right.ID, LeftArchived: left.Archived, RightArchived: right.Archived, + GraphRelation: graphRelation(left.ID, right.ID, edges), Relation: RelationUnknown, + LeftRecords: int64(len(leftRecords)), RightRecords: int64(len(rightRecords)), + } + result.SharedPrefixRecords, err = sharedPrefix(ctx, leftFile, leftRecords, rightFile, rightRecords) + if err != nil { + return Comparison{}, err + } + result.SharedRecords, err = sharedRecordCount(ctx, leftFile, leftRecords, rightFile, rightRecords) + if err != nil { + return Comparison{}, err + } + leftContained, err := contain.Check(ctx, + contain.Input{ID: left.ID, Path: left.RolloutPath}, + contain.Input{ID: right.ID, Path: right.RolloutPath}, + contain.Options{IgnoreSessionMeta: true}, + ) + if err != nil { + return Comparison{}, err + } + beforeComparisonSourceValidation() + if err := verifySourceUnchanged(left.RolloutPath, leftFile, leftSnapshot); err != nil { + return Comparison{}, fmt.Errorf("left rollout changed during comparison: %w", err) + } + if err := verifySourceUnchanged(right.RolloutPath, rightFile, rightSnapshot); err != nil { + return Comparison{}, fmt.Errorf("right rollout changed during comparison: %w", err) + } + rightContained, err := contain.Check(ctx, + contain.Input{ID: right.ID, Path: right.RolloutPath}, + contain.Input{ID: left.ID, Path: left.RolloutPath}, + contain.Options{IgnoreSessionMeta: true}, + ) + if err != nil { + return Comparison{}, err + } + result.LeftContainedInRight = leftContained.Contained && leftContained.VerifiedExact + result.RightContainedInLeft = rightContained.Contained && rightContained.VerifiedExact + switch { + case result.LeftContainedInRight && result.RightContainedInLeft: + result.Relation = RelationIdentical + result.VerifiedExact = true + case result.LeftContainedInRight: + result.Relation = RelationLeftContained + result.VerifiedExact = true + case result.RightContainedInLeft: + result.Relation = RelationRightContained + result.VerifiedExact = true + case result.SharedPrefixRecords > 0: + result.Relation = RelationIndependentTails + result.VerifiedExact = true + case result.SharedRecords > 0: + result.Relation = RelationSharedRecords + result.VerifiedExact = true + } + return result, nil +} + +func graphRelation(leftID string, rightID string, edges []codex.SpawnEdge) GraphRelation { + if leftID == rightID { + return GraphSeed + } + if reachable(leftID, rightID, edges) { + return GraphAncestor + } + if reachable(rightID, leftID, edges) { + return GraphDescendant + } + if connected(leftID, rightID, edges) { + return GraphCollateral + } + return GraphNone +} + +func reachable(start string, target string, edges []codex.SpawnEdge) bool { + children := make(map[string][]string) + for _, edge := range edges { + children[edge.ParentID] = append(children[edge.ParentID], edge.ChildID) + } + seen := map[string]struct{}{start: {}} + queue := []string{start} + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + for _, child := range children[current] { + if child == target { + return true + } + if _, exists := seen[child]; exists { + continue + } + seen[child] = struct{}{} + queue = append(queue, child) + } + } + return false +} + +func connected(start string, target string, edges []codex.SpawnEdge) bool { + adjacent := make(map[string][]string) + for _, edge := range edges { + adjacent[edge.ParentID] = append(adjacent[edge.ParentID], edge.ChildID) + adjacent[edge.ChildID] = append(adjacent[edge.ChildID], edge.ParentID) + } + seen := map[string]struct{}{start: {}} + queue := []string{start} + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + for _, next := range adjacent[current] { + if next == target { + return true + } + if _, exists := seen[next]; exists { + continue + } + seen[next] = struct{}{} + queue = append(queue, next) + } + } + return false +} + +type record struct { + digest [sha256.Size]byte + size int64 + start int64 + end int64 +} + +func scanFile(ctx context.Context, file *os.File) ([]record, error) { + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, err + } + reader := bufio.NewReaderSize(file, 1024*1024) + records := make([]record, 0, 1024) + var offset int64 + var physicalIndex int64 + for { + if err := ctx.Err(); err != nil { + return nil, err + } + hasher := sha256.New() + start := offset + var size int64 + var firstCapture []byte + captureComplete := true + hasData := false + reachedEOF := false + for { + fragment, readErr := reader.ReadSlice('\n') + if len(fragment) > 0 { + hasData = true + _, _ = hasher.Write(fragment) + size += int64(len(fragment)) + offset += int64(len(fragment)) + if physicalIndex == 0 && captureComplete { + if len(firstCapture)+len(fragment) <= 8*1024*1024 { + firstCapture = append(firstCapture, fragment...) + } else { + firstCapture = nil + captureComplete = false + } + } + } + switch { + case readErr == nil: + goto complete + case errors.Is(readErr, bufio.ErrBufferFull): + continue + case errors.Is(readErr, io.EOF): + reachedEOF = true + goto complete + default: + return nil, readErr + } + } + complete: + if !hasData { + return records, nil + } + skip := physicalIndex == 0 && captureComplete && isSessionMeta(firstCapture) + if !skip { + var digest [sha256.Size]byte + copy(digest[:], hasher.Sum(nil)) + records = append(records, record{digest: digest, size: size, start: start, end: offset}) + } + physicalIndex++ + if reachedEOF { + return records, nil + } + } +} + +func isSessionMeta(data []byte) bool { + var envelope struct { + Type string `json:"type"` + } + return json.Unmarshal(bytes.TrimSuffix(data, []byte{'\n'}), &envelope) == nil && envelope.Type == "session_meta" +} + +func sharedPrefix(ctx context.Context, leftFile *os.File, left []record, rightFile *os.File, right []record) (int64, error) { + limit := min(len(left), len(right)) + var count int64 + for index := 0; index < limit; index++ { + if left[index].size != right[index].size || left[index].digest != right[index].digest { + break + } + exact, err := equalRanges(ctx, leftFile, left[index], rightFile, right[index]) + if err != nil { + return 0, err + } + if !exact { + break + } + count++ + } + return count, nil +} + +func sharedRecordCount(ctx context.Context, leftFile *os.File, left []record, rightFile *os.File, right []record) (int64, error) { + type key struct { + digest [sha256.Size]byte + size int64 + } + candidates := make(map[key][]int) + for index, item := range right { + candidates[key{digest: item.digest, size: item.size}] = append(candidates[key{digest: item.digest, size: item.size}], index) + } + used := make([]bool, len(right)) + next := make(map[key]int, len(candidates)) + collisions := make(map[key][]int) + var shared int64 + for _, leftRecord := range left { + fingerprint := key{digest: leftRecord.digest, size: leftRecord.size} + matched := false + for _, index := range collisions[fingerprint] { + if used[index] { + continue + } + exact, err := equalRanges(ctx, leftFile, leftRecord, rightFile, right[index]) + if err != nil { + return 0, err + } + if exact { + used[index] = true + shared++ + matched = true + break + } + } + if matched { + continue + } + positions := candidates[fingerprint] + for next[fingerprint] < len(positions) { + index := positions[next[fingerprint]] + next[fingerprint]++ + exact, err := equalRanges(ctx, leftFile, leftRecord, rightFile, right[index]) + if err != nil { + return 0, err + } + if exact { + used[index] = true + shared++ + matched = true + break + } + collisions[fingerprint] = append(collisions[fingerprint], index) + } + } + return shared, nil +} + +func equalRanges(ctx context.Context, leftFile *os.File, left record, rightFile *os.File, right record) (bool, error) { + if left.size != right.size { + return false, nil + } + leftReader := io.NewSectionReader(leftFile, left.start, left.size) + rightReader := io.NewSectionReader(rightFile, right.start, right.size) + leftBuffer := make([]byte, 128*1024) + rightBuffer := make([]byte, len(leftBuffer)) + for remaining := left.size; remaining > 0; { + if err := ctx.Err(); err != nil { + return false, err + } + chunk := int64(len(leftBuffer)) + if remaining < chunk { + chunk = remaining + } + if _, err := io.ReadFull(leftReader, leftBuffer[:chunk]); err != nil { + return false, err + } + if _, err := io.ReadFull(rightReader, rightBuffer[:chunk]); err != nil { + return false, err + } + if !bytes.Equal(leftBuffer[:chunk], rightBuffer[:chunk]) { + return false, nil + } + remaining -= chunk + } + return true, nil +} + +func snapshotFile(file *os.File) (sourceSnapshot, error) { + info, err := file.Stat() + if err != nil { + return sourceSnapshot{}, err + } + if !info.Mode().IsRegular() { + return sourceSnapshot{}, errors.New("rollout path is not a regular file") + } + return sourceSnapshot{FileInfo: info}, nil +} + +func verifySourceUnchanged(path string, file *os.File, before sourceSnapshot) error { + afterHandle, err := file.Stat() + if err != nil { + return err + } + afterPath, err := os.Stat(path) + if err != nil { + return err + } + if !os.SameFile(before.FileInfo, afterHandle) || !os.SameFile(before.FileInfo, afterPath) { + return errors.New("rollout identity changed") + } + if before.FileInfo.Size() != afterHandle.Size() || !before.FileInfo.ModTime().Equal(afterHandle.ModTime()) { + return errors.New("rollout size or modification time changed") + } + if afterHandle.Size() != afterPath.Size() || !afterHandle.ModTime().Equal(afterPath.ModTime()) { + return errors.New("rollout path state differs from open file") + } + return nil +} diff --git a/internal/family/family_test.go b/internal/family/family_test.go new file mode 100644 index 0000000..8ddeedc --- /dev/null +++ b/internal/family/family_test.go @@ -0,0 +1,175 @@ +package family + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/samekind/codexfold/internal/codex" +) + +func TestBuildReportsConnectedFamilyStateWithoutInferringUsefulness(t *testing.T) { + sessions := []codex.Session{ + {ID: "root", Title: "Root", RolloutPath: "/rollouts/root.jsonl", Archived: false}, + {ID: "child", Title: "Child", RolloutPath: "/rollouts/child.jsonl", Archived: true}, + {ID: "grandchild", Title: "Grandchild", RolloutPath: "/rollouts/grandchild.jsonl", Archived: false}, + {ID: "unrelated", Title: "Unrelated", RolloutPath: "/rollouts/unrelated.jsonl", Archived: true}, + } + edges := []codex.SpawnEdge{ + {ParentID: "root", ChildID: "child", Status: "closed"}, + {ParentID: "child", ChildID: "grandchild", Status: "open"}, + {ParentID: "root", ChildID: "missing", Status: "closed"}, + } + report, err := Build("child", sessions, edges) + if err != nil { + t.Fatal(err) + } + if len(report.Members) != 3 || len(report.Edges) != 3 || len(report.MissingSessionIDs) != 1 || report.MissingSessionIDs[0] != "missing" { + t.Fatalf("family report = %#v", report) + } + byID := make(map[string]Member) + for _, member := range report.Members { + byID[member.ID] = member + } + if byID["child"].RelationToSeed != GraphSeed || !byID["child"].Archived { + t.Fatalf("seed member = %#v", byID["child"]) + } + if byID["root"].RelationToSeed != GraphAncestor || byID["grandchild"].RelationToSeed != GraphDescendant { + t.Fatalf("graph relations = %#v", byID) + } +} + +func TestCompareClassifiesExactContainmentIndependentTailsAndUnknown(t *testing.T) { + root := t.TempDir() + write := func(name string, records ...string) codex.Session { + path := filepath.Join(root, name+".jsonl") + data := "" + for _, record := range records { + data += record + "\n" + } + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + return codex.Session{ID: name, RolloutPath: path, Archived: name == "left"} + } + metaLeft := `{"type":"session_meta","id":"left"}` + metaRight := `{"type":"session_meta","id":"right"}` + a := `{"value":"a"}` + b := `{"value":"b"}` + c := `{"value":"c"}` + x := `{"value":"x"}` + y := `{"value":"y"}` + + identical, err := Compare(context.Background(), + write("identical-left", metaLeft, a, b), write("identical-right", metaRight, a, b), nil, + ) + if err != nil || identical.Relation != RelationIdentical || !identical.VerifiedExact { + t.Fatalf("identical comparison = %#v err=%v", identical, err) + } + + contained, err := Compare(context.Background(), + write("left", metaLeft, a, b), write("container", metaRight, x, a, b, y), nil, + ) + if err != nil || contained.Relation != RelationLeftContained || !contained.LeftContainedInRight || !contained.VerifiedExact { + t.Fatalf("contained comparison = %#v err=%v", contained, err) + } + + tails, err := Compare(context.Background(), + write("tail-left", metaLeft, a, b), write("tail-right", metaRight, a, c), nil, + ) + if err != nil || tails.Relation != RelationIndependentTails || tails.SharedPrefixRecords != 1 || tails.SharedRecords != 1 { + t.Fatalf("independent-tail comparison = %#v err=%v", tails, err) + } + + unknown, err := Compare(context.Background(), + write("unknown-left", metaLeft, x), write("unknown-right", metaRight, y), nil, + ) + if err != nil || unknown.Relation != RelationUnknown || unknown.SharedRecords != 0 { + t.Fatalf("unknown comparison = %#v err=%v", unknown, err) + } +} + +func TestCompareReportsGraphEvidenceSeparatelyFromContent(t *testing.T) { + root := t.TempDir() + leftPath := filepath.Join(root, "left.jsonl") + rightPath := filepath.Join(root, "right.jsonl") + if err := os.WriteFile(leftPath, []byte("{\"v\":1}\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(rightPath, []byte("{\"v\":2}\n"), 0o600); err != nil { + t.Fatal(err) + } + comparison, err := Compare(context.Background(), + codex.Session{ID: "left", RolloutPath: leftPath}, + codex.Session{ID: "right", RolloutPath: rightPath}, + []codex.SpawnEdge{{ParentID: "left", ChildID: "right", Status: "open"}}, + ) + if err != nil || comparison.GraphRelation != GraphAncestor || comparison.Relation != RelationUnknown { + t.Fatalf("graph/content evidence was conflated: %#v err=%v", comparison, err) + } +} + +func TestCompareRejectsSourceMutationBeforeReturningEvidence(t *testing.T) { + root := t.TempDir() + leftPath := filepath.Join(root, "left.jsonl") + rightPath := filepath.Join(root, "right.jsonl") + data := []byte("{\"type\":\"session_meta\"}\n{\"value\":1}\n") + if err := os.WriteFile(leftPath, data, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(rightPath, data, 0o600); err != nil { + t.Fatal(err) + } + previous := beforeComparisonSourceValidation + beforeComparisonSourceValidation = func() { + file, err := os.OpenFile(rightPath, os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteString("{\"mutated\":true}\n"); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { beforeComparisonSourceValidation = previous }) + _, err := Compare(context.Background(), + codex.Session{ID: "left", RolloutPath: leftPath}, + codex.Session{ID: "right", RolloutPath: rightPath}, nil, + ) + if err == nil || !strings.Contains(err.Error(), "changed during comparison") { + t.Fatalf("source mutation error = %v", err) + } +} + +func TestCompareRepeatedRecordsAvoidsQuadraticFileReopens(t *testing.T) { + root := t.TempDir() + writeRepeated := func(name string) codex.Session { + path := filepath.Join(root, name+".jsonl") + var data strings.Builder + data.WriteString("{\"type\":\"session_meta\",\"id\":\"") + data.WriteString(name) + data.WriteString("\"}\n") + for range 1000 { + data.WriteString("{\"value\":\"same repeated record\"}\n") + } + if err := os.WriteFile(path, []byte(data.String()), 0o600); err != nil { + t.Fatal(err) + } + return codex.Session{ID: name, RolloutPath: path} + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + comparison, err := Compare(ctx, writeRepeated("left"), writeRepeated("right"), nil) + if err != nil { + t.Fatal(err) + } + if comparison.Relation != RelationIdentical || comparison.SharedRecords != 1000 { + t.Fatalf("repeated comparison = %#v", comparison) + } +} diff --git a/internal/fold/doctor.go b/internal/fold/doctor.go index b04a7e4..72537e9 100644 --- a/internal/fold/doctor.go +++ b/internal/fold/doctor.go @@ -2,11 +2,13 @@ package fold import ( "context" + "errors" "fmt" "io/fs" "os" "path/filepath" - "strings" + + "github.com/samekind/codexfold/internal/storage" ) type DoctorIssue struct { @@ -16,13 +18,21 @@ type DoctorIssue struct { } type DoctorResult struct { - StoreDir string `json:"store_dir"` - ManifestCount int `json:"manifest_count"` - VerifiedManifestCount int `json:"verified_manifest_count"` - ObjectReferenceCount int `json:"object_reference_count"` - UniqueObjectCount int `json:"unique_object_count"` - IssueCount int `json:"issue_count"` - Issues []DoctorIssue `json:"issues"` + StoreDir string `json:"store_dir"` + ManifestCount int `json:"manifest_count"` + VerifiedManifestCount int `json:"verified_manifest_count"` + ObjectReferenceCount int `json:"object_reference_count"` + UniqueObjectCount int `json:"unique_object_count"` + IssueCount int `json:"issue_count"` + Issues []DoctorIssue `json:"issues"` + Storage storage.Inventory `json:"storage"` + StorageLimits storage.Limits `json:"storage_limits"` + AvailableBytes int64 `json:"available_bytes"` +} + +type loadedManifest struct { + Path string + Manifest Manifest } func Doctor(ctx context.Context, storeDir string) (DoctorResult, error) { @@ -35,7 +45,8 @@ func Doctor(ctx context.Context, storeDir string) (DoctorResult, error) { result.Issues = append(result.Issues, loadIssues...) store := NewObjectStore(storeDir) unique := make(map[string]ObjectRef) - for _, manifest := range manifests { + for _, loaded := range manifests { + manifest := loaded.Manifest if err := ctx.Err(); err != nil { return DoctorResult{}, err } @@ -45,7 +56,7 @@ func Doctor(ctx context.Context, storeDir string) (DoctorResult, error) { } if err := verifyStoredManifest(ctx, store, manifest); err != nil { result.Issues = append(result.Issues, DoctorIssue{ - Scope: "manifest", Path: ManifestPath(storeDir, manifest.Session.ID), Error: err.Error(), + Scope: "manifest", Path: loaded.Path, Error: err.Error(), }) } else { result.VerifiedManifestCount++ @@ -62,32 +73,50 @@ func Doctor(ctx context.Context, storeDir string) (DoctorResult, error) { }) } } + result.Storage, err = storage.Scan(ctx, storage.Options{StoreDir: storeDir, AllowMetadataIssues: true}) + if err != nil { + result.Issues = append(result.Issues, DoctorIssue{Scope: "storage", Path: storeDir, Error: err.Error()}) + } else { + for _, issue := range result.Storage.Issues { + result.Issues = append(result.Issues, DoctorIssue{Scope: "storage", Path: storeDir, Error: issue}) + } + } + result.StorageLimits, err = storage.LoadLimits(storeDir) + if err != nil { + result.Issues = append(result.Issues, DoctorIssue{Scope: "storage-policy", Path: filepath.Join(storeDir, storage.PolicyFilename), Error: err.Error()}) + } + result.AvailableBytes, err = storage.AvailableBytes(storeDir) + if err != nil { + result.Issues = append(result.Issues, DoctorIssue{Scope: "storage-space", Path: storeDir, Error: err.Error()}) + } result.IssueCount = len(result.Issues) return result, nil } -func loadAllManifests(storeDir string) ([]Manifest, []DoctorIssue, error) { +func loadAllManifests(storeDir string) ([]loadedManifest, []DoctorIssue, error) { manifestDir := filepath.Join(storeDir, "manifests") - entries, err := os.ReadDir(manifestDir) - if err != nil { - if os.IsNotExist(err) { - return []Manifest{}, []DoctorIssue{}, nil - } - return nil, nil, fmt.Errorf("read manifest directory: %w", err) - } - manifests := make([]Manifest, 0, len(entries)) + manifests := make([]loadedManifest, 0) issues := make([]DoctorIssue, 0) - for _, entry := range entries { + err := filepath.WalkDir(manifestDir, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { - continue + return nil } - sessionID := strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())) - manifest, err := LoadManifest(storeDir, sessionID) + manifest, err := LoadManifestPath(path) if err != nil { - issues = append(issues, DoctorIssue{Scope: "manifest", Path: filepath.Join(manifestDir, entry.Name()), Error: err.Error()}) - continue + issues = append(issues, DoctorIssue{Scope: "manifest", Path: path, Error: err.Error()}) + return nil } - manifests = append(manifests, manifest) + manifests = append(manifests, loadedManifest{Path: path, Manifest: manifest}) + return nil + }) + if errors.Is(err, os.ErrNotExist) { + return []loadedManifest{}, []DoctorIssue{}, nil + } + if err != nil { + return nil, nil, fmt.Errorf("read manifest directory: %w", err) } return manifests, issues, nil } diff --git a/internal/fold/doctor_gc_test.go b/internal/fold/doctor_gc_test.go index add712e..02f4dc9 100644 --- a/internal/fold/doctor_gc_test.go +++ b/internal/fold/doctor_gc_test.go @@ -7,8 +7,7 @@ import ( "path/filepath" "strings" "testing" - - "github.com/jstar0/codexfold/internal/codex" + "time" ) func TestDoctorDetectsReferencedObjectCorruption(t *testing.T) { @@ -18,7 +17,7 @@ func TestDoctorDetectsReferencedObjectCorruption(t *testing.T) { if err := os.WriteFile(sourcePath, []byte("{\"value\":\"large-field-value\"}\n"), 0o644); err != nil { t.Fatalf("write source: %v", err) } - if _, err := Fold(context.Background(), codex.Session{ID: "doctor", RolloutPath: sourcePath, Archived: true}, FoldOptions{ + if _, err := Fold(context.Background(), Session{ID: "doctor", RolloutPath: sourcePath, Archived: true}, FoldOptions{ StoreDir: storeDir, Apply: true, FieldThreshold: 4, }); err != nil { t.Fatalf("Fold returned error: %v", err) @@ -30,6 +29,9 @@ func TestDoctorDetectsReferencedObjectCorruption(t *testing.T) { if clean.IssueCount != 0 || clean.ManifestCount != 1 { t.Fatalf("unexpected clean doctor result: %#v", clean) } + if clean.Storage.LogicalSessionBytes == 0 || clean.Storage.TotalPhysicalBytes == 0 || clean.StorageLimits.MaxPhysicalBytes == 0 || clean.AvailableBytes == 0 { + t.Fatalf("doctor storage accounting is incomplete: %#v", clean) + } manifest, err := LoadManifest(storeDir, "doctor") if err != nil { t.Fatalf("load manifest: %v", err) @@ -54,7 +56,7 @@ func TestGCDryRunAndApplyRemoveOnlyUnreferencedObjects(t *testing.T) { if err := os.WriteFile(sourcePath, []byte("{\"value\":\"large-field-value\"}\n"), 0o644); err != nil { t.Fatalf("write source: %v", err) } - if _, err := Fold(context.Background(), codex.Session{ID: "gc", RolloutPath: sourcePath, Archived: true}, FoldOptions{ + if _, err := Fold(context.Background(), Session{ID: "gc", RolloutPath: sourcePath, Archived: true}, FoldOptions{ StoreDir: storeDir, Apply: true, FieldThreshold: 4, }); err != nil { t.Fatalf("Fold returned error: %v", err) @@ -90,6 +92,82 @@ func TestGCDryRunAndApplyRemoveOnlyUnreferencedObjects(t *testing.T) { } } +func TestGCIncludesBoundedGenerationAndTemporaryCleanup(t *testing.T) { + root := t.TempDir() + storeDir := filepath.Join(root, "store") + sourcePath := filepath.Join(root, "rollout.jsonl") + if err := os.WriteFile(sourcePath, []byte("{\"value\":\"storage-gc\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Fold(context.Background(), Session{ID: "session", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 4}); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(storeDir, "packs"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(storeDir, "packs", "CURRENT"), []byte("gen-3\n"), 0o600); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-2 * time.Hour) + for _, generation := range []string{"gen-1", "gen-2", "gen-3"} { + directory := filepath.Join(storeDir, "packs", generation) + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(directory, "pack-000001.pack"), []byte(generation), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(directory, old.Add(time.Duration(generation[len(generation)-1]-'0')*time.Minute), old.Add(time.Duration(generation[len(generation)-1]-'0')*time.Minute)); err != nil { + t.Fatal(err) + } + } + temporary := filepath.Join(storeDir, ".backing-abandoned.tmp") + if err := os.WriteFile(temporary, []byte("temporary"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(temporary, old, old); err != nil { + t.Fatal(err) + } + + result, err := GC(context.Background(), storeDir, true) + if err != nil { + t.Fatalf("GC: %v", err) + } + if result.Storage.RemovedCount != 2 || result.ActualReclaimedBytes <= 0 { + t.Fatalf("bounded storage was not collected: %#v", result) + } + if _, err := os.Stat(filepath.Join(storeDir, "packs", "gen-1")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("old pack generation remains: %v", err) + } + if _, err := os.Stat(filepath.Join(storeDir, "packs", "gen-2")); err != nil { + t.Fatalf("previous pack generation was removed: %v", err) + } + if _, err := os.Stat(temporary); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("abandoned temporary remains: %v", err) + } +} + +func TestDoctorAndGCKeepGenerationManifestObjects(t *testing.T) { + root := t.TempDir() + storeDir := filepath.Join(root, "store") + sourcePath := filepath.Join(root, "generation.jsonl") + if err := os.WriteFile(sourcePath, []byte("{\"value\":\"generation-only-field\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + manifestPath := filepath.Join(storeDir, "manifests", "generations", "session", "2.json") + if _, err := Fold(context.Background(), Session{ID: "session", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, ManifestPathOverride: manifestPath, Apply: true, FieldThreshold: 4}); err != nil { + t.Fatalf("Fold generation: %v", err) + } + doctor, err := Doctor(context.Background(), storeDir) + if err != nil || doctor.ManifestCount != 1 || doctor.IssueCount != 0 { + t.Fatalf("generation manifest not covered by doctor: %#v err=%v", doctor, err) + } + gc, err := GC(context.Background(), storeDir, true) + if err != nil || gc.OrphanCount != 0 || gc.Referenced == 0 { + t.Fatalf("generation object treated as orphan: %#v err=%v", gc, err) + } +} + func TestRemoveSourceRequiresGuardAndCanMaterializeAgain(t *testing.T) { root := t.TempDir() storeDir := filepath.Join(root, "store") @@ -98,13 +176,13 @@ func TestRemoveSourceRequiresGuardAndCanMaterializeAgain(t *testing.T) { if err := os.WriteFile(sourcePath, source, 0o644); err != nil { t.Fatalf("write source: %v", err) } - _, err := Fold(context.Background(), codex.Session{ID: "active", RolloutPath: sourcePath}, FoldOptions{ + _, err := Fold(context.Background(), Session{ID: "active", RolloutPath: sourcePath}, FoldOptions{ StoreDir: storeDir, Apply: true, RemoveSource: true, FieldThreshold: 4, }) if err == nil { t.Fatalf("non-archived source removal should require --allow-active") } - result, err := Fold(context.Background(), codex.Session{ID: "archived", RolloutPath: sourcePath, Archived: true}, FoldOptions{ + result, err := Fold(context.Background(), Session{ID: "archived", RolloutPath: sourcePath, Archived: true}, FoldOptions{ StoreDir: storeDir, Apply: true, RemoveSource: true, FieldThreshold: 4, }) if err != nil { diff --git a/internal/fold/fold.go b/internal/fold/fold.go index ac7478a..0874e68 100644 --- a/internal/fold/fold.go +++ b/internal/fold/fold.go @@ -10,45 +10,59 @@ import ( "fmt" "hash" "io" + "math" "os" + "path/filepath" + "strings" - "github.com/jstar0/codexfold/internal/cdc" - "github.com/jstar0/codexfold/internal/codex" - "github.com/jstar0/codexfold/internal/jsonraw" + "github.com/samekind/codexfold/internal/cdc" + "github.com/samekind/codexfold/internal/jsonraw" + "github.com/samekind/codexfold/internal/storage" ) type FoldOptions struct { - StoreDir string - Apply bool - Overwrite bool - RemoveSource bool - AllowActive bool - FieldThreshold int64 - MaxJSONLineBytes int64 - CDC cdc.Options - beforeCommit func() error + StoreDir string + ManifestPathOverride string + Apply bool + Overwrite bool + RemoveSource bool + AllowActive bool + FieldThreshold int64 + MaxJSONLineBytes int64 + CDC cdc.Options + Budget storage.Checker + beforeCommit func() error +} + +type Session struct { + ID string + Title string + CWD string + RolloutPath string + Archived bool } type FoldResult struct { - SessionID string `json:"session_id"` - SourcePath string `json:"source_path"` - ManifestPath string `json:"manifest_path"` - SourceBytes int64 `json:"source_bytes"` - SourceSHA256 string `json:"source_sha256"` - PartCount int `json:"part_count"` - FieldParts int `json:"field_parts"` - ResidualParts int `json:"residual_parts"` - UniqueObjects int `json:"unique_objects"` - ReusedObjects int `json:"reused_objects"` - NewStoredBytes int64 `json:"new_stored_bytes"` - OversizedLines int64 `json:"oversized_lines"` - InvalidJSONLines int64 `json:"invalid_json_lines"` - Verified bool `json:"verified"` - DryRun bool `json:"dry_run"` - RemovedSource bool `json:"removed_source"` + SessionID string `json:"session_id"` + SourcePath string `json:"source_path"` + ManifestPath string `json:"manifest_path"` + SourceBytes int64 `json:"source_bytes"` + SourceSHA256 string `json:"source_sha256"` + PartCount int `json:"part_count"` + FieldParts int `json:"field_parts"` + ResidualParts int `json:"residual_parts"` + UniqueObjects int `json:"unique_objects"` + ReusedObjects int `json:"reused_objects"` + NewStoredBytes int64 `json:"new_stored_bytes"` + OversizedLines int64 `json:"oversized_lines"` + InvalidJSONLines int64 `json:"invalid_json_lines"` + Verified bool `json:"verified"` + DryRun bool `json:"dry_run"` + RemovedSource bool `json:"removed_source"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` } -func Fold(ctx context.Context, session codex.Session, options FoldOptions) (FoldResult, error) { +func Fold(ctx context.Context, session Session, options FoldOptions) (FoldResult, error) { if options.StoreDir == "" { return FoldResult{}, errors.New("fold store directory is required") } @@ -68,6 +82,14 @@ func Fold(ctx context.Context, session codex.Session, options FoldOptions) (Fold options.CDC = cdc.Options{MinBytes: 4 * 1024, AverageBytes: 16 * 1024, MaxBytes: 64 * 1024} } manifestPath := ManifestPath(options.StoreDir, session.ID) + if options.ManifestPathOverride != "" { + manifestPath = filepath.Clean(options.ManifestPathOverride) + manifestRoot := filepath.Join(options.StoreDir, "manifests") + relative, err := filepath.Rel(manifestRoot, manifestPath) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return FoldResult{}, errors.New("manifest override must remain inside the fold manifest directory") + } + } if options.Apply && !options.Overwrite { if _, err := os.Stat(manifestPath); err == nil { return FoldResult{}, fmt.Errorf("fold manifest already exists: %s", manifestPath) @@ -90,6 +112,38 @@ func Fold(ctx context.Context, session codex.Session, options FoldOptions) (Fold if err != nil { return FoldResult{}, fmt.Errorf("stat rollout: %w", err) } + var storageAssessment storage.Assessment + if options.Apply { + budget := options.Budget + if budget == nil { + guard, err := storage.DefaultGuard(options.StoreDir) + if err != nil { + return FoldResult{}, err + } + budget = guard + } + persistentBytes, err := estimateFoldStorageBytes(before.Size()) + if err != nil { + return FoldResult{}, err + } + maximumObjectBytes := min(before.Size(), max(options.CDC.MaxBytes, options.MaxJSONLineBytes)) + temporaryBytes, err := estimateFoldStorageBytes(maximumObjectBytes) + if err != nil { + return FoldResult{}, err + } + reclaimableBytes := int64(0) + if options.RemoveSource { + reclaimableBytes = before.Size() + } + storageAssessment, err = budget.Check(ctx, storage.Projection{ + Operation: "fold", AdditionalPersistentBytes: persistentBytes, + TemporaryBytes: temporaryBytes, TemporaryPersistentOverlapBytes: min(temporaryBytes, persistentBytes), + ReclaimableBytes: reclaimableBytes, + }) + if err != nil { + return FoldResult{}, err + } + } manifest := Manifest{ Version: ManifestVersion, @@ -260,7 +314,7 @@ complete: if err := store.SyncPending(ctx); err != nil { return FoldResult{}, err } - if err := writeManifest(options.StoreDir, manifest, options.Overwrite); err != nil { + if err := writeManifestPath(manifestPath, manifest, options.Overwrite); err != nil { return FoldResult{}, err } if options.RemoveSource { @@ -269,9 +323,22 @@ complete: } result.RemovedSource = true } + result.Storage = storage.CompleteAccounting(ctx, storageAssessment, options.StoreDir) return result, nil } +func estimateFoldStorageBytes(rawBytes int64) (int64, error) { + const fixedOverhead = int64(1 << 20) + if rawBytes < 0 { + return 0, errors.New("fold byte estimate cannot be negative") + } + overhead := rawBytes/16 + fixedOverhead + if rawBytes > math.MaxInt64-overhead { + return 0, errors.New("fold byte estimate overflow") + } + return rawBytes + overhead, nil +} + func verifyCurrentSource(path string, initial os.FileInfo, source ManifestSource) error { file, err := os.Open(path) if err != nil { diff --git a/internal/fold/fold_test.go b/internal/fold/fold_test.go index b17ee24..a0a1d2a 100644 --- a/internal/fold/fold_test.go +++ b/internal/fold/fold_test.go @@ -9,8 +9,8 @@ import ( "strings" "testing" - "github.com/jstar0/codexfold/internal/cdc" - "github.com/jstar0/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/cdc" + "github.com/samekind/codexfold/internal/storage" ) func TestFoldRejectsSourceMutationBeforeManifestCommit(t *testing.T) { @@ -21,7 +21,7 @@ func TestFoldRejectsSourceMutationBeforeManifestCommit(t *testing.T) { t.Fatalf("write source: %v", err) } - _, err := Fold(context.Background(), codex.Session{ + _, err := Fold(context.Background(), Session{ ID: "changing", RolloutPath: sourcePath, Archived: true, }, FoldOptions{ StoreDir: storeDir, Apply: true, FieldThreshold: 4, @@ -55,6 +55,91 @@ func TestFoldRejectsSourceMutationBeforeManifestCommit(t *testing.T) { } } +func TestFoldBudgetRejectsBeforeWritingObjectsOrManifest(t *testing.T) { + root := t.TempDir() + sourcePath := filepath.Join(root, "rollout.jsonl") + storeDir := filepath.Join(root, "store") + if err := os.WriteFile(sourcePath, []byte("{\"value\":\"budgeted-field\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + checker := &foldRejectingChecker{} + _, err := Fold(context.Background(), Session{ID: "budget", RolloutPath: sourcePath, Archived: true}, FoldOptions{ + StoreDir: storeDir, Apply: true, FieldThreshold: 4, Budget: checker, + }) + if !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("Fold error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 || checker.Projection.Operation != "fold" || checker.Projection.AdditionalPersistentBytes <= 0 { + t.Fatalf("unexpected fold budget projection: %#v", checker) + } + if _, err := os.Stat(storeDir); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("fold store exists after preflight rejection: %v", err) + } +} + +func TestFoldReportsProjectedAndActualPhysicalAccounting(t *testing.T) { + root := t.TempDir() + sourcePath := filepath.Join(root, "source.jsonl") + storeDir := filepath.Join(root, "store") + source := []byte("{\"value\":\"physical-accounting\"}\n") + if err := os.WriteFile(sourcePath, source, 0o600); err != nil { + t.Fatal(err) + } + result, err := Fold(context.Background(), Session{ID: "accounting", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 4}) + if err != nil { + t.Fatal(err) + } + if result.Storage == nil || result.Storage.Budget.ProjectedPeakBytes <= 0 || result.Storage.After.LogicalSessionBytes != int64(len(source)) { + t.Fatalf("fold storage accounting is incomplete: %#v", result.Storage) + } + if result.Storage.ActualReclaimedBytes != 0 { + t.Fatalf("fold with retained source claimed reclamation: %#v", result.Storage) + } +} + +func TestUnfoldBudgetRejectsBeforeCreatingRestoreTarget(t *testing.T) { + root := t.TempDir() + sourcePath := filepath.Join(root, "source.jsonl") + storeDir := filepath.Join(root, "store") + if err := os.WriteFile(sourcePath, []byte("{\"value\":\"restore-budget\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Fold(context.Background(), Session{ID: "restore", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 4}); err != nil { + t.Fatal(err) + } + checker := &foldRejectingChecker{} + target := filepath.Join(root, "output", "restored.jsonl") + if _, err := UnfoldWithOptions(context.Background(), storeDir, "restore", UnfoldOptions{TargetPath: target, Budget: checker}); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("UnfoldWithOptions error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 || checker.Projection.Operation != "unfold" { + t.Fatalf("unexpected unfold budget projection: %#v", checker) + } + if _, err := os.Stat(filepath.Dir(target)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("restore target directory exists after preflight rejection: %v", err) + } +} + +func TestUnfoldReportsStorageBudgetWithoutClaimingReclamation(t *testing.T) { + root := t.TempDir() + sourcePath := filepath.Join(root, "source.jsonl") + storeDir := filepath.Join(root, "store") + if err := os.WriteFile(sourcePath, []byte("{\"value\":\"unfold-accounting\"}\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Fold(context.Background(), Session{ID: "unfold-accounting", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 4}); err != nil { + t.Fatal(err) + } + target := filepath.Join(root, "restored.jsonl") + result, err := Unfold(context.Background(), storeDir, "unfold-accounting", target, false) + if err != nil { + t.Fatal(err) + } + if result.Storage == nil || result.Storage.Budget.ProjectedPeakBytes <= result.Storage.Budget.CurrentPhysicalBytes || result.Storage.ActualReclaimedBytes != 0 { + t.Fatalf("unfold storage accounting is incomplete: %#v", result.Storage) + } +} + func TestFoldRejectsSameSizeMutationEvenWhenMtimeIsRestored(t *testing.T) { root := t.TempDir() sourcePath := filepath.Join(root, "rollout.jsonl") @@ -72,7 +157,7 @@ func TestFoldRejectsSameSizeMutationEvenWhenMtimeIsRestored(t *testing.T) { t.Fatalf("stat source: %v", err) } - _, err = Fold(context.Background(), codex.Session{ + _, err = Fold(context.Background(), Session{ ID: "same-size-change", RolloutPath: sourcePath, Archived: true, }, FoldOptions{ StoreDir: storeDir, Apply: true, FieldThreshold: 4, @@ -102,7 +187,7 @@ func TestFoldCreatesVerifiedManifestAndReusesRepeatedField(t *testing.T) { t.Fatalf("write source: %v", err) } - result, err := Fold(context.Background(), codex.Session{ + result, err := Fold(context.Background(), Session{ ID: "fixture", Title: "Fixture", CWD: "/workspace", RolloutPath: sourcePath, Archived: true, }, FoldOptions{ StoreDir: storeDir, @@ -148,7 +233,7 @@ func TestFoldDryRunDoesNotCreateStore(t *testing.T) { if err := os.WriteFile(sourcePath, []byte("{\"value\":\"large-value\"}\n"), 0o644); err != nil { t.Fatalf("write source: %v", err) } - result, err := Fold(context.Background(), codex.Session{ID: "dry", RolloutPath: sourcePath, Archived: true}, FoldOptions{ + result, err := Fold(context.Background(), Session{ID: "dry", RolloutPath: sourcePath, Archived: true}, FoldOptions{ StoreDir: storeDir, FieldThreshold: 4, }) if err != nil { @@ -162,6 +247,38 @@ func TestFoldDryRunDoesNotCreateStore(t *testing.T) { } } +func TestFoldWritesToExplicitGenerationManifestWithoutReplacingPrimary(t *testing.T) { + root := t.TempDir() + storeDir := filepath.Join(root, "store") + sourcePath := filepath.Join(root, "session.jsonl") + data := []byte("{\"value\":\"generation-manifest\"}\n") + if err := os.WriteFile(sourcePath, data, 0o600); err != nil { + t.Fatal(err) + } + primary := ManifestPath(storeDir, "session") + if err := os.MkdirAll(filepath.Dir(primary), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(primary, []byte("primary-sentinel"), 0o600); err != nil { + t.Fatal(err) + } + generationPath := filepath.Join(storeDir, "manifests", "generations", "session", "2.json") + result, err := Fold(context.Background(), Session{ID: "session", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true, FieldThreshold: 8, ManifestPathOverride: generationPath}) + if err != nil { + t.Fatalf("Fold generation manifest: %v", err) + } + if result.ManifestPath != generationPath { + t.Fatalf("manifest path = %q, want %q", result.ManifestPath, generationPath) + } + if primaryData, err := os.ReadFile(primary); err != nil || string(primaryData) != "primary-sentinel" { + t.Fatalf("primary manifest changed: %q err=%v", primaryData, err) + } + manifest, err := LoadManifestPath(generationPath) + if err != nil || manifest.Source.Bytes != int64(len(data)) { + t.Fatalf("load generation manifest: %#v err=%v", manifest, err) + } +} + func TestFoldRoundTripsEmptyInvalidAndOversizedRollouts(t *testing.T) { for _, test := range []struct { name string @@ -182,7 +299,7 @@ func TestFoldRoundTripsEmptyInvalidAndOversizedRollouts(t *testing.T) { if err := os.WriteFile(sourcePath, test.source, 0o644); err != nil { t.Fatalf("write source: %v", err) } - result, err := Fold(context.Background(), codex.Session{ID: test.name, RolloutPath: sourcePath, Archived: true}, FoldOptions{ + result, err := Fold(context.Background(), Session{ID: test.name, RolloutPath: sourcePath, Archived: true}, FoldOptions{ StoreDir: storeDir, Apply: true, FieldThreshold: 4, MaxJSONLineBytes: test.maxLineBytes, }) if err != nil { @@ -211,7 +328,7 @@ func TestFoldHonorsCanceledContextWithoutManifest(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := Fold(ctx, codex.Session{ID: "canceled", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true}) + _, err := Fold(ctx, Session{ID: "canceled", RolloutPath: sourcePath, Archived: true}, FoldOptions{StoreDir: storeDir, Apply: true}) if !errors.Is(err, context.Canceled) { t.Fatalf("Fold error = %v, want context.Canceled", err) } @@ -219,3 +336,14 @@ func TestFoldHonorsCanceledContextWithoutManifest(t *testing.T) { t.Fatalf("manifest committed after cancellation: %v", statErr) } } + +type foldRejectingChecker struct { + Calls int + Projection storage.Projection +} + +func (c *foldRejectingChecker) Check(_ context.Context, projection storage.Projection) (storage.Assessment, error) { + c.Calls++ + c.Projection = projection + return storage.Assessment{}, storage.ErrBudgetExceeded +} diff --git a/internal/fold/gc.go b/internal/fold/gc.go index 9139455..eefe48e 100644 --- a/internal/fold/gc.go +++ b/internal/fold/gc.go @@ -6,30 +6,52 @@ import ( "os" "path/filepath" "strings" + + "github.com/samekind/codexfold/internal/storage" ) type GCResult struct { - StoreDir string `json:"store_dir"` - DryRun bool `json:"dry_run"` - Referenced int `json:"referenced_objects"` - OrphanCount int `json:"orphan_count"` - OrphanBytes int64 `json:"orphan_bytes"` - RemovedCount int `json:"removed_count"` - RemovedBytes int64 `json:"removed_bytes"` + StoreDir string `json:"store_dir"` + DryRun bool `json:"dry_run"` + Referenced int `json:"referenced_objects"` + OrphanCount int `json:"orphan_count"` + OrphanBytes int64 `json:"orphan_bytes"` + RemovedCount int `json:"removed_count"` + RemovedBytes int64 `json:"removed_bytes"` + Storage storage.StorageGCResult `json:"storage"` + ProjectedReclaimableBytes int64 `json:"projected_reclaimable_bytes"` + ActualReclaimedBytes int64 `json:"actual_reclaimed_bytes"` } func GC(ctx context.Context, storeDir string, apply bool) (GCResult, error) { result := GCResult{StoreDir: storeDir, DryRun: !apply} - manifests, issues, err := loadAllManifests(storeDir) + _, issues, err := loadAllManifests(storeDir) if err != nil { return GCResult{}, err } if len(issues) > 0 { return GCResult{}, fmt.Errorf("refusing GC with %d invalid manifest(s)", len(issues)) } + before, err := storage.Scan(ctx, storage.Options{StoreDir: storeDir, AllowMetadataIssues: true}) + if err != nil { + return GCResult{}, err + } + storageResult, err := storage.Collect(ctx, storage.GCOptions{StoreDir: storeDir, Apply: apply}) + if err != nil { + return GCResult{}, err + } + result.Storage = storageResult + result.ProjectedReclaimableBytes = storageResult.ProjectedReclaimableBytes + manifests, issues, err := loadAllManifests(storeDir) + if err != nil { + return GCResult{}, err + } + if len(issues) > 0 { + return GCResult{}, fmt.Errorf("refusing loose-object GC with %d invalid manifest(s)", len(issues)) + } referenced := make(map[string]struct{}) - for _, manifest := range manifests { - for _, part := range manifest.Parts { + for _, loaded := range manifests { + for _, part := range loaded.Manifest.Parts { referenced[part.Object.SHA256] = struct{}{} } } @@ -56,5 +78,13 @@ func GC(ctx context.Context, storeDir string, apply bool) (GCResult, error) { if err != nil { return GCResult{}, err } + result.ProjectedReclaimableBytes += result.OrphanBytes + after, err := storage.Scan(ctx, storage.Options{StoreDir: storeDir, AllowMetadataIssues: true}) + if err != nil { + return GCResult{}, err + } + if before.TotalPhysicalBytes > after.TotalPhysicalBytes { + result.ActualReclaimedBytes = before.TotalPhysicalBytes - after.TotalPhysicalBytes + } return result, nil } diff --git a/internal/fold/manifest.go b/internal/fold/manifest.go index 8044a17..e4703b0 100644 --- a/internal/fold/manifest.go +++ b/internal/fold/manifest.go @@ -62,7 +62,10 @@ func LoadManifest(storeDir string, sessionID string) (Manifest, error) { if err := validateSessionID(sessionID); err != nil { return Manifest{}, err } - path := ManifestPath(storeDir, sessionID) + return LoadManifestPath(ManifestPath(storeDir, sessionID)) +} + +func LoadManifestPath(path string) (Manifest, error) { data, err := os.ReadFile(path) if err != nil { return Manifest{}, fmt.Errorf("read fold manifest: %w", err) @@ -81,7 +84,13 @@ func writeManifest(storeDir string, manifest Manifest, overwrite bool) error { if err := validateSessionID(manifest.Session.ID); err != nil { return err } - path := ManifestPath(storeDir, manifest.Session.ID) + return writeManifestPath(ManifestPath(storeDir, manifest.Session.ID), manifest, overwrite) +} + +func writeManifestPath(path string, manifest Manifest, overwrite bool) error { + if err := validateManifest(manifest); err != nil { + return err + } if !overwrite { if _, err := os.Stat(path); err == nil { return fmt.Errorf("fold manifest already exists: %s", path) diff --git a/internal/fold/stream.go b/internal/fold/stream.go new file mode 100644 index 0000000..c93db55 --- /dev/null +++ b/internal/fold/stream.go @@ -0,0 +1,57 @@ +package fold + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "hash" + "io" + "os" + + "github.com/klauspost/compress/zstd" +) + +type objectStream struct { + ref ObjectRef + file *os.File + decoder *zstd.Decoder + hash hash.Hash + read int64 + done bool +} + +func (s *ObjectStore) OpenStream(ref ObjectRef) (io.ReadCloser, error) { + file, err := os.Open(s.ObjectPath(ref.SHA256)) + if err != nil { + return nil, fmt.Errorf("open object %s: %w", ref.SHA256, err) + } + decoder, err := zstd.NewReader(file) + if err != nil { + _ = file.Close() + return nil, fmt.Errorf("create object stream %s: %w", ref.SHA256, err) + } + return &objectStream{ref: ref, file: file, decoder: decoder, hash: sha256.New()}, nil +} + +func (s *objectStream) Read(destination []byte) (int, error) { + n, err := s.decoder.Read(destination) + if n > 0 { + _, _ = s.hash.Write(destination[:n]) + s.read += int64(n) + } + if err == io.EOF && !s.done { + s.done = true + if s.read != s.ref.RawBytes { + return n, fmt.Errorf("object %s raw size %d, want %d", s.ref.SHA256, s.read, s.ref.RawBytes) + } + if hex.EncodeToString(s.hash.Sum(nil)) != s.ref.SHA256 { + return n, fmt.Errorf("object %s SHA-256 mismatch", s.ref.SHA256) + } + } + return n, err +} + +func (s *objectStream) Close() error { + s.decoder.Close() + return s.file.Close() +} diff --git a/internal/fold/unfold.go b/internal/fold/unfold.go index 84e1b61..5ad5c5a 100644 --- a/internal/fold/unfold.go +++ b/internal/fold/unfold.go @@ -6,30 +6,62 @@ import ( "fmt" "os" "path/filepath" + + "github.com/samekind/codexfold/internal/storage" ) type UnfoldResult struct { - SessionID string `json:"session_id"` - ManifestPath string `json:"manifest_path"` - TargetPath string `json:"target_path"` - Bytes int64 `json:"bytes"` - SHA256 string `json:"sha256"` - Verified bool `json:"verified"` + SessionID string `json:"session_id"` + ManifestPath string `json:"manifest_path"` + TargetPath string `json:"target_path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + Verified bool `json:"verified"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` +} + +type UnfoldOptions struct { + TargetPath string + Overwrite bool + Budget storage.Checker } func Unfold(ctx context.Context, storeDir string, sessionID string, targetPath string, overwrite bool) (UnfoldResult, error) { + return UnfoldWithOptions(ctx, storeDir, sessionID, UnfoldOptions{TargetPath: targetPath, Overwrite: overwrite}) +} + +func UnfoldWithOptions(ctx context.Context, storeDir string, sessionID string, options UnfoldOptions) (UnfoldResult, error) { manifest, err := LoadManifest(storeDir, sessionID) if err != nil { return UnfoldResult{}, err } + targetPath := options.TargetPath if targetPath == "" { targetPath = manifest.Session.RolloutPath } - if _, err := os.Stat(targetPath); err == nil && !overwrite { + reclaimableBytes := int64(0) + if info, err := os.Stat(targetPath); err == nil && !options.Overwrite { return UnfoldResult{}, fmt.Errorf("restore target already exists: %s", targetPath) + } else if err == nil && info.Mode().IsRegular() { + reclaimableBytes = info.Size() } else if err != nil && !os.IsNotExist(err) { return UnfoldResult{}, err } + budget := options.Budget + if budget == nil { + guard, err := storage.DefaultGuard(storeDir) + if err != nil { + return UnfoldResult{}, err + } + budget = guard + } + storageAssessment, err := budget.Check(ctx, storage.Projection{ + Operation: "unfold", AdditionalPersistentBytes: manifest.Source.Bytes, TemporaryBytes: manifest.Source.Bytes, + TemporaryPersistentOverlapBytes: manifest.Source.Bytes, ReclaimableBytes: reclaimableBytes, + }) + if err != nil { + return UnfoldResult{}, err + } if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil { return UnfoldResult{}, fmt.Errorf("create restore directory: %w", err) } @@ -71,7 +103,7 @@ func Unfold(ctx context.Context, storeDir string, sessionID string, targetPath s return UnfoldResult{}, fmt.Errorf("close restored rollout: %w", err) } commit := os.Rename - if overwrite { + if options.Overwrite { commit = replaceFile } if err := commit(temporaryPath, targetPath); err != nil { @@ -83,5 +115,6 @@ func Unfold(ctx context.Context, storeDir string, sessionID string, targetPath s return UnfoldResult{ SessionID: sessionID, ManifestPath: ManifestPath(storeDir, sessionID), TargetPath: targetPath, Bytes: bytesWritten, SHA256: manifest.Source.SHA256, Verified: true, + Storage: storage.CompleteAccounting(ctx, storageAssessment, storeDir), }, nil } diff --git a/internal/fsctl/benchmark.go b/internal/fsctl/benchmark.go new file mode 100644 index 0000000..8380a29 --- /dev/null +++ b/internal/fsctl/benchmark.go @@ -0,0 +1,177 @@ +package fsctl + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "math/rand" + "os" + "runtime" + "sort" + "time" +) + +type BenchmarkOptions struct { + SequentialBlockBytes int + RandomBlockBytes int + RandomReads int + Seed int64 + BypassOSCache bool +} + +type SequentialMetric struct { + Bytes int64 `json:"bytes"` + Duration time.Duration `json:"duration"` + BytesPerSecond float64 `json:"bytes_per_second"` +} + +type RandomMetric struct { + Reads int `json:"reads"` + P50 time.Duration `json:"p50"` + P95 time.Duration `json:"p95"` + P99 time.Duration `json:"p99"` +} + +type BenchmarkReport struct { + Native SequentialMetric `json:"native"` + Virtual SequentialMetric `json:"virtual"` + Random RandomMetric `json:"random"` + GoSysBytes uint64 `json:"go_sys_bytes"` + OSCacheBypassRequested bool `json:"os_cache_bypass_requested"` + OSCacheBypassApplied bool `json:"os_cache_bypass_applied"` +} + +func Benchmark(ctx context.Context, nativePath string, virtual Readable, options BenchmarkOptions) (BenchmarkReport, error) { + if options.SequentialBlockBytes <= 0 { + options.SequentialBlockBytes = 1 << 20 + } + if options.RandomBlockBytes <= 0 { + options.RandomBlockBytes = 4 << 10 + } + if options.RandomReads <= 0 { + options.RandomReads = 1000 + } + native, err := os.Open(nativePath) + if err != nil { + return BenchmarkReport{}, err + } + defer native.Close() + bypassApplied := false + if options.BypassOSCache { + bypassApplied, err = configureNoCache(native) + if err != nil { + return BenchmarkReport{}, err + } + } + info, err := native.Stat() + if err != nil { + return BenchmarkReport{}, err + } + if info.Size() != virtual.Size() { + return BenchmarkReport{}, errors.New("benchmark native and virtual sizes differ") + } + nativeMetric, err := benchmarkNativeSequential(ctx, native, info.Size(), options.SequentialBlockBytes) + if err != nil { + return BenchmarkReport{}, err + } + virtualMetric, err := benchmarkVirtualSequential(ctx, virtual, options.SequentialBlockBytes) + if err != nil { + return BenchmarkReport{}, err + } + randomMetric, err := benchmarkRandom(ctx, native, virtual, info.Size(), options) + if err != nil { + return BenchmarkReport{}, err + } + var memory runtime.MemStats + runtime.ReadMemStats(&memory) + return BenchmarkReport{Native: nativeMetric, Virtual: virtualMetric, Random: randomMetric, GoSysBytes: memory.Sys, OSCacheBypassRequested: options.BypassOSCache, OSCacheBypassApplied: bypassApplied}, nil +} + +func benchmarkNativeSequential(ctx context.Context, file *os.File, size int64, blockBytes int) (SequentialMetric, error) { + buffer := make([]byte, blockBytes) + start := time.Now() + var offset int64 + for offset < size { + if err := ctx.Err(); err != nil { + return SequentialMetric{}, err + } + need := blockBytes + if remaining := size - offset; int64(need) > remaining { + need = int(remaining) + } + n, err := file.ReadAt(buffer[:need], offset) + if n != need || (err != nil && !errors.Is(err, io.EOF)) { + return SequentialMetric{}, fmt.Errorf("native sequential read at %d: n=%d err=%v", offset, n, err) + } + offset += int64(n) + } + return sequentialMetric(offset, time.Since(start)), nil +} + +func benchmarkVirtualSequential(ctx context.Context, virtual Readable, blockBytes int) (SequentialMetric, error) { + buffer := make([]byte, blockBytes) + start := time.Now() + var offset int64 + for offset < virtual.Size() { + need := blockBytes + if remaining := virtual.Size() - offset; int64(need) > remaining { + need = int(remaining) + } + n, err := virtual.ReadAt(ctx, buffer[:need], offset) + if n != need || (err != nil && !errors.Is(err, io.EOF)) { + return SequentialMetric{}, fmt.Errorf("virtual sequential read at %d: n=%d err=%v", offset, n, err) + } + offset += int64(n) + } + return sequentialMetric(offset, time.Since(start)), nil +} + +func benchmarkRandom(ctx context.Context, native *os.File, virtual Readable, size int64, options BenchmarkOptions) (RandomMetric, error) { + if size == 0 { + return RandomMetric{}, nil + } + random := rand.New(rand.NewSource(options.Seed)) + nativeBuffer := make([]byte, options.RandomBlockBytes) + virtualBuffer := make([]byte, options.RandomBlockBytes) + durations := make([]time.Duration, 0, options.RandomReads) + for index := 0; index < options.RandomReads; index++ { + offset := random.Int63n(size) + length := options.RandomBlockBytes + if remaining := size - offset; int64(length) > remaining { + length = int(remaining) + } + nativeN, nativeErr := native.ReadAt(nativeBuffer[:length], offset) + start := time.Now() + virtualN, virtualErr := virtual.ReadAt(ctx, virtualBuffer[:length], offset) + duration := time.Since(start) + if duration <= 0 { + duration = time.Nanosecond + } + if nativeN != length || virtualN != length || !bytes.Equal(nativeBuffer[:length], virtualBuffer[:length]) || (nativeErr != nil && !errors.Is(nativeErr, io.EOF)) || (virtualErr != nil && !errors.Is(virtualErr, io.EOF)) { + return RandomMetric{}, fmt.Errorf("random benchmark read %d differs at offset %d", index, offset) + } + durations = append(durations, duration) + } + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + return RandomMetric{Reads: len(durations), P50: percentile(durations, 50), P95: percentile(durations, 95), P99: percentile(durations, 99)}, nil +} + +func sequentialMetric(bytesRead int64, duration time.Duration) SequentialMetric { + if duration <= 0 { + duration = time.Nanosecond + } + return SequentialMetric{Bytes: bytesRead, Duration: duration, BytesPerSecond: float64(bytesRead) / duration.Seconds()} +} + +func percentile(values []time.Duration, percent int) time.Duration { + if len(values) == 0 { + return 0 + } + index := (len(values)*percent + 99) / 100 + if index < 1 { + index = 1 + } + return values[index-1] +} diff --git a/internal/fsctl/doctor.go b/internal/fsctl/doctor.go new file mode 100644 index 0000000..3b270c5 --- /dev/null +++ b/internal/fsctl/doctor.go @@ -0,0 +1,84 @@ +package fsctl + +import ( + "context" + "fmt" + + "github.com/samekind/codexfold/internal/storage" +) + +const ( + ComponentDaemon = "daemon" + ComponentMount = "mount" + ComponentPack = "pack" + ComponentManifest = "manifest" + ComponentDelta = "delta" + ComponentBacking = "backing" + ComponentRoute = "route" + ComponentFallback = "fallback" + ComponentJournal = "journal" + ComponentClient = "client" + ComponentStorage = "storage" +) + +var RequiredComponents = []string{ComponentDaemon, ComponentMount, ComponentPack, ComponentManifest, ComponentDelta, ComponentBacking, ComponentRoute, ComponentFallback, ComponentJournal, ComponentClient, ComponentStorage} + +type Check struct { + Component string + Run func(context.Context) error +} + +type Issue struct { + Component string `json:"component"` + Severity string `json:"severity"` + SessionID string `json:"session_id,omitempty"` + Generation uint64 `json:"generation,omitempty"` + Message string `json:"message"` + Remediation string `json:"remediation,omitempty"` +} + +type DoctorReport struct { + Healthy bool `json:"healthy"` + IssueCount int `json:"issue_count"` + Issues []Issue `json:"issues,omitempty"` + ComponentHealth map[string]bool `json:"component_health"` + Storage storage.Inventory `json:"storage"` + StorageLimits storage.Limits `json:"storage_limits"` + AvailableBytes int64 `json:"available_bytes"` +} + +func Doctor(ctx context.Context, checks []Check) DoctorReport { + report := DoctorReport{Healthy: true, ComponentHealth: make(map[string]bool)} + seen := make(map[string]bool) + for _, check := range checks { + if check.Component == "" || check.Run == nil { + report.Issues = append(report.Issues, Issue{Component: check.Component, Severity: "error", Message: "invalid doctor check"}) + continue + } + if seen[check.Component] { + report.Issues = append(report.Issues, Issue{Component: check.Component, Severity: "error", Message: "duplicate doctor check"}) + continue + } + seen[check.Component] = true + if err := ctx.Err(); err != nil { + report.ComponentHealth[check.Component] = false + report.Issues = append(report.Issues, Issue{Component: check.Component, Severity: "error", Message: err.Error()}) + continue + } + if err := check.Run(ctx); err != nil { + report.ComponentHealth[check.Component] = false + report.Issues = append(report.Issues, Issue{Component: check.Component, Severity: "error", Message: err.Error()}) + } else { + report.ComponentHealth[check.Component] = true + } + } + for _, component := range RequiredComponents { + if !seen[component] { + report.ComponentHealth[component] = false + report.Issues = append(report.Issues, Issue{Component: component, Severity: "error", Message: fmt.Sprintf("required %s check is missing", component), Remediation: "register and run the required component check"}) + } + } + report.IssueCount = len(report.Issues) + report.Healthy = report.IssueCount == 0 + return report +} diff --git a/internal/fsctl/fsctl_test.go b/internal/fsctl/fsctl_test.go new file mode 100644 index 0000000..71c935d --- /dev/null +++ b/internal/fsctl/fsctl_test.go @@ -0,0 +1,120 @@ +package fsctl + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "path/filepath" + "testing" +) + +func TestStatusAcceptsOnlyCanonicalCapabilities(t *testing.T) { + for _, capability := range []Capability{StorageEngine, FSEnginePreview, PlatformCanary, Capability("production-ready:macos"), CrossPlatformReady} { + if _, err := NewStatus(capability, "darwin"); err != nil { + t.Fatalf("NewStatus(%q) returned error: %v", capability, err) + } + } + for _, capability := range []Capability{"transparent", "stable", "production-ready"} { + if _, err := NewStatus(capability, "darwin"); err == nil { + t.Fatalf("NewStatus(%q) should reject non-canonical capability", capability) + } + } +} + +func TestShadowComparesCompleteAndRandomBytes(t *testing.T) { + root := t.TempDir() + data := bytes.Repeat([]byte("shadow-source-"), 1000) + nativePath := filepath.Join(root, "native.jsonl") + if err := os.WriteFile(nativePath, data, 0o600); err != nil { + t.Fatalf("write native: %v", err) + } + result, err := Shadow(context.Background(), nativePath, byteReader(data), ShadowOptions{BlockBytes: 257, RandomReads: 10000, Seed: 42}) + if err != nil { + t.Fatalf("Shadow returned error: %v", err) + } + if !result.Verified || result.RandomReads != 10000 || result.ComparedBytes != int64(len(data)) { + t.Fatalf("unexpected shadow result: %#v", result) + } + + corrupt := append([]byte(nil), data...) + corrupt[len(corrupt)/2] ^= 1 + if result, err := Shadow(context.Background(), nativePath, byteReader(corrupt), ShadowOptions{BlockBytes: 257, RandomReads: 100, Seed: 42}); err == nil || result.Verified { + t.Fatalf("Shadow should reject one-byte mismatch: result=%#v err=%v", result, err) + } +} + +func TestDoctorRequiresEveryComponentAndSeparatesDaemonFromMount(t *testing.T) { + checks := make([]Check, 0, len(RequiredComponents)) + for _, component := range RequiredComponents { + component := component + checks = append(checks, Check{Component: component, Run: func(context.Context) error { + if component == ComponentMount { + return errors.New("mount unavailable") + } + return nil + }}) + } + report := Doctor(context.Background(), checks) + if report.Healthy || report.ComponentHealth[ComponentDaemon] != true || report.ComponentHealth[ComponentMount] != false { + t.Fatalf("doctor did not separate daemon and mount: %#v", report) + } + if len(report.Issues) != 1 || report.Issues[0].Component != ComponentMount { + t.Fatalf("unexpected doctor issues: %#v", report.Issues) + } + + report = Doctor(context.Background(), checks[:len(checks)-1]) + if report.Healthy || report.IssueCount < 2 { + t.Fatalf("doctor should report a missing required component: %#v", report) + } +} + +func TestBenchmarkMeasuresNativeAndVirtualReads(t *testing.T) { + root := t.TempDir() + data := bytes.Repeat([]byte("benchmark-data-"), 10000) + nativePath := filepath.Join(root, "native.jsonl") + if err := os.WriteFile(nativePath, data, 0o600); err != nil { + t.Fatalf("write native: %v", err) + } + report, err := Benchmark(context.Background(), nativePath, byteReader(data), BenchmarkOptions{SequentialBlockBytes: 4096, RandomBlockBytes: 4096, RandomReads: 200, Seed: 9}) + if err != nil { + t.Fatalf("Benchmark returned error: %v", err) + } + if report.Native.Bytes != int64(len(data)) || report.Virtual.Bytes != int64(len(data)) || report.Random.Reads != 200 { + t.Fatalf("unexpected benchmark report: %#v", report) + } + if report.Native.Duration <= 0 || report.Virtual.Duration <= 0 || report.Random.P95 <= 0 { + t.Fatalf("benchmark durations were not recorded: %#v", report) + } +} + +func TestBenchmarkRecordsRequestedOSCacheBypass(t *testing.T) { + path := filepath.Join(t.TempDir(), "native.jsonl") + data := []byte("cache-bypass") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + report, err := Benchmark(context.Background(), path, byteReader(data), BenchmarkOptions{BypassOSCache: true, RandomReads: 1}) + if err != nil { + t.Fatal(err) + } + if !report.OSCacheBypassRequested { + t.Fatalf("benchmark did not record cache-bypass request: %#v", report) + } +} + +type byteReader []byte + +func (r byteReader) Size() int64 { return int64(len(r)) } + +func (r byteReader) ReadAt(_ context.Context, destination []byte, offset int64) (int, error) { + if offset >= int64(len(r)) { + return 0, io.EOF + } + n := copy(destination, r[offset:]) + if n < len(destination) { + return n, io.EOF + } + return n, nil +} diff --git a/internal/fsctl/nocache_darwin.go b/internal/fsctl/nocache_darwin.go new file mode 100644 index 0000000..0f999d0 --- /dev/null +++ b/internal/fsctl/nocache_darwin.go @@ -0,0 +1,14 @@ +//go:build darwin + +package fsctl + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func configureNoCache(file *os.File) (bool, error) { + _, err := unix.FcntlInt(file.Fd(), unix.F_NOCACHE, 1) + return err == nil, err +} diff --git a/internal/fsctl/nocache_other.go b/internal/fsctl/nocache_other.go new file mode 100644 index 0000000..834b9c1 --- /dev/null +++ b/internal/fsctl/nocache_other.go @@ -0,0 +1,7 @@ +//go:build !darwin + +package fsctl + +import "os" + +func configureNoCache(*os.File) (bool, error) { return false, nil } diff --git a/internal/fsctl/shadow.go b/internal/fsctl/shadow.go new file mode 100644 index 0000000..d84737c --- /dev/null +++ b/internal/fsctl/shadow.go @@ -0,0 +1,123 @@ +package fsctl + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "math/rand" + "os" +) + +type Readable interface { + Size() int64 + ReadAt(context.Context, []byte, int64) (int, error) +} + +type ShadowOptions struct { + BlockBytes int + RandomReads int + Seed int64 +} + +type ShadowResult struct { + NativePath string `json:"native_path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` + ComparedBytes int64 `json:"compared_bytes"` + RandomReads int `json:"random_reads"` + Verified bool `json:"verified"` +} + +func Shadow(ctx context.Context, nativePath string, virtual Readable, options ShadowOptions) (ShadowResult, error) { + if options.BlockBytes <= 0 { + options.BlockBytes = 1 << 20 + } + if options.RandomReads < 0 { + return ShadowResult{}, errors.New("shadow random read count cannot be negative") + } + native, err := os.Open(nativePath) + if err != nil { + return ShadowResult{}, err + } + defer native.Close() + info, err := native.Stat() + if err != nil { + return ShadowResult{}, err + } + if info.Size() != virtual.Size() { + return ShadowResult{}, fmt.Errorf("shadow size mismatch native=%d virtual=%d", info.Size(), virtual.Size()) + } + nativeHash := sha256.New() + virtualHash := sha256.New() + nativeBuffer := make([]byte, options.BlockBytes) + virtualBuffer := make([]byte, options.BlockBytes) + var offset int64 + for offset < info.Size() { + if err := ctx.Err(); err != nil { + return ShadowResult{}, err + } + need := options.BlockBytes + if remaining := info.Size() - offset; int64(need) > remaining { + need = int(remaining) + } + nativeN, nativeErr := native.ReadAt(nativeBuffer[:need], offset) + virtualN, virtualErr := virtual.ReadAt(ctx, virtualBuffer[:need], offset) + if nativeN != need || virtualN != need || (nativeErr != nil && !errors.Is(nativeErr, io.EOF)) || (virtualErr != nil && !errors.Is(virtualErr, io.EOF)) { + return ShadowResult{}, fmt.Errorf("shadow read failed at offset %d: native=(%d,%v) virtual=(%d,%v)", offset, nativeN, nativeErr, virtualN, virtualErr) + } + if !bytes.Equal(nativeBuffer[:need], virtualBuffer[:need]) { + return ShadowResult{}, fmt.Errorf("shadow byte mismatch at offset %d", offset) + } + _, _ = nativeHash.Write(nativeBuffer[:need]) + _, _ = virtualHash.Write(virtualBuffer[:need]) + offset += int64(need) + } + nativeDigest := hex.EncodeToString(nativeHash.Sum(nil)) + if nativeDigest != hex.EncodeToString(virtualHash.Sum(nil)) { + return ShadowResult{}, errors.New("shadow complete SHA-256 mismatch") + } + random := rand.New(rand.NewSource(options.Seed)) + for index := 0; index < options.RandomReads && info.Size() > 0; index++ { + readOffset := random.Int63n(info.Size()) + length := 1 + random.Intn(options.BlockBytes) + if remaining := info.Size() - readOffset; int64(length) > remaining { + length = int(remaining) + } + nativeN, nativeErr := native.ReadAt(nativeBuffer[:length], readOffset) + virtualN, virtualErr := virtual.ReadAt(ctx, virtualBuffer[:length], readOffset) + if nativeN != length || virtualN != length || !bytes.Equal(nativeBuffer[:length], virtualBuffer[:length]) || (nativeErr != nil && !errors.Is(nativeErr, io.EOF)) || (virtualErr != nil && !errors.Is(virtualErr, io.EOF)) { + return ShadowResult{}, fmt.Errorf("shadow random read %d differs at offset %d", index, readOffset) + } + } + after, err := hashFile(nativePath) + if err != nil { + return ShadowResult{}, err + } + if after.Bytes != info.Size() || after.SHA256 != nativeDigest { + return ShadowResult{}, errors.New("native source changed during shadow verification") + } + return ShadowResult{NativePath: nativePath, Bytes: info.Size(), SHA256: nativeDigest, ComparedBytes: offset, RandomReads: options.RandomReads, Verified: true}, nil +} + +type fileDigest struct { + Bytes int64 + SHA256 string +} + +func hashFile(path string) (fileDigest, error) { + file, err := os.Open(path) + if err != nil { + return fileDigest{}, err + } + defer file.Close() + hasher := sha256.New() + bytesRead, err := io.Copy(hasher, file) + if err != nil { + return fileDigest{}, err + } + return fileDigest{Bytes: bytesRead, SHA256: hex.EncodeToString(hasher.Sum(nil))}, nil +} diff --git a/internal/fsctl/status.go b/internal/fsctl/status.go new file mode 100644 index 0000000..496f65b --- /dev/null +++ b/internal/fsctl/status.go @@ -0,0 +1,39 @@ +package fsctl + +import ( + "fmt" + "strings" + + "github.com/samekind/codexfold/internal/storage" +) + +type Capability string + +const ( + StorageEngine Capability = "storage-engine" + FSEnginePreview Capability = "fs-engine-preview" + PlatformCanary Capability = "platform-canary" + CrossPlatformReady Capability = "cross-platform-ready" +) + +type Status struct { + Capability Capability `json:"capability"` + Platform string `json:"platform"` + Storage storage.Inventory `json:"storage"` + StorageLimits storage.Limits `json:"storage_limits"` + AvailableBytes int64 `json:"available_bytes"` +} + +func NewStatus(capability Capability, platform string) (Status, error) { + valid := capability == StorageEngine || capability == FSEnginePreview || capability == PlatformCanary || capability == CrossPlatformReady + if strings.HasPrefix(string(capability), "production-ready:") && strings.TrimPrefix(string(capability), "production-ready:") != "" { + valid = true + } + if !valid { + return Status{}, fmt.Errorf("non-canonical filesystem capability %q", capability) + } + if platform == "" { + return Status{}, fmt.Errorf("status platform is required") + } + return Status{Capability: capability, Platform: platform}, nil +} diff --git a/internal/fskitproto/client.go b/internal/fskitproto/client.go new file mode 100644 index 0000000..335d0cf --- /dev/null +++ b/internal/fskitproto/client.go @@ -0,0 +1,149 @@ +package fskitproto + +import ( + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "time" +) + +const DescriptorFilename = "descriptor.bin" + +type StatusError struct { + Operation Op + Errno syscall.Errno +} + +func (e StatusError) Error() string { + return fmt.Sprintf("FSKit operation %d failed: %s", e.Operation, e.Errno) +} + +type Client struct { + mu sync.Mutex + connection net.Conn + generation uint64 + maxPayload uint32 + nextID uint64 +} + +func DialResource(resourcePath string, timeout time.Duration) (*Client, error) { + descriptorPath, err := ResourceDescriptorPath(resourcePath) + if err != nil { + return nil, err + } + data, err := os.ReadFile(descriptorPath) + if err != nil { + return nil, fmt.Errorf("read FSKit resource: %w", err) + } + descriptor, err := DecodeDescriptor(data) + if err != nil { + return nil, err + } + return Dial(descriptor, timeout) +} + +func ResourceDescriptorPath(resourcePath string) (string, error) { + if !filepath.IsAbs(resourcePath) { + return "", errors.New("absolute FSKit resource path is required") + } + resourcePath = filepath.Clean(resourcePath) + if UsesDirectoryResource(resourcePath) { + return filepath.Join(resourcePath, DescriptorFilename), nil + } + _, err := os.Stat(resourcePath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return "", err + } + return resourcePath, nil +} + +func UsesDirectoryResource(resourcePath string) bool { + if info, err := os.Stat(filepath.Clean(resourcePath)); err == nil { + return info.IsDir() + } + return !strings.EqualFold(filepath.Ext(resourcePath), ".bin") +} + +func Dial(descriptor Descriptor, timeout time.Duration) (*Client, error) { + if timeout <= 0 { + timeout = 5 * time.Second + } + connection, err := net.DialTimeout("unix", descriptor.SocketPath, timeout) + if err != nil { + return nil, fmt.Errorf("connect to FSKit daemon: %w", err) + } + client := &Client{connection: connection, generation: descriptor.Generation, maxPayload: DefaultMaxPayload, nextID: 1} + encoder := NewEncoder(len(descriptor.Token) + 4) + encoder.Bytes(descriptor.Token) + response, err := client.callLocked(OpHello, 0, encoder.Data()) + if err != nil { + _ = connection.Close() + return nil, err + } + decoder := NewDecoder(response) + maxPayload, err := decoder.Uint32() + if err != nil || maxPayload < 4096 { + _ = connection.Close() + return nil, errors.New("invalid FSKit daemon hello response") + } + if _, err := decoder.Uint64(); err != nil || decoder.Done() != nil { + _ = connection.Close() + return nil, errors.New("invalid FSKit daemon hello namespace response") + } + client.maxPayload = maxPayload + return client, nil +} + +func (c *Client) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + if c.connection == nil { + return nil + } + err := c.connection.Close() + c.connection = nil + return err +} + +func (c *Client) Call(operation Op, payload []byte) ([]byte, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.callLocked(operation, c.generation, payload) +} + +func (c *Client) callLocked(operation Op, generation uint64, payload []byte) ([]byte, error) { + if c.connection == nil { + return nil, net.ErrClosed + } + requestID := c.nextID + c.nextID++ + if err := WriteFrame(c.connection, Frame{ + Kind: KindRequest, Op: operation, RequestID: requestID, Generation: generation, Payload: payload, + }, c.maxPayload); err != nil { + return nil, err + } + response, err := ReadFrame(c.connection, c.maxPayload) + if err != nil { + return nil, err + } + if response.Kind != KindResponse || response.Op != operation || response.RequestID != requestID || response.Generation != c.generation { + return nil, errors.New("mismatched FSKit daemon response") + } + if response.Status != 0 { + return nil, StatusError{Operation: operation, Errno: syscall.Errno(response.Status)} + } + return response.Payload, nil +} + +func ErrorNumber(err error) syscall.Errno { + var status StatusError + if errors.As(err, &status) { + return status.Errno + } + return syscall.EIO +} diff --git a/internal/fskitproto/codec.go b/internal/fskitproto/codec.go new file mode 100644 index 0000000..a25495f --- /dev/null +++ b/internal/fskitproto/codec.go @@ -0,0 +1,294 @@ +package fskitproto + +import ( + "encoding/binary" + "errors" + "fmt" + "time" +) + +type Encoder struct { + data []byte +} + +func NewEncoder(capacity int) *Encoder { + return &Encoder{data: make([]byte, 0, capacity)} +} + +func (e *Encoder) Data() []byte { return e.data } + +func (e *Encoder) Raw(value []byte) { e.data = append(e.data, value...) } + +func (e *Encoder) Uint8(value uint8) { e.data = append(e.data, value) } + +func (e *Encoder) Uint16(value uint16) { + start := len(e.data) + e.data = append(e.data, 0, 0) + binary.LittleEndian.PutUint16(e.data[start:], value) +} + +func (e *Encoder) Uint32(value uint32) { + start := len(e.data) + e.data = append(e.data, 0, 0, 0, 0) + binary.LittleEndian.PutUint32(e.data[start:], value) +} + +func (e *Encoder) Uint64(value uint64) { + start := len(e.data) + e.data = append(e.data, make([]byte, 8)...) + binary.LittleEndian.PutUint64(e.data[start:], value) +} + +func (e *Encoder) Int64(value int64) { e.Uint64(uint64(value)) } + +func (e *Encoder) String(value string) { + e.Bytes([]byte(value)) +} + +func (e *Encoder) Bytes(value []byte) { + e.Uint32(uint32(len(value))) + e.Raw(value) +} + +func (e *Encoder) Time(value time.Time) { + if value.IsZero() { + e.Int64(0) + e.Uint32(0) + return + } + e.Int64(value.Unix()) + e.Uint32(uint32(value.Nanosecond())) +} + +func (e *Encoder) Entry(entry Entry) { + e.entry(entry, false) +} + +func (e *Encoder) EntryForCapabilities(entry Entry, capabilities uint32) { + e.entry(entry, capabilities&CapabilityContentGeneration != 0) +} + +func (e *Encoder) entry(entry Entry, includeContentGeneration bool) { + e.String(entry.Path) + e.String(entry.Name) + e.Uint64(entry.NodeID) + e.Uint64(entry.ParentID) + e.Uint8(uint8(entry.Type)) + e.Uint32(entry.Mode) + e.Uint32(entry.UID) + e.Uint32(entry.GID) + e.Uint64(entry.Size) + e.Uint64(entry.AllocSize) + e.Time(entry.ModTime) + e.Time(entry.ChangeTime) + e.Time(entry.AccessTime) + e.Uint64(entry.NamespaceID) + if includeContentGeneration { + e.Uint64(entry.ContentGeneration) + } +} + +func (e *Encoder) StatFS(stat StatFS) { + e.Uint32(stat.BlockSize) + e.Uint32(stat.IOSize) + e.Uint64(stat.TotalBytes) + e.Uint64(stat.AvailableBytes) + e.Uint64(stat.FreeBytes) + e.Uint64(stat.UsedBytes) + e.Uint64(stat.TotalFiles) + e.Uint64(stat.FreeFiles) +} + +type Decoder struct { + data []byte + offset int +} + +func NewDecoder(data []byte) *Decoder { return &Decoder{data: data} } + +func (d *Decoder) remaining() int { return len(d.data) - d.offset } + +func (d *Decoder) Remaining() int { return d.remaining() } + +func (d *Decoder) Raw(length int) ([]byte, error) { + if length < 0 || d.remaining() < length { + return nil, errors.New("truncated FSKit protocol payload") + } + value := d.data[d.offset : d.offset+length] + d.offset += length + return value, nil +} + +func (d *Decoder) Uint8() (uint8, error) { + value, err := d.Raw(1) + if err != nil { + return 0, err + } + return value[0], nil +} + +func (d *Decoder) Uint16() (uint16, error) { + value, err := d.Raw(2) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint16(value), nil +} + +func (d *Decoder) Uint32() (uint32, error) { + value, err := d.Raw(4) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint32(value), nil +} + +func (d *Decoder) Uint64() (uint64, error) { + value, err := d.Raw(8) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint64(value), nil +} + +func (d *Decoder) Int64() (int64, error) { + value, err := d.Uint64() + return int64(value), err +} + +func (d *Decoder) Bytes(limit int) ([]byte, error) { + length, err := d.Uint32() + if err != nil { + return nil, err + } + if uint64(length) > uint64(^uint(0)>>1) || (limit > 0 && length > uint32(limit)) { + return nil, fmt.Errorf("FSKit protocol byte field %d exceeds limit %d", length, limit) + } + return d.Raw(int(length)) +} + +func (d *Decoder) String(limit int) (string, error) { + value, err := d.Bytes(limit) + if err != nil { + return "", err + } + return string(value), nil +} + +func (d *Decoder) Time() (time.Time, error) { + seconds, err := d.Int64() + if err != nil { + return time.Time{}, err + } + nanoseconds, err := d.Uint32() + if err != nil { + return time.Time{}, err + } + if seconds == 0 && nanoseconds == 0 { + return time.Time{}, nil + } + if nanoseconds >= 1_000_000_000 { + return time.Time{}, errors.New("invalid FSKit protocol timestamp") + } + return time.Unix(seconds, int64(nanoseconds)), nil +} + +func (d *Decoder) Entry() (Entry, error) { + return d.entry(false) +} + +func (d *Decoder) EntryForCapabilities(capabilities uint32) (Entry, error) { + return d.entry(capabilities&CapabilityContentGeneration != 0) +} + +func (d *Decoder) entry(includeContentGeneration bool) (Entry, error) { + var entry Entry + var err error + if entry.Path, err = d.String(1 << 20); err != nil { + return Entry{}, err + } + if entry.Name, err = d.String(4096); err != nil { + return Entry{}, err + } + if entry.NodeID, err = d.Uint64(); err != nil { + return Entry{}, err + } + if entry.ParentID, err = d.Uint64(); err != nil { + return Entry{}, err + } + typeValue, err := d.Uint8() + if err != nil { + return Entry{}, err + } + entry.Type = EntryType(typeValue) + if entry.Mode, err = d.Uint32(); err != nil { + return Entry{}, err + } + if entry.UID, err = d.Uint32(); err != nil { + return Entry{}, err + } + if entry.GID, err = d.Uint32(); err != nil { + return Entry{}, err + } + if entry.Size, err = d.Uint64(); err != nil { + return Entry{}, err + } + if entry.AllocSize, err = d.Uint64(); err != nil { + return Entry{}, err + } + if entry.ModTime, err = d.Time(); err != nil { + return Entry{}, err + } + if entry.ChangeTime, err = d.Time(); err != nil { + return Entry{}, err + } + if entry.AccessTime, err = d.Time(); err != nil { + return Entry{}, err + } + if entry.NamespaceID, err = d.Uint64(); err != nil { + return Entry{}, err + } + if includeContentGeneration { + if entry.ContentGeneration, err = d.Uint64(); err != nil { + return Entry{}, err + } + } + return entry, nil +} + +func (d *Decoder) StatFS() (StatFS, error) { + var stat StatFS + var err error + if stat.BlockSize, err = d.Uint32(); err != nil { + return StatFS{}, err + } + if stat.IOSize, err = d.Uint32(); err != nil { + return StatFS{}, err + } + if stat.TotalBytes, err = d.Uint64(); err != nil { + return StatFS{}, err + } + if stat.AvailableBytes, err = d.Uint64(); err != nil { + return StatFS{}, err + } + if stat.FreeBytes, err = d.Uint64(); err != nil { + return StatFS{}, err + } + if stat.UsedBytes, err = d.Uint64(); err != nil { + return StatFS{}, err + } + if stat.TotalFiles, err = d.Uint64(); err != nil { + return StatFS{}, err + } + if stat.FreeFiles, err = d.Uint64(); err != nil { + return StatFS{}, err + } + return stat, nil +} + +func (d *Decoder) Done() error { + if d.remaining() != 0 { + return fmt.Errorf("FSKit protocol payload has %d trailing bytes", d.remaining()) + } + return nil +} diff --git a/internal/fskitproto/protocol.go b/internal/fskitproto/protocol.go new file mode 100644 index 0000000..030df5e --- /dev/null +++ b/internal/fskitproto/protocol.go @@ -0,0 +1,274 @@ +package fskitproto + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "time" +) + +const ( + Version uint16 = 2 + HeaderSize = 40 + DefaultMaxPayload = 32 << 20 + OpenFlagSnapshot uint32 = 1 << 31 + CapabilityNativeReadFD uint32 = 1 << 0 + CapabilitySharedReadFD uint32 = 1 << 1 + CapabilitySharedWindow uint32 = 1 << 2 + CapabilitySharedFileWindow uint32 = 1 << 3 + CapabilityContentGeneration uint32 = 1 << 4 + FlagNativeReadFD uint32 = 1 << 0 + FlagSharedReadFD uint32 = 1 << 1 + FlagSharedWindow uint32 = 1 << 2 + FlagSharedFileWindow uint32 = 1 << 3 + NativeReadFDMarker = byte(0x46) + SharedReadFDMarker = byte(0x53) + SharedWindowFDMarker = byte(0x57) + SharedFileWindowFDMarker = byte(0x52) +) + +var ( + frameMagic = [4]byte{'C', 'F', 'S', 'P'} + descriptorMagic = [4]byte{'C', 'F', 'S', 'R'} +) + +type Kind uint8 + +const ( + KindRequest Kind = 1 + KindResponse Kind = 2 +) + +type Op uint8 + +const ( + OpHello Op = iota + 1 + OpPing + OpGetattr + OpReadDir + OpOpen + OpCreate + OpRead + OpWrite + OpFsync + OpFlush + OpRelease + OpTruncate + OpMkdir + OpRename + OpUnlink + OpRmdir + OpStatfs + OpSync + OpNamespaceVersion + OpSetattr + OpGetXattr + OpSetXattr + OpListXattrs +) + +const ( + SetAttrMode uint32 = 1 << iota + SetAttrUID + SetAttrGID + SetAttrAccessTime + SetAttrModifyTime +) + +type XattrPolicy uint32 + +const ( + XattrAlwaysSet XattrPolicy = iota + XattrMustCreate + XattrMustReplace + XattrDelete +) + +type EntryType uint8 + +const ( + EntryUnknown EntryType = iota + EntryFile + EntryDirectory + EntrySymlink +) + +type Frame struct { + Kind Kind + Op Op + Flags uint32 + RequestID uint64 + Generation uint64 + Status int32 + Payload []byte +} + +func ReadFrame(reader io.Reader, maxPayload uint32) (Frame, error) { + if maxPayload == 0 { + maxPayload = DefaultMaxPayload + } + header := make([]byte, HeaderSize) + if _, err := io.ReadFull(reader, header); err != nil { + return Frame{}, err + } + if string(header[:4]) != string(frameMagic[:]) { + return Frame{}, errors.New("invalid FSKit protocol magic") + } + if version := binary.LittleEndian.Uint16(header[4:6]); version != Version { + return Frame{}, fmt.Errorf("unsupported FSKit protocol version %d", version) + } + payloadLength := binary.LittleEndian.Uint32(header[32:36]) + if payloadLength > maxPayload { + return Frame{}, fmt.Errorf("FSKit protocol payload %d exceeds limit %d", payloadLength, maxPayload) + } + payload := make([]byte, payloadLength) + if _, err := io.ReadFull(reader, payload); err != nil { + return Frame{}, err + } + return Frame{ + Kind: Kind(header[6]), + Op: Op(header[7]), + Flags: binary.LittleEndian.Uint32(header[8:12]), + RequestID: binary.LittleEndian.Uint64(header[12:20]), + Generation: binary.LittleEndian.Uint64(header[20:28]), + Status: int32(binary.LittleEndian.Uint32(header[28:32])), + Payload: payload, + }, nil +} + +func WriteFrame(writer io.Writer, frame Frame, maxPayload uint32) error { + if err := WriteFrameHeader(writer, frame, len(frame.Payload), maxPayload); err != nil { + return err + } + return WriteFramePayload(writer, frame.Payload) +} + +// WriteFrameHeader writes only the fixed protocol header. The caller may then +// stream the payload without first materializing it in a Go buffer. +func WriteFrameHeader(writer io.Writer, frame Frame, payloadLength int, maxPayload uint32) error { + if maxPayload == 0 { + maxPayload = DefaultMaxPayload + } + if payloadLength < 0 || uint64(payloadLength) > uint64(maxPayload) { + return fmt.Errorf("FSKit protocol payload %d exceeds limit %d", payloadLength, maxPayload) + } + header := make([]byte, HeaderSize) + copy(header[:4], frameMagic[:]) + binary.LittleEndian.PutUint16(header[4:6], Version) + header[6] = byte(frame.Kind) + header[7] = byte(frame.Op) + binary.LittleEndian.PutUint32(header[8:12], frame.Flags) + binary.LittleEndian.PutUint64(header[12:20], frame.RequestID) + binary.LittleEndian.PutUint64(header[20:28], frame.Generation) + binary.LittleEndian.PutUint32(header[28:32], uint32(frame.Status)) + binary.LittleEndian.PutUint32(header[32:36], uint32(payloadLength)) + return writeAll(writer, header) +} + +func WriteFramePayload(writer io.Writer, payload []byte) error { + return writeAll(writer, payload) +} + +func writeAll(writer io.Writer, data []byte) error { + for len(data) > 0 { + n, err := writer.Write(data) + if err != nil { + return err + } + if n == 0 { + return io.ErrUnexpectedEOF + } + data = data[n:] + } + return nil +} + +type Entry struct { + Path string + Name string + NodeID uint64 + ParentID uint64 + Type EntryType + Mode uint32 + UID uint32 + GID uint32 + Size uint64 + AllocSize uint64 + ModTime time.Time + ChangeTime time.Time + AccessTime time.Time + NamespaceID uint64 + ContentGeneration uint64 +} + +type StatFS struct { + BlockSize uint32 + IOSize uint32 + TotalBytes uint64 + AvailableBytes uint64 + FreeBytes uint64 + UsedBytes uint64 + TotalFiles uint64 + FreeFiles uint64 +} + +type Descriptor struct { + Generation uint64 + SocketPath string + Token []byte +} + +func EncodeDescriptor(descriptor Descriptor) ([]byte, error) { + if descriptor.Generation == 0 { + return nil, errors.New("descriptor generation is required") + } + if descriptor.SocketPath == "" || len(descriptor.SocketPath) > 4096 { + return nil, errors.New("descriptor socket path is invalid") + } + if len(descriptor.Token) < 16 || len(descriptor.Token) > 256 { + return nil, errors.New("descriptor token length is invalid") + } + encoder := NewEncoder(24 + len(descriptor.SocketPath) + len(descriptor.Token)) + encoder.Raw(descriptorMagic[:]) + encoder.Uint16(Version) + encoder.Uint16(0) + encoder.Uint64(descriptor.Generation) + encoder.String(descriptor.SocketPath) + encoder.Bytes(descriptor.Token) + return encoder.Data(), nil +} + +func DecodeDescriptor(data []byte) (Descriptor, error) { + decoder := NewDecoder(data) + magic, err := decoder.Raw(4) + if err != nil || string(magic) != string(descriptorMagic[:]) { + return Descriptor{}, errors.New("invalid FSKit resource descriptor magic") + } + version, err := decoder.Uint16() + if err != nil { + return Descriptor{}, err + } + if version != Version { + return Descriptor{}, fmt.Errorf("unsupported FSKit resource descriptor version %d", version) + } + if _, err := decoder.Uint16(); err != nil { + return Descriptor{}, err + } + generation, err := decoder.Uint64() + if err != nil || generation == 0 { + return Descriptor{}, errors.New("invalid FSKit resource descriptor generation") + } + socketPath, err := decoder.String(4096) + if err != nil || socketPath == "" { + return Descriptor{}, errors.New("invalid FSKit resource descriptor socket path") + } + token, err := decoder.Bytes(256) + if err != nil || len(token) < 16 { + return Descriptor{}, errors.New("invalid FSKit resource descriptor token") + } + if err := decoder.Done(); err != nil { + return Descriptor{}, err + } + return Descriptor{Generation: generation, SocketPath: socketPath, Token: token}, nil +} diff --git a/internal/fskitproto/protocol_test.go b/internal/fskitproto/protocol_test.go new file mode 100644 index 0000000..d858d0f --- /dev/null +++ b/internal/fskitproto/protocol_test.go @@ -0,0 +1,175 @@ +package fskitproto + +import ( + "bytes" + "os" + "path/filepath" + "reflect" + "testing" + "time" +) + +func TestFrameRoundTrip(t *testing.T) { + want := Frame{Kind: KindRequest, Op: OpWrite, Flags: 7, RequestID: 99, Generation: 42, Status: -5, Payload: []byte("payload")} + var buffer bytes.Buffer + if err := WriteFrame(&buffer, want, 0); err != nil { + t.Fatal(err) + } + got, err := ReadFrame(&buffer, 0) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("frame = %#v, want %#v", got, want) + } +} + +func TestFrameHeaderAndPayloadRoundTrip(t *testing.T) { + want := Frame{Kind: KindResponse, Op: OpRead, RequestID: 12, Generation: 34, Status: 0} + payload := []byte("streamed payload") + var buffer bytes.Buffer + if err := WriteFrameHeader(&buffer, want, len(payload), 0); err != nil { + t.Fatal(err) + } + if err := WriteFramePayload(&buffer, payload); err != nil { + t.Fatal(err) + } + got, err := ReadFrame(&buffer, 0) + if err != nil { + t.Fatal(err) + } + want.Payload = payload + if !reflect.DeepEqual(got, want) { + t.Fatalf("frame = %#v, want %#v", got, want) + } +} + +func TestEntryRoundTrip(t *testing.T) { + want := Entry{ + Path: "/sessions/2026/07/session.jsonl", Name: "session.jsonl", NodeID: 9, ParentID: 8, + Type: EntryFile, Mode: 0o600, UID: 501, GID: 20, Size: 1234, AllocSize: 4096, + ModTime: time.Unix(1_700_000_000, 123), ChangeTime: time.Unix(1_700_000_001, 456), + AccessTime: time.Unix(1_700_000_002, 789), NamespaceID: 33, ContentGeneration: 7, + } + encoder := NewEncoder(256) + encoder.EntryForCapabilities(want, CapabilityContentGeneration) + decoder := NewDecoder(encoder.Data()) + got, err := decoder.EntryForCapabilities(CapabilityContentGeneration) + if err != nil { + t.Fatal(err) + } + if err := decoder.Done(); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("entry = %#v, want %#v", got, want) + } +} + +func TestEntryContentGenerationIsBackwardCompatible(t *testing.T) { + want := Entry{Path: "/sessions", Name: "sessions", Type: EntryDirectory, ContentGeneration: 9} + legacy := NewEncoder(128) + legacy.Entry(want) + legacyDecoder := NewDecoder(legacy.Data()) + legacyEntry, err := legacyDecoder.Entry() + if err != nil || legacyDecoder.Done() != nil { + t.Fatalf("legacy entry decode: entry=%#v err=%v", legacyEntry, err) + } + if legacyEntry.ContentGeneration != 0 { + t.Fatalf("legacy content generation = %d, want 0", legacyEntry.ContentGeneration) + } + + negotiated := NewEncoder(136) + negotiated.EntryForCapabilities(want, CapabilityContentGeneration) + negotiatedDecoder := NewDecoder(negotiated.Data()) + negotiatedEntry, err := negotiatedDecoder.EntryForCapabilities(CapabilityContentGeneration) + if err != nil || negotiatedDecoder.Done() != nil { + t.Fatalf("negotiated entry decode: entry=%#v err=%v", negotiatedEntry, err) + } + if negotiatedEntry.ContentGeneration != want.ContentGeneration { + t.Fatalf("negotiated content generation = %d, want %d", negotiatedEntry.ContentGeneration, want.ContentGeneration) + } +} + +func TestDescriptorRoundTrip(t *testing.T) { + want := Descriptor{Generation: 17, SocketPath: "/tmp/codexfold.sock", Token: bytes.Repeat([]byte{0x5a}, 32)} + encoded, err := EncodeDescriptor(want) + if err != nil { + t.Fatal(err) + } + got, err := DecodeDescriptor(encoded) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("descriptor = %#v, want %#v", got, want) + } +} + +func TestResourceDescriptorPathSupportsSecurityScopedDirectoryAndLegacyFile(t *testing.T) { + root := t.TempDir() + directoryResource := filepath.Join(root, "native-fskit") + if err := os.Mkdir(directoryResource, 0o700); err != nil { + t.Fatal(err) + } + path, err := ResourceDescriptorPath(directoryResource) + if err != nil || path != filepath.Join(directoryResource, DescriptorFilename) { + t.Fatalf("directory descriptor = %q err=%v", path, err) + } + legacy := filepath.Join(root, "resource.bin") + if err := os.WriteFile(legacy, []byte("legacy"), 0o600); err != nil { + t.Fatal(err) + } + path, err = ResourceDescriptorPath(legacy) + if err != nil || path != legacy { + t.Fatalf("legacy descriptor = %q err=%v", path, err) + } +} + +func TestReadFrameRejectsOversizedPayloadBeforeAllocation(t *testing.T) { + frame := Frame{Kind: KindRequest, Op: OpWrite, Payload: bytes.Repeat([]byte("x"), 64)} + var buffer bytes.Buffer + if err := WriteFrame(&buffer, frame, 128); err != nil { + t.Fatal(err) + } + if _, err := ReadFrame(&buffer, 32); err == nil { + t.Fatal("ReadFrame unexpectedly accepted an oversized payload") + } +} + +func TestDefaultFrameLimitSupportsBoundedReadAhead(t *testing.T) { + frame := Frame{Kind: KindResponse, Op: OpRead} + var accepted bytes.Buffer + if err := WriteFrameHeader(&accepted, frame, 31<<20, 0); err != nil { + t.Fatalf("31 MiB read-ahead frame: %v", err) + } + var rejected bytes.Buffer + if err := WriteFrameHeader(&rejected, frame, DefaultMaxPayload+1, 0); err == nil { + t.Fatal("default frame limit accepted a payload larger than its bound") + } +} + +func TestTransferCapabilitiesAndFlagsRemainDistinct(t *testing.T) { + values := []uint32{ + CapabilityNativeReadFD, + CapabilitySharedReadFD, + CapabilitySharedWindow, + CapabilitySharedFileWindow, + CapabilityContentGeneration, + FlagNativeReadFD, + FlagSharedReadFD, + FlagSharedWindow, + FlagSharedFileWindow, + } + for index, value := range values { + if value == 0 || value&(value-1) != 0 { + t.Fatalf("transfer value %d = %#x, want one bit", index, value) + } + } + if CapabilitySharedFileWindow == CapabilitySharedWindow { + t.Fatal("shared-file and POSIX shared-memory capabilities overlap") + } + if FlagSharedFileWindow == FlagSharedWindow { + t.Fatal("shared-file and POSIX shared-memory flags overlap") + } +} diff --git a/internal/launcher/parent.go b/internal/launcher/parent.go new file mode 100644 index 0000000..8a589ee --- /dev/null +++ b/internal/launcher/parent.go @@ -0,0 +1,50 @@ +package launcher + +import ( + "context" + "errors" + "os" + "strconv" + "time" +) + +const ParentPIDEnvironment = "CODEXFOLD_LAUNCHER_PARENT_PID" + +func MonitorContext(parent context.Context) (context.Context, context.CancelFunc, error) { + return monitorParent(parent, os.Getenv(ParentPIDEnvironment), os.Getppid, 50*time.Millisecond) +} + +func monitorParent(parent context.Context, value string, currentParent func() int, interval time.Duration) (context.Context, context.CancelFunc, error) { + ctx, cancel := context.WithCancel(parent) + if value == "" { + return ctx, cancel, nil + } + expected, err := strconv.Atoi(value) + if err != nil || expected <= 1 { + cancel() + return nil, nil, errors.New("invalid CodexFold launcher parent PID") + } + if currentParent == nil || currentParent() != expected { + cancel() + return nil, nil, errors.New("CodexFold launcher parent is already unavailable") + } + if interval <= 0 { + interval = 50 * time.Millisecond + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if currentParent() != expected { + cancel() + return + } + } + } + }() + return ctx, cancel, nil +} diff --git a/internal/launcher/parent_test.go b/internal/launcher/parent_test.go new file mode 100644 index 0000000..c6174a7 --- /dev/null +++ b/internal/launcher/parent_test.go @@ -0,0 +1,46 @@ +package launcher + +import ( + "context" + "sync/atomic" + "testing" + "time" +) + +func TestMonitorParentCancelsWhenLauncherDisappears(t *testing.T) { + var parent atomic.Int64 + parent.Store(42) + ctx, cancel, err := monitorParent(context.Background(), "42", func() int { return int(parent.Load()) }, time.Millisecond) + if err != nil { + t.Fatal(err) + } + defer cancel() + parent.Store(1) + select { + case <-ctx.Done(): + case <-time.After(time.Second): + t.Fatal("launcher parent loss did not cancel the context") + } +} + +func TestMonitorParentRejectsInvalidOrAlreadyLostLauncher(t *testing.T) { + for _, value := range []string{"abc", "0", "1", "43"} { + if _, _, err := monitorParent(context.Background(), value, func() int { return 42 }, time.Millisecond); err == nil { + t.Fatalf("monitorParent(%q) succeeded", value) + } + } +} + +func TestMonitorParentIsDisabledWithoutLauncherEnvironment(t *testing.T) { + parent := context.Background() + ctx, cancel, err := monitorParent(parent, "", func() int { return 1 }, time.Millisecond) + if err != nil { + t.Fatal(err) + } + defer cancel() + select { + case <-ctx.Done(): + t.Fatal("unset launcher environment canceled an ordinary process") + default: + } +} diff --git a/internal/mountfs/dependency_boundary_test.go b/internal/mountfs/dependency_boundary_test.go new file mode 100644 index 0000000..9edd662 --- /dev/null +++ b/internal/mountfs/dependency_boundary_test.go @@ -0,0 +1,54 @@ +package mountfs + +import ( + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestAdapterCoreDoesNotDependOnCodexDatabase(t *testing.T) { + repoRoot := dependencyRepoRoot(t) + + assertPackageExcludesDependencies(t, repoRoot, "./internal/fold", []string{ + "github.com/samekind/codexfold/internal/codex", + "modernc.org", + }) + assertPackageExcludesDependencies(t, repoRoot, "./internal/mountfs", []string{ + "github.com/samekind/codexfold/internal/codex", + "github.com/samekind/codexfold/internal/service", + "modernc.org", + }) +} + +func dependencyRepoRoot(t *testing.T) string { + t.Helper() + _, currentFile, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("resolve dependency boundary test source path") + } + return filepath.Clean(filepath.Join(filepath.Dir(currentFile), "../..")) +} + +func assertPackageExcludesDependencies(t *testing.T, repoRoot, packagePath string, forbidden []string) { + t.Helper() + cmd := exec.Command("go", "list", "-deps", packagePath) + cmd.Dir = repoRoot + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("list dependencies for %s: %v\n%s", packagePath, err, output) + } + + dependencies := make(map[string]struct{}) + for _, dependency := range strings.Fields(string(output)) { + dependencies[dependency] = struct{}{} + } + for dependency := range dependencies { + for _, forbiddenPrefix := range forbidden { + if dependency == forbiddenPrefix || strings.HasPrefix(dependency, forbiddenPrefix+"/") { + t.Errorf("%s must not depend on %s", packagePath, dependency) + } + } + } +} diff --git a/internal/mountfs/file_metadata_darwin.go b/internal/mountfs/file_metadata_darwin.go new file mode 100644 index 0000000..e45acc3 --- /dev/null +++ b/internal/mountfs/file_metadata_darwin.go @@ -0,0 +1,28 @@ +//go:build darwin + +package mountfs + +import ( + "fmt" + "os" + "syscall" + "time" +) + +func fileObjectIdentity(info os.FileInfo) string { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return "" + } + return fmt.Sprintf("native:%d:%d", stat.Dev, stat.Ino) +} + +func fileOwnershipAndTimes(info os.FileInfo) (uint32, uint32, time.Time, time.Time) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return uint32(os.Getuid()), uint32(os.Getgid()), info.ModTime(), info.ModTime() + } + return stat.Uid, stat.Gid, + time.Unix(stat.Atimespec.Sec, stat.Atimespec.Nsec), + time.Unix(stat.Ctimespec.Sec, stat.Ctimespec.Nsec) +} diff --git a/internal/mountfs/file_metadata_linux.go b/internal/mountfs/file_metadata_linux.go new file mode 100644 index 0000000..f123a00 --- /dev/null +++ b/internal/mountfs/file_metadata_linux.go @@ -0,0 +1,28 @@ +//go:build linux + +package mountfs + +import ( + "fmt" + "os" + "syscall" + "time" +) + +func fileObjectIdentity(info os.FileInfo) string { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return "" + } + return fmt.Sprintf("native:%d:%d", stat.Dev, stat.Ino) +} + +func fileOwnershipAndTimes(info os.FileInfo) (uint32, uint32, time.Time, time.Time) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return uint32(os.Getuid()), uint32(os.Getgid()), info.ModTime(), info.ModTime() + } + return stat.Uid, stat.Gid, + time.Unix(stat.Atim.Sec, stat.Atim.Nsec), + time.Unix(stat.Ctim.Sec, stat.Ctim.Nsec) +} diff --git a/internal/mountfs/file_metadata_other.go b/internal/mountfs/file_metadata_other.go new file mode 100644 index 0000000..055ea44 --- /dev/null +++ b/internal/mountfs/file_metadata_other.go @@ -0,0 +1,16 @@ +//go:build !darwin && !linux + +package mountfs + +import ( + "os" + "time" +) + +func fileObjectIdentity(os.FileInfo) string { + return "" +} + +func fileOwnershipAndTimes(info os.FileInfo) (uint32, uint32, time.Time, time.Time) { + return uint32(os.Getuid()), uint32(os.Getgid()), info.ModTime(), info.ModTime() +} diff --git a/internal/mountfs/filesystem.go b/internal/mountfs/filesystem.go new file mode 100644 index 0000000..5f5ab13 --- /dev/null +++ b/internal/mountfs/filesystem.go @@ -0,0 +1,2093 @@ +package mountfs + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + "unicode/utf8" + + "github.com/samekind/codexfold/internal/fskitproto" + "github.com/samekind/codexfold/internal/vfs" +) + +type Attr struct { + Mode uint32 `json:"mode"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` + Size int64 `json:"size"` + ModTime time.Time `json:"mod_time"` + ChangeTime time.Time `json:"change_time"` + AccessTime time.Time `json:"access_time"` + ObjectID string `json:"-"` + DirectoryGeneration uint64 `json:"-"` +} + +type SetAttrRequest struct { + Valid uint32 + Mode uint32 + UID uint32 + GID uint32 + AccessTime time.Time + ModTime time.Time +} + +type sessionOwner struct { + mu sync.Mutex + closer io.Closer + references int + retired bool + closeOnce sync.Once + closeErr error +} + +func newSessionOwner(closer io.Closer) *sessionOwner { + return &sessionOwner{closer: closer} +} + +func (o *sessionOwner) acquire() bool { + if o == nil { + return true + } + o.mu.Lock() + defer o.mu.Unlock() + if o.retired { + return false + } + o.references++ + return true +} + +func (o *sessionOwner) release() error { + if o == nil { + return nil + } + o.mu.Lock() + if o.references <= 0 { + o.mu.Unlock() + return errors.New("session owner reference underflow") + } + o.references-- + ready := o.retired && o.references == 0 + o.mu.Unlock() + if ready { + return o.close() + } + return nil +} + +func (o *sessionOwner) retire() error { + if o == nil { + return nil + } + o.mu.Lock() + o.retired = true + ready := o.references == 0 + o.mu.Unlock() + if ready { + return o.close() + } + return nil +} + +func (o *sessionOwner) close() error { + if o == nil { + return nil + } + o.closeOnce.Do(func() { + if o.closer != nil { + o.closeErr = o.closer.Close() + } + }) + return o.closeErr +} + +type fileHandle struct { + mu sync.Mutex + path string + session *vfs.Session + owner *sessionOwner + native *os.File + nativePath string + nativeAppend *nativeAppendState + read *vfs.ReadHandle + write *vfs.WriteHandle + append bool + appendStream bool + appendFloor int64 + appendOffset int64 +} + +type directoryState struct { + generation uint64 + changedAt time.Time +} + +type Filesystem struct { + mu sync.RWMutex + loadMu sync.Mutex + sessions map[string]*vfs.Session + owners map[string]*sessionOwner + paths map[string]string + retained map[string]string + nativeFirst map[string]struct{} + directories map[string]struct{} + directoryStates map[string]directoryState + handles map[uint64]*fileHandle + next uint64 + loader func(string) (*vfs.Session, io.Closer, error) + canonical bool + nativeRoot string + nativeJournalRoot string + nativeAppends map[string]*nativeAppendState + // nativeNamespaceRefreshMount is the live FSKit mount used only to make + // externally-created native paths visible after macOS cached a negative + // lookup. The in-flight map prevents that refresh probe from ever creating + // a path if its native source disappears during the probe. + nativeNamespaceRefreshMount string + nativeNamespaceRefreshInFlight map[string]uint32 + nativeMutationMu sync.Mutex + nativeInternalMutations map[string]time.Time + namespaceVersion atomic.Uint64 + activeIO atomic.Int64 + lastIO atomic.Int64 +} + +func New() *Filesystem { + now := time.Now() + filesystem := &Filesystem{ + sessions: make(map[string]*vfs.Session), owners: make(map[string]*sessionOwner), + directoryStates: map[string]directoryState{"/": {generation: 1, changedAt: now}}, + handles: make(map[uint64]*fileHandle), next: 1, + } + filesystem.namespaceVersion.Store(1) + filesystem.lastIO.Store(time.Now().UnixNano()) + return filesystem +} + +func NewCanonical() *Filesystem { + now := time.Now() + filesystem := &Filesystem{ + sessions: make(map[string]*vfs.Session), owners: make(map[string]*sessionOwner), paths: make(map[string]string), + retained: make(map[string]string), nativeFirst: make(map[string]struct{}), + directories: map[string]struct{}{`/`: {}, `/sessions`: {}, `/archived_sessions`: {}}, + directoryStates: map[string]directoryState{ + `/`: {generation: 1, changedAt: now}, + `/sessions`: {generation: 1, changedAt: now}, + `/archived_sessions`: {generation: 1, changedAt: now}, + }, + handles: make(map[uint64]*fileHandle), nativeAppends: make(map[string]*nativeAppendState), + nativeNamespaceRefreshInFlight: make(map[string]uint32), + nativeInternalMutations: make(map[string]time.Time), + next: 1, canonical: true, + } + filesystem.namespaceVersion.Store(1) + filesystem.lastIO.Store(time.Now().UnixNano()) + return filesystem +} + +func (f *Filesystem) IOIdleFor(duration time.Duration) bool { + if f.activeIO.Load() != 0 { + return false + } + last := f.lastIO.Load() + return last == 0 || time.Since(time.Unix(0, last)) >= duration +} + +func (f *Filesystem) beginIO() func() { + f.activeIO.Add(1) + return func() { + f.lastIO.Store(time.Now().UnixNano()) + f.activeIO.Add(-1) + } +} + +const nativeInternalMutationSuppressionWindow = 2 * time.Second + +func (f *Filesystem) markNativeInternalMutation(nativePath string) { + if nativePath == "" { + return + } + f.nativeMutationMu.Lock() + if f.nativeInternalMutations == nil { + f.nativeInternalMutations = make(map[string]time.Time) + } + f.nativeInternalMutations[filepath.Clean(nativePath)] = time.Now() + f.nativeMutationMu.Unlock() +} + +func (f *Filesystem) nativeInternalMutationSuppressed(nativePath string) bool { + if nativePath == "" { + return false + } + cleaned := filepath.Clean(nativePath) + f.nativeMutationMu.Lock() + defer f.nativeMutationMu.Unlock() + changedAt, exists := f.nativeInternalMutations[cleaned] + if !exists { + return false + } + if time.Since(changedAt) <= nativeInternalMutationSuppressionWindow { + return true + } + delete(f.nativeInternalMutations, cleaned) + return false +} + +func (f *Filesystem) NamespaceVersion() uint64 { + return f.namespaceVersion.Load() +} + +func (f *Filesystem) bumpNamespaceVersion() { + f.namespaceVersion.Add(1) +} + +func (f *Filesystem) bumpDirectoryGeneration(name string) { + f.mu.Lock() + f.bumpDirectoryGenerationLocked(cleanPath(name), time.Now()) + f.mu.Unlock() +} + +func (f *Filesystem) bumpDirectoryGenerations(names []string) { + if len(names) == 0 { + return + } + now := time.Now() + f.mu.Lock() + for _, name := range names { + f.bumpDirectoryGenerationLocked(cleanPath(name), now) + } + f.mu.Unlock() +} + +func (f *Filesystem) bumpDirectoryGenerationLocked(name string, changedAt time.Time) { + if name == "" || name == "." { + name = "/" + } + state := f.ensureDirectoryStateLocked(name, changedAt) + state.generation++ + if !changedAt.After(state.changedAt) { + changedAt = state.changedAt.Add(time.Nanosecond) + } + state.changedAt = changedAt + f.directoryStates[name] = state +} + +func (f *Filesystem) ensureDirectoryStateLocked(name string, changedAt time.Time) directoryState { + if f.directoryStates == nil { + f.directoryStates = make(map[string]directoryState) + } + if state, exists := f.directoryStates[name]; exists { + return state + } + if changedAt.IsZero() { + changedAt = time.Now() + } + state := directoryState{generation: 1, changedAt: changedAt} + f.directoryStates[name] = state + return state +} + +func (f *Filesystem) directoryAttr(name string, attribute Attr) Attr { + cleaned := cleanPath(name) + initialTime := attribute.ChangeTime + if initialTime.IsZero() { + initialTime = attribute.ModTime + } + f.mu.Lock() + state := f.ensureDirectoryStateLocked(cleaned, initialTime) + f.mu.Unlock() + if attribute.ObjectID == "" { + attribute.ObjectID = "synthetic:" + cleaned + } + attribute.DirectoryGeneration = state.generation + if attribute.ModTime.IsZero() || attribute.ModTime.Before(state.changedAt) { + attribute.ModTime = state.changedAt + } + if attribute.ChangeTime.IsZero() || attribute.ChangeTime.Before(state.changedAt) { + attribute.ChangeTime = state.changedAt + } + if attribute.AccessTime.IsZero() { + attribute.AccessTime = state.changedAt + } + return attribute +} + +func (f *Filesystem) SetNativeRoot(root string) { + if root != "" { + root = filepath.Clean(root) + } + f.mu.Lock() + if f.nativeRoot != root { + f.nativeAppends = make(map[string]*nativeAppendState) + } + f.nativeRoot = root + f.nativeJournalRoot = "" + if root != "" { + f.nativeJournalRoot = filepath.Join(root, ".codexfold-native-journal") + } + for retained := range f.retained { + delete(f.retained, retained) + } + for sessionID, session := range f.sessions { + f.registerRetainedPathLocked(sessionID, session) + } + f.mu.Unlock() +} + +// SetNativeNamespaceRefreshMount installs the mounted namespace used by the +// Darwin watcher to repair a negative kernel lookup after an external native +// path appears. It never changes where session bytes are stored. +func (f *Filesystem) SetNativeNamespaceRefreshMount(mount string) { + if mount != "" { + mount = filepath.Clean(mount) + } + f.mu.Lock() + f.nativeNamespaceRefreshMount = mount + f.mu.Unlock() +} + +func (f *Filesystem) nativeNamespaceRefreshMountPath() string { + f.mu.RLock() + mount := f.nativeNamespaceRefreshMount + f.mu.RUnlock() + return mount +} + +func (f *Filesystem) beginNativeNamespaceRefresh(name string) func() { + cleaned := cleanPath(name) + if cleaned == "" || !canonicalNamespacePath(cleaned) { + return func() {} + } + f.mu.Lock() + if f.nativeNamespaceRefreshInFlight == nil { + f.nativeNamespaceRefreshInFlight = make(map[string]uint32) + } + f.nativeNamespaceRefreshInFlight[cleaned]++ + f.mu.Unlock() + var once sync.Once + return func() { + once.Do(func() { + f.mu.Lock() + if count := f.nativeNamespaceRefreshInFlight[cleaned]; count <= 1 { + delete(f.nativeNamespaceRefreshInFlight, cleaned) + } else { + f.nativeNamespaceRefreshInFlight[cleaned] = count - 1 + } + f.mu.Unlock() + }) + } +} + +func (f *Filesystem) nativeNamespaceRefreshActive(name string) bool { + cleaned := cleanPath(name) + if cleaned == "" { + return false + } + f.mu.RLock() + active := f.nativeNamespaceRefreshInFlight[cleaned] != 0 + f.mu.RUnlock() + return active +} + +func (f *Filesystem) refreshNativeNamespacePath(name string, directory bool) error { + cleaned := cleanPath(name) + if cleaned == "" || !canonicalNamespacePath(cleaned) { + return nil + } + mount := f.nativeNamespaceRefreshMountPath() + if mount == "" { + return nil + } + f.mu.RLock() + root := f.nativeRoot + f.mu.RUnlock() + if root == "" { + return nil + } + nativePath := nativePathFromRoot(root, cleaned) + info, err := os.Lstat(nativePath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if info.IsDir() != directory { + return nil + } + release := f.beginNativeNamespaceRefresh(cleaned) + defer release() + mountedPath := nativePathFromRoot(mount, cleaned) + if directory { + err = os.Mkdir(mountedPath, 0o700) + } else { + var file *os.File + file, err = os.OpenFile(mountedPath, os.O_RDONLY|os.O_CREATE, 0) + if file != nil { + closeErr := file.Close() + if err == nil { + err = closeErr + } + } + } + if err == nil && !directory { + return nil + } + if err == nil { + return fmt.Errorf("native namespace refresh unexpectedly created %s", cleaned) + } + if errors.Is(err, os.ErrExist) || errors.Is(err, os.ErrNotExist) { + return nil + } + return err +} + +func (f *Filesystem) RecoverNativeAppendTransactions() error { + f.mu.RLock() + root := f.nativeRoot + journalRoot := f.nativeJournalRoot + f.mu.RUnlock() + if root == "" { + return nil + } + if err := recoverNativeAppendTransactions(root, journalRoot); err != nil { + return err + } + f.mu.Lock() + f.nativeAppends = make(map[string]*nativeAppendState) + f.mu.Unlock() + return nil +} + +func (f *Filesystem) AddSession(sessionID string, session *vfs.Session) error { + return f.AddSessionOwned(sessionID, session, nil) +} + +// AddSessionOwned transfers ownership of closer to the filesystem, including +// when validation or insertion fails. +func (f *Filesystem) AddSessionOwned(sessionID string, session *vfs.Session, closer io.Closer) error { + owner := newSessionOwner(closer) + if sessionID == "" || strings.ContainsAny(sessionID, "/\\\x00") || session == nil { + return errors.Join(errors.New("safe session ID and session are required"), owner.retire()) + } + f.mu.Lock() + if _, exists := f.sessions[sessionID]; exists { + f.mu.Unlock() + return errors.Join(errors.New("session is already mounted"), owner.retire()) + } + f.sessions[sessionID] = session + f.owners[sessionID] = owner + f.mu.Unlock() + f.bumpNamespaceVersion() + return nil +} + +func (f *Filesystem) UpsertSession(sessionID string, session *vfs.Session) error { + return f.UpsertSessionOwned(sessionID, session, nil) +} + +// UpsertSessionOwned transfers ownership of closer to the filesystem. An old +// owner is retired after the replacement becomes visible and stays alive only +// while existing file handles still reference it. +func (f *Filesystem) UpsertSessionOwned(sessionID string, session *vfs.Session, closer io.Closer) error { + owner := newSessionOwner(closer) + if sessionID == "" || strings.ContainsAny(sessionID, "/\\\x00") || session == nil { + return errors.Join(errors.New("safe session ID and session are required"), owner.retire()) + } + f.mu.Lock() + previous := f.owners[sessionID] + f.sessions[sessionID] = session + f.owners[sessionID] = owner + f.mu.Unlock() + f.bumpNamespaceVersion() + return previous.retire() +} + +func (f *Filesystem) AddSessionAt(sessionID string, name string, session *vfs.Session) error { + return f.AddSessionAtOwned(sessionID, name, session, nil) +} + +// AddSessionAtOwned is the canonical-namespace form of AddSessionOwned. +func (f *Filesystem) AddSessionAtOwned(sessionID string, name string, session *vfs.Session, closer io.Closer) error { + owner := newSessionOwner(closer) + cleaned := cleanPath(name) + if !f.canonical || !safeSessionID(sessionID) || session == nil || !canonicalSessionPath(cleaned) { + return errors.Join(errors.New("canonical filesystem, safe session ID, path, and session are required"), owner.retire()) + } + f.mu.Lock() + if _, exists := f.sessions[sessionID]; exists { + f.mu.Unlock() + return errors.Join(errors.New("session is already mounted"), owner.retire()) + } + if _, exists := f.paths[cleaned]; exists { + f.mu.Unlock() + return errors.Join(errors.New("session path is already mounted"), owner.retire()) + } + f.ensureDirectoryChainLocked(path.Dir(cleaned)) + f.sessions[sessionID] = session + f.owners[sessionID] = owner + f.paths[cleaned] = sessionID + f.bumpDirectoryGenerationLocked(path.Dir(cleaned), time.Now()) + delete(f.nativeFirst, sessionID) + f.registerRetainedPathLocked(sessionID, session) + f.mu.Unlock() + f.bumpNamespaceVersion() + return nil +} + +func (f *Filesystem) UpsertSessionAt(sessionID string, name string, session *vfs.Session) error { + return f.UpsertSessionAtOwned(sessionID, name, session, nil) +} + +// UpsertSessionAtOwned is the canonical-namespace form of UpsertSessionOwned. +func (f *Filesystem) UpsertSessionAtOwned(sessionID string, name string, session *vfs.Session, closer io.Closer) error { + owner := newSessionOwner(closer) + cleaned := cleanPath(name) + if !f.canonical || !safeSessionID(sessionID) || session == nil || !canonicalSessionPath(cleaned) { + return errors.Join(errors.New("canonical filesystem, safe session ID, path, and session are required"), owner.retire()) + } + f.mu.Lock() + f.ensureDirectoryChainLocked(path.Dir(cleaned)) + var previousPath string + for route, currentID := range f.paths { + if currentID == sessionID { + previousPath = route + delete(f.paths, route) + } + } + if err := moveManagedMetadata(f.nativeRoot, previousPath, cleaned); err != nil { + if previousPath != "" { + f.paths[previousPath] = sessionID + } + f.mu.Unlock() + return errors.Join(err, owner.retire()) + } + previous := f.owners[sessionID] + f.sessions[sessionID] = session + f.owners[sessionID] = owner + f.paths[cleaned] = sessionID + if previousPath != cleaned { + now := time.Now() + if previousPath != "" { + f.bumpDirectoryGenerationLocked(path.Dir(previousPath), now) + } + f.bumpDirectoryGenerationLocked(path.Dir(cleaned), now) + } + delete(f.nativeFirst, sessionID) + f.registerRetainedPathLocked(sessionID, session) + f.mu.Unlock() + f.bumpNamespaceVersion() + return previous.retire() +} + +func (f *Filesystem) MoveSessionAt(sessionID string, name string) error { + cleaned := cleanPath(name) + if !f.canonical || !safeSessionID(sessionID) || !canonicalSessionPath(cleaned) { + return errors.New("canonical filesystem, safe session ID, and path are required") + } + f.mu.Lock() + defer f.mu.Unlock() + if _, exists := f.sessions[sessionID]; !exists { + return os.ErrNotExist + } + f.ensureDirectoryChainLocked(path.Dir(cleaned)) + var previousPath string + for route, currentID := range f.paths { + if currentID == sessionID { + previousPath = route + delete(f.paths, route) + } + } + if err := moveManagedMetadata(f.nativeRoot, previousPath, cleaned); err != nil { + if previousPath != "" { + f.paths[previousPath] = sessionID + } + return err + } + f.paths[cleaned] = sessionID + if previousPath != cleaned { + now := time.Now() + if previousPath != "" { + f.bumpDirectoryGenerationLocked(path.Dir(previousPath), now) + } + f.bumpDirectoryGenerationLocked(path.Dir(cleaned), now) + } + f.bumpNamespaceVersion() + return nil +} + +func (f *Filesystem) PreferNativeSession(sessionID string) error { + if !f.canonical || !safeSessionID(sessionID) { + return errors.New("canonical filesystem and safe session ID are required") + } + f.mu.Lock() + defer f.mu.Unlock() + if _, exists := f.sessions[sessionID]; !exists { + return os.ErrNotExist + } + f.nativeFirst[sessionID] = struct{}{} + return nil +} + +func (f *Filesystem) RemoveSession(sessionID string) error { + if !safeSessionID(sessionID) { + return errors.New("safe session ID is required") + } + f.mu.Lock() + if _, exists := f.sessions[sessionID]; !exists { + f.mu.Unlock() + return os.ErrNotExist + } + owner := f.owners[sessionID] + delete(f.sessions, sessionID) + delete(f.owners, sessionID) + delete(f.nativeFirst, sessionID) + for route, currentID := range f.paths { + if currentID == sessionID { + delete(f.paths, route) + f.bumpDirectoryGenerationLocked(path.Dir(route), time.Now()) + } + } + for retained, currentID := range f.retained { + if currentID == sessionID { + delete(f.retained, retained) + } + } + f.mu.Unlock() + f.bumpNamespaceVersion() + return owner.retire() +} + +// CloseSessions retires every mounted session owner. Resources referenced by +// already-open file handles remain valid until those handles are released. +func (f *Filesystem) CloseSessions() error { + f.mu.Lock() + owners := make([]*sessionOwner, 0, len(f.owners)) + for _, owner := range f.owners { + owners = append(owners, owner) + } + f.sessions = make(map[string]*vfs.Session) + f.owners = make(map[string]*sessionOwner) + for route := range f.paths { + f.bumpDirectoryGenerationLocked(path.Dir(route), time.Now()) + delete(f.paths, route) + } + for retained := range f.retained { + delete(f.retained, retained) + } + for sessionID := range f.nativeFirst { + delete(f.nativeFirst, sessionID) + } + f.mu.Unlock() + f.bumpNamespaceVersion() + var result error + for _, owner := range owners { + result = errors.Join(result, owner.retire()) + } + return result +} + +func (f *Filesystem) SetSessionLoader(loader func(string) (*vfs.Session, error)) { + if loader == nil { + f.SetOwnedSessionLoader(nil) + return + } + f.SetOwnedSessionLoader(func(sessionID string) (*vfs.Session, io.Closer, error) { + session, err := loader(sessionID) + return session, nil, err + }) +} + +func (f *Filesystem) SetOwnedSessionLoader(loader func(string) (*vfs.Session, io.Closer, error)) { + f.mu.Lock() + f.loader = loader + f.mu.Unlock() +} + +func (f *Filesystem) ReadDir(name string) ([]string, syscall.Errno) { + cleaned := cleanPath(name) + if !f.canonical && cleaned != "/" { + return nil, syscall.ENOTDIR + } + f.mu.RLock() + if f.canonical { + if !canonicalNamespacePath(cleaned) { + f.mu.RUnlock() + return nil, syscall.ENOTDIR + } + _, virtualDirectory := f.directories[cleaned] + nativeRoot := f.nativeRoot + if !virtualDirectory && nativeRoot == "" { + f.mu.RUnlock() + return nil, syscall.ENOTDIR + } + if !virtualDirectory && nativeRoot != "" { + info, err := os.Stat(nativePathFromRoot(nativeRoot, cleaned)) + if err != nil || !info.IsDir() { + f.mu.RUnlock() + return nil, syscall.ENOTDIR + } + } + entrySet := make(map[string]struct{}) + hiddenEntries := make(map[string]struct{}) + for retained := range f.retained { + if path.Dir(retained) == cleaned { + hiddenEntries[path.Base(retained)] = struct{}{} + } + } + for directory := range f.directories { + if directory != cleaned && path.Dir(directory) == cleaned { + entrySet[path.Base(directory)] = struct{}{} + } + } + for route := range f.paths { + if path.Dir(route) == cleaned { + entrySet[path.Base(route)] = struct{}{} + } + } + f.mu.RUnlock() + if nativeRoot != "" && cleaned != "/" { + if nativeEntries, err := os.ReadDir(nativePathFromRoot(nativeRoot, cleaned)); err == nil { + for _, entry := range nativeEntries { + if _, hidden := hiddenEntries[entry.Name()]; hidden { + continue + } + entrySet[entry.Name()] = struct{}{} + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, errnoFor(err) + } + } + entries := make([]string, 0, len(entrySet)) + for entry := range entrySet { + entries = append(entries, entry) + } + sort.Strings(entries) + return entries, 0 + } + entries := make([]string, 0, len(f.sessions)) + for sessionID := range f.sessions { + entries = append(entries, sessionID+".jsonl") + } + f.mu.RUnlock() + sort.Strings(entries) + return entries, 0 +} + +func (f *Filesystem) Getattr(name string) (Attr, syscall.Errno) { + cleaned := cleanPath(name) + if cleaned == "/" { + if nativePath, ok := f.nativeMetadataPath(cleaned); ok { + if info, err := os.Lstat(nativePath); err == nil { + return f.directoryAttr(cleaned, attrFromFileInfo(info, 0)), 0 + } + } + return f.directoryAttr(cleaned, syntheticAttr(syscall.S_IFDIR|0o700)), 0 + } + if f.canonical { + f.mu.RLock() + _, directory := f.directories[cleaned] + f.mu.RUnlock() + if directory { + if nativePath, ok := f.nativeMetadataPath(cleaned); ok { + if info, err := os.Lstat(nativePath); err == nil { + return f.directoryAttr(cleaned, attrFromFileInfo(info, 0)), 0 + } + } + return f.directoryAttr(cleaned, syntheticAttr(syscall.S_IFDIR|0o700)), 0 + } + if session, _, errno := f.sessionForPath(cleaned); errno == 0 { + return sessionAttr(session, f.managedObjectID(cleaned)) + } + if nativePath, ok := f.nativePath(cleaned); ok { + info, err := os.Lstat(nativePath) + if err == nil { + size := info.Size() + if state := f.nativeAppendState(nativePath); state != nil { + size = state.VisibleSizeForBacking(size) + } + attribute := attrFromFileInfo(info, size) + if info.IsDir() { + attribute = f.directoryAttr(cleaned, attribute) + } + return attribute, 0 + } + if !errors.Is(err, os.ErrNotExist) { + return Attr{}, errnoFor(err) + } + } + } + session, _, errno := f.sessionForPath(cleaned) + if errno != 0 { + return Attr{}, errno + } + return sessionAttr(session, "") +} + +func sessionAttr(session *vfs.Session, objectID string) (Attr, syscall.Errno) { + info, err := session.VisibleInfo() + if err != nil { + return Attr{}, errnoFor(err) + } + metadata, err := os.Lstat(session.MetadataPath()) + if err != nil { + return Attr{}, errnoFor(err) + } + attribute := attrFromFileInfo(metadata, info.Size) + attribute.ObjectID = objectID + return attribute, 0 +} + +func (f *Filesystem) managedObjectID(name string) string { + f.mu.RLock() + sessionID := f.paths[name] + f.mu.RUnlock() + if sessionID == "" { + return "" + } + return "managed:" + sessionID +} + +func (f *Filesystem) SetAttributes(name string, request SetAttrRequest) syscall.Errno { + metadataPath, _, errno := f.metadataPath(name) + if errno != 0 { + return errno + } + info, err := os.Lstat(metadataPath) + if err != nil { + return errnoFor(err) + } + if request.Valid&fskitproto.SetAttrUID != 0 || request.Valid&fskitproto.SetAttrGID != 0 { + uid, gid, _, _ := fileOwnershipAndTimes(info) + if request.Valid&fskitproto.SetAttrUID != 0 { + uid = request.UID + } + if request.Valid&fskitproto.SetAttrGID != 0 { + gid = request.GID + } + if err := os.Chown(metadataPath, int(uid), int(gid)); err != nil { + return errnoFor(err) + } + } + if request.Valid&fskitproto.SetAttrMode != 0 { + if err := os.Chmod(metadataPath, os.FileMode(request.Mode)&os.ModePerm); err != nil { + return errnoFor(err) + } + } + if request.Valid&(fskitproto.SetAttrAccessTime|fskitproto.SetAttrModifyTime) != 0 { + _, _, accessTime, _ := fileOwnershipAndTimes(info) + modifyTime := info.ModTime() + if request.Valid&fskitproto.SetAttrAccessTime != 0 { + accessTime = request.AccessTime + } + if request.Valid&fskitproto.SetAttrModifyTime != 0 { + modifyTime = request.ModTime + } + if err := os.Chtimes(metadataPath, accessTime, modifyTime); err != nil { + return errnoFor(err) + } + } + return 0 +} + +func (f *Filesystem) GetXattr(name string, attribute string) ([]byte, syscall.Errno) { + xattrPath, managed, errno := f.xattrPath(name, false) + if errno != 0 { + return nil, errno + } + value, err := platformGetXattr(xattrPath, attribute) + if err != nil { + if managed && errors.Is(err, os.ErrNotExist) { + return nil, xattrMissingErrno() + } + return nil, errnoFor(err) + } + return value, 0 +} + +func (f *Filesystem) SetXattr(name string, attribute string, value []byte, policy fskitproto.XattrPolicy) syscall.Errno { + if attribute == "" || strings.ContainsRune(attribute, '\x00') { + return syscall.EINVAL + } + createCarrier := policy != fskitproto.XattrDelete + xattrPath, managed, errno := f.xattrPath(name, createCarrier) + if errno != 0 { + return errno + } + if policy == fskitproto.XattrDelete { + if managed { + if _, err := os.Lstat(xattrPath); errors.Is(err, os.ErrNotExist) { + return xattrMissingErrno() + } else if err != nil { + return errnoFor(err) + } + } + return errnoFor(platformRemoveXattr(xattrPath, attribute)) + } + return errnoFor(platformSetXattr(xattrPath, attribute, value, policy)) +} + +func (f *Filesystem) ListXattrs(name string) ([]string, syscall.Errno) { + xattrPath, managed, errno := f.xattrPath(name, false) + if errno != 0 { + return nil, errno + } + attributes, err := platformListXattrs(xattrPath) + if err != nil { + if managed && errors.Is(err, os.ErrNotExist) { + return []string{}, 0 + } + return nil, errnoFor(err) + } + sort.Strings(attributes) + return attributes, 0 +} + +func syntheticAttr(mode uint32) Attr { + return Attr{ + Mode: mode, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), + } +} + +func attrFromFileInfo(info os.FileInfo, size int64) Attr { + mode := uint32(info.Mode().Perm()) + switch { + case info.Mode()&os.ModeSymlink != 0: + mode |= syscall.S_IFLNK + case info.IsDir(): + mode |= syscall.S_IFDIR + default: + mode |= syscall.S_IFREG + } + if size == 0 && !info.IsDir() { + size = info.Size() + } + uid, gid, accessTime, changeTime := fileOwnershipAndTimes(info) + return Attr{ + Mode: mode, UID: uid, GID: gid, Size: size, + ModTime: info.ModTime(), ChangeTime: changeTime, AccessTime: accessTime, + ObjectID: fileObjectIdentity(info), + } +} + +func (f *Filesystem) Open(name string, flags int) (uint64, syscall.Errno) { + session, owner, errno := f.acquireSessionForPath(name) + if errno != 0 { + if !f.canonical { + return 0, errno + } + nativePath, ok := f.nativePath(cleanPath(name)) + if !ok { + return 0, errno + } + // FUSE may split or retry one append syscall as positional writes. Keep + // append and truncate semantics in the transaction layer instead of the + // backing descriptor. + nativeFlags := flags &^ (os.O_APPEND | os.O_TRUNC) + native, err := os.OpenFile(nativePath, nativeFlags, 0o600) + if err != nil { + return 0, errnoFor(err) + } + state, err := f.loadNativeAppendState(nativePath) + if err != nil { + _ = native.Close() + return 0, errnoFor(err) + } + access := flags & (os.O_WRONLY | os.O_RDWR) + writable := access == os.O_WRONLY || access == os.O_RDWR + if writable && flags&os.O_TRUNC != 0 { + if err := state.Truncate(0); err != nil { + _ = native.Close() + return 0, errnoFor(err) + } + } + f.mu.Lock() + handleID := f.next + f.next++ + f.handles[handleID] = &fileHandle{ + path: cleanPath(name), native: native, nativePath: nativePath, nativeAppend: state, + // macOS may strip O_APPEND before invoking FUSE. Every writable + // canonical JSONL handle therefore uses positional transaction staging. + append: writable && nativeTransactionPath(cleanPath(name)), + } + f.mu.Unlock() + return handleID, 0 + } + releaseOwner := true + defer func() { + if releaseOwner { + _ = owner.release() + } + }() + handle := &fileHandle{path: cleanPath(name), session: session, owner: owner, append: flags&os.O_APPEND != 0} + access := flags & (os.O_WRONLY | os.O_RDWR) + if access != os.O_WRONLY { + reader, err := session.OpenReader() + if err != nil { + return 0, errnoFor(err) + } + handle.read = reader + } + if access == os.O_WRONLY || access == os.O_RDWR { + writer, err := session.OpenWriter() + if err != nil { + if handle.read != nil { + _ = handle.read.Close() + } + return 0, errnoFor(err) + } + handle.write = writer + if flags&os.O_TRUNC != 0 { + if err := writer.Truncate(context.Background(), 0); err != nil { + _ = writer.Close() + if handle.read != nil { + _ = handle.read.Close() + } + return 0, errnoFor(err) + } + } + } + f.mu.Lock() + handleID := f.next + f.next++ + f.handles[handleID] = handle + f.mu.Unlock() + releaseOwner = false + return handleID, 0 +} + +func (f *Filesystem) Read(handleID uint64, destination []byte, offset int64) (int, syscall.Errno) { + endIO := f.beginIO() + defer endIO() + handle, errno := f.handle(handleID) + if errno != 0 { + return 0, syscall.EBADF + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.native != nil { + n, err := handle.nativeAppend.ReadAt(handle.native, destination, offset) + if err != nil && !errors.Is(err, io.EOF) { + return n, errnoFor(err) + } + return n, 0 + } + if handle.read == nil { + return 0, syscall.EBADF + } + n, err := handle.read.ReadAt(context.Background(), destination, offset) + if err != nil && !errors.Is(err, io.EOF) { + return n, errnoFor(err) + } + return n, 0 +} + +// StreamNativeRead exposes only a stable, ordinary native file range to the +// Darwin FSKit transport. Virtual sessions and files with a pending append +// deliberately return handled=false so the caller can use the normal buffered +// path. +func (f *Filesystem) StreamNativeRead(handleID uint64, offset int64, length int, stream func(*os.File, int64, int) (int, error)) (handled bool, n int, err error) { + if offset < 0 || length < 0 || stream == nil { + return true, 0, errors.New("invalid native stream read") + } + handle, errno := f.handle(handleID) + if errno != 0 { + return true, 0, errno + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.native == nil || handle.nativeAppend == nil { + return false, 0, nil + } + info, statErr := handle.native.Stat() + if statErr != nil { + return true, 0, statErr + } + if !info.Mode().IsRegular() { + return false, 0, nil + } + if err := handle.nativeAppend.RefreshIfIdle(); err != nil { + return true, 0, err + } + n, err = handle.nativeAppend.StreamRead(handle.native, offset, length, stream) + if errors.Is(err, errNativeAppendPending) || errors.Is(err, errNativeReadStale) { + return false, 0, nil + } + return true, n, err +} + +// StreamBufferedRead keeps the file handle stable while emitting a bounded +// sequence of chunks. The callback receives the complete response length on +// every call so transports can write their framing before the first chunk. +func (f *Filesystem) StreamBufferedRead(handleID uint64, offset int64, length int, chunkBytes int, stream func(total int, chunk []byte) error) (n int, err error) { + if offset < 0 || length < 0 || chunkBytes <= 0 || stream == nil { + return 0, errors.New("invalid buffered stream read") + } + endIO := f.beginIO() + defer endIO() + handle, errno := f.handle(handleID) + if errno != 0 { + return 0, errno + } + handle.mu.Lock() + defer handle.mu.Unlock() + + var size int64 + if handle.native != nil { + if handle.nativeAppend == nil { + return 0, syscall.EBADF + } + if err := handle.nativeAppend.RefreshIfIdle(); err != nil { + return 0, err + } + size = handle.nativeAppend.VisibleSize() + } else { + if handle.read == nil { + return 0, syscall.EBADF + } + size = handle.read.Size() + } + + total := length + if offset >= size { + total = 0 + } else if remaining := size - offset; int64(total) > remaining { + total = int(remaining) + } + if total == 0 { + return 0, stream(0, nil) + } + if chunkBytes > total { + chunkBytes = total + } + buffer := make([]byte, chunkBytes) + for n < total { + need := min(len(buffer), total-n) + var count int + var readErr error + if handle.native != nil { + count, readErr = handle.nativeAppend.ReadAt(handle.native, buffer[:need], offset+int64(n)) + } else { + count, readErr = handle.read.ReadAt(context.Background(), buffer[:need], offset+int64(n)) + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + return n, readErr + } + if count <= 0 || count > need { + return n, io.ErrUnexpectedEOF + } + if err := stream(total, buffer[:count]); err != nil { + return n, err + } + n += count + if count != need { + return n, io.ErrUnexpectedEOF + } + } + return n, nil +} + +// WithNativeReadFD keeps the handle and append-state locks held while the +// caller transfers a read-only native descriptor to a cooperating transport. +// The callback must not retain the *os.File; the receiver owns the duplicated +// descriptor created by the OS descriptor-transfer operation. +func (f *Filesystem) WithNativeReadFD(handleID uint64, send func(*os.File) error) (handled bool, err error) { + if send == nil { + return true, errors.New("native descriptor callback is required") + } + handle, errno := f.handle(handleID) + if errno != 0 { + return true, errno + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.native == nil || handle.nativeAppend == nil { + return false, nil + } + if info, statErr := handle.native.Stat(); statErr != nil { + return true, statErr + } else if !info.Mode().IsRegular() { + return false, nil + } + if err := handle.nativeAppend.RefreshIfIdle(); err != nil { + return true, err + } + if _, err := handle.nativeAppend.StreamRead(handle.native, 0, 0, func(file *os.File, _ int64, _ int) (int, error) { + return 0, send(file) + }); errors.Is(err, errNativeAppendPending) || errors.Is(err, errNativeReadStale) { + return false, nil + } else { + return true, err + } +} + +func (f *Filesystem) Write(handleID uint64, data []byte, offset int64) (int, syscall.Errno) { + endIO := f.beginIO() + defer endIO() + handle, errno := f.handle(handleID) + if errno != 0 { + return 0, syscall.EBADF + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.native != nil { + var n int + var err error + if handle.append { + n, err = handle.nativeAppend.Stage(data, offset) + } else { + f.markNativeInternalMutation(handle.nativePath) + n, err = handle.nativeAppend.WriteAt(handle.native, data, offset) + } + return n, errnoFor(err) + } + if handle.write == nil { + return 0, syscall.EBADF + } + var n int + var err error + if handle.append { + n, err = handle.write.Append(context.Background(), data) + } else { + info, infoErr := handle.session.VisibleInfo() + if infoErr != nil { + return 0, errnoFor(infoErr) + } + if offset == info.Size { + n, err = handle.write.Append(context.Background(), data) + if err == nil { + if !handle.appendStream { + handle.appendFloor = offset + } + handle.appendStream = true + handle.appendOffset = offset + int64(n) + } + } else if handle.appendStream && + offset >= handle.appendFloor && offset < handle.appendOffset && + info.Size == handle.appendOffset && completeJSONL(data) { + n, err = handle.write.Append(context.Background(), data) + if err == nil { + handle.appendOffset += int64(n) + } + } else { + handle.appendStream = false + n, err = handle.write.WriteAt(context.Background(), data, offset) + } + } + if err != nil { + return n, errnoFor(err) + } + if handle.read != nil { + if errno := refreshReader(handle); errno != 0 { + return n, errno + } + } + return n, 0 +} + +func (f *Filesystem) UseRandomWrites(handleID uint64) syscall.Errno { + handle, errno := f.handle(handleID) + if errno != 0 { + return syscall.EBADF + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.native != nil && handle.append { + f.markNativeInternalMutation(handle.nativePath) + if err := handle.nativeAppend.Commit(); err != nil { + return errnoFor(err) + } + } + handle.append = false + handle.appendStream = false + return 0 +} + +func (f *Filesystem) Truncate(handleID uint64, size int64) syscall.Errno { + handle, errno := f.handle(handleID) + if errno != 0 { + return syscall.EBADF + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.native != nil { + f.markNativeInternalMutation(handle.nativePath) + return errnoFor(handle.nativeAppend.Truncate(size)) + } + if handle.write == nil { + return syscall.EBADF + } + if err := handle.write.Truncate(context.Background(), size); err != nil { + return errnoFor(err) + } + if handle.appendStream && size != handle.appendOffset { + handle.appendStream = false + } + if handle.read != nil { + return refreshReader(handle) + } + return 0 +} + +func (f *Filesystem) TruncatePath(name string, size int64) syscall.Errno { + session, owner, errno := f.acquireSessionForPath(name) + if errno != 0 { + if f.canonical { + if nativePath, ok := f.nativePath(cleanPath(name)); ok { + state, err := f.loadNativeAppendState(nativePath) + if err != nil { + return errnoFor(err) + } + f.markNativeInternalMutation(nativePath) + return errnoFor(state.Truncate(size)) + } + } + return errno + } + defer owner.release() + if handle := f.lockActiveWriter(session); handle != nil { + defer handle.mu.Unlock() + if err := handle.write.Truncate(context.Background(), size); err != nil { + return errnoFor(err) + } + if handle.appendStream && size != handle.appendOffset { + handle.appendStream = false + } + if handle.read != nil { + return refreshReader(handle) + } + return 0 + } + writer, err := session.OpenWriter() + if err != nil { + return errnoFor(err) + } + truncateErr := writer.Truncate(context.Background(), size) + closeErr := writer.Close() + if truncateErr != nil { + return errnoFor(truncateErr) + } + return errnoFor(closeErr) +} + +func completeJSONL(data []byte) bool { + if len(data) == 0 || data[len(data)-1] != '\n' || !utf8.Valid(data) { + return false + } + for len(data) > 0 { + end := bytes.IndexByte(data, '\n') + if end <= 0 || !json.Valid(data[:end]) { + return false + } + data = data[end+1:] + } + return true +} + +func (f *Filesystem) lockActiveWriter(session *vfs.Session) *fileHandle { + f.mu.RLock() + defer f.mu.RUnlock() + for _, handle := range f.handles { + if handle.session == session && handle.write != nil { + handle.mu.Lock() + return handle + } + } + return nil +} + +func (f *Filesystem) Fsync(handleID uint64) syscall.Errno { + handle, errno := f.handle(handleID) + if errno != 0 { + return errno + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.native != nil { + if handle.append { + f.markNativeInternalMutation(handle.nativePath) + return errnoFor(handle.nativeAppend.CommitAvailable()) + } + return errnoFor(handle.native.Sync()) + } + if handle.write == nil { + return 0 + } + return errnoFor(handle.write.Sync()) +} + +func (f *Filesystem) Flush(handleID uint64) syscall.Errno { + handle, errno := f.handle(handleID) + if errno != 0 { + return errno + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.native != nil && handle.append { + f.markNativeInternalMutation(handle.nativePath) + if f.nativeAppendIsLastWriter(handleID, handle.nativeAppend) { + return errnoFor(handle.nativeAppend.Commit()) + } + return errnoFor(handle.nativeAppend.CommitAvailable()) + } + return 0 +} + +func (f *Filesystem) Release(handleID uint64) syscall.Errno { + f.mu.Lock() + handle, ok := f.handles[handleID] + lastNativeWriter := false + if ok { + lastNativeWriter = f.nativeAppendIsLastWriterLocked(handleID, handle.nativeAppend) + delete(f.handles, handleID) + } + f.mu.Unlock() + if !ok { + return syscall.EBADF + } + handle.mu.Lock() + defer handle.mu.Unlock() + if handle.native != nil { + var commitErr error + if handle.append { + f.markNativeInternalMutation(handle.nativePath) + if lastNativeWriter { + commitErr = handle.nativeAppend.Commit() + } else { + commitErr = handle.nativeAppend.CommitAvailable() + } + } + return errnoFor(errors.Join(commitErr, handle.native.Close())) + } + var result error + if handle.read != nil { + if err := handle.read.Close(); err != nil { + result = errors.Join(result, err) + } + } + if handle.write != nil { + if err := handle.write.Close(); err != nil { + result = errors.Join(result, err) + } + } + result = errors.Join(result, handle.owner.release()) + return errnoFor(result) +} + +func (f *Filesystem) nativeAppendIsLastWriter(handleID uint64, state *nativeAppendState) bool { + f.mu.RLock() + defer f.mu.RUnlock() + return f.nativeAppendIsLastWriterLocked(handleID, state) +} + +func (f *Filesystem) nativeAppendIsLastWriterLocked(handleID uint64, state *nativeAppendState) bool { + if state == nil { + return false + } + for currentID, current := range f.handles { + if currentID != handleID && current.nativeAppend == state && current.append { + return false + } + } + return true +} + +func refreshReader(handle *fileHandle) syscall.Errno { + reader, err := handle.session.OpenReader() + if err != nil { + return errnoFor(err) + } + old := handle.read + handle.read = reader + if old != nil { + if err := old.Close(); err != nil { + return errnoFor(err) + } + } + return 0 +} + +func (f *Filesystem) Mkdir(name string, _ uint32) syscall.Errno { + cleaned := cleanPath(name) + if !f.canonical || cleaned == "" || cleaned == "/" || !canonicalNamespacePath(cleaned) { + return syscall.EPERM + } + f.mu.Lock() + if _, exists := f.directories[cleaned]; exists { + f.mu.Unlock() + return syscall.EEXIST + } + if _, exists := f.paths[cleaned]; exists { + f.mu.Unlock() + return syscall.EEXIST + } + root := f.nativeRoot + _, virtualParent := f.directories[path.Dir(cleaned)] + f.mu.Unlock() + if !virtualParent && root == "" { + return syscall.ENOENT + } + if root != "" { + if err := os.Mkdir(nativePathFromRoot(root, cleaned), 0o700); err != nil { + return errnoFor(err) + } + } + f.mu.Lock() + defer f.mu.Unlock() + f.directories[cleaned] = struct{}{} + f.ensureDirectoryStateLocked(cleaned, time.Now()) + f.bumpDirectoryGenerationLocked(path.Dir(cleaned), time.Now()) + f.bumpNamespaceVersion() + return 0 +} + +func (f *Filesystem) Rename(oldName string, newName string) syscall.Errno { + if !f.canonical { + return syscall.EPERM + } + oldPath, newPath := cleanPath(oldName), cleanPath(newName) + openUnlinkRename := canonicalSessionPath(oldPath) && fskitOpenUnlinkPath(newPath) && path.Dir(oldPath) == path.Dir(newPath) + if !canonicalSessionPath(oldPath) || (!canonicalSessionPath(newPath) && !openUnlinkRename) { + return syscall.EPERM + } + f.mu.Lock() + sessionID, exists := f.paths[oldPath] + if !exists { + if _, hidden := f.retained[oldPath]; hidden { + f.mu.Unlock() + return syscall.ENOENT + } + root := f.nativeRoot + oldNative := nativePathFromRoot(root, oldPath) + newNative := nativePathFromRoot(root, newPath) + appendState := f.nativeAppends[filepath.Clean(oldNative)] + f.mu.Unlock() + if root == "" { + return syscall.ENOENT + } + var renameErr error + if appendState != nil { + renameErr = appendState.Relocate(newNative) + } else { + renameErr = os.Rename(oldNative, newNative) + } + if renameErr != nil { + return errnoFor(renameErr) + } + f.mu.Lock() + delete(f.nativeAppends, filepath.Clean(oldNative)) + if appendState != nil { + f.nativeAppends[filepath.Clean(newNative)] = appendState + } else { + delete(f.nativeAppends, filepath.Clean(newNative)) + } + for _, handle := range f.handles { + if handle.path == oldPath { + handle.path = newPath + handle.nativePath = newNative + } + } + f.mu.Unlock() + f.bumpNamespaceVersion() + return 0 + } + defer f.mu.Unlock() + if _, exists := f.directories[path.Dir(newPath)]; !exists { + root := f.nativeRoot + info, err := os.Stat(nativePathFromRoot(root, path.Dir(newPath))) + if root == "" || err != nil || !info.IsDir() { + return syscall.ENOENT + } + } + if _, exists := f.paths[newPath]; exists { + return syscall.EEXIST + } + if err := moveManagedXattrCarrier(f.nativeRoot, oldPath, newPath); err != nil { + return errnoFor(err) + } + delete(f.paths, oldPath) + f.paths[newPath] = sessionID + now := time.Now() + f.bumpDirectoryGenerationLocked(path.Dir(oldPath), now) + if path.Dir(newPath) != path.Dir(oldPath) { + f.bumpDirectoryGenerationLocked(path.Dir(newPath), now) + } + for _, handle := range f.handles { + if handle.path == oldPath { + handle.path = newPath + } + } + f.bumpNamespaceVersion() + return 0 +} + +func (f *Filesystem) Unlink(name string) syscall.Errno { + if !f.canonical { + return syscall.EPERM + } + cleaned := cleanPath(name) + f.mu.RLock() + _, managed := f.paths[cleaned] + busy := f.nativePathBusyLocked(cleaned) + f.mu.RUnlock() + if managed { + if fskitOpenUnlinkPath(cleaned) { + f.mu.Lock() + delete(f.paths, cleaned) + delete(f.retained, cleaned) + f.bumpDirectoryGenerationLocked(path.Dir(cleaned), time.Now()) + f.mu.Unlock() + f.bumpNamespaceVersion() + return 0 + } + return syscall.EPERM + } + if busy { + return syscall.EBUSY + } + nativePath, ok := f.nativePath(cleaned) + if !ok { + return syscall.ENOENT + } + err := os.Remove(nativePath) + if err == nil { + f.mu.Lock() + delete(f.nativeAppends, filepath.Clean(nativePath)) + f.mu.Unlock() + f.bumpNamespaceVersion() + } + return errnoFor(err) +} + +func (f *Filesystem) Rmdir(name string) syscall.Errno { + if !f.canonical { + return syscall.EPERM + } + cleaned := cleanPath(name) + if cleaned == "/" || cleaned == "/sessions" || cleaned == "/archived_sessions" || !canonicalNamespacePath(cleaned) { + return syscall.EPERM + } + f.mu.RLock() + for route := range f.paths { + if path.Dir(route) == cleaned || strings.HasPrefix(route, cleaned+"/") { + f.mu.RUnlock() + return syscall.ENOTEMPTY + } + } + root := f.nativeRoot + f.mu.RUnlock() + if root == "" { + return syscall.ENOENT + } + if err := os.Remove(nativePathFromRoot(root, cleaned)); err != nil { + return errnoFor(err) + } + f.mu.Lock() + delete(f.directories, cleaned) + delete(f.directoryStates, cleaned) + f.bumpDirectoryGenerationLocked(path.Dir(cleaned), time.Now()) + f.mu.Unlock() + f.bumpNamespaceVersion() + return 0 +} + +func (f *Filesystem) SyncAll() syscall.Errno { + f.mu.RLock() + handles := make([]uint64, 0, len(f.handles)) + for handleID := range f.handles { + handles = append(handles, handleID) + } + f.mu.RUnlock() + for _, handleID := range handles { + if errno := f.Fsync(handleID); errno != 0 && errno != syscall.EBADF { + return errno + } + } + return 0 +} + +func (f *Filesystem) loadNativeAppendState(nativePath string) (*nativeAppendState, error) { + cleaned := filepath.Clean(nativePath) + f.mu.RLock() + state := f.nativeAppends[cleaned] + journalRoot := f.nativeJournalRoot + f.mu.RUnlock() + if state != nil { + if err := state.RefreshIfIdle(); err != nil { + return nil, err + } + return state, nil + } + created, err := newNativeAppendState(cleaned, journalRoot) + if err != nil { + return nil, err + } + f.mu.Lock() + if state = f.nativeAppends[cleaned]; state == nil { + f.nativeAppends[cleaned] = created + state = created + } + f.mu.Unlock() + if state != created { + if err := state.RefreshIfIdle(); err != nil { + return nil, err + } + } + return state, nil +} + +func (f *Filesystem) nativeAppendState(nativePath string) *nativeAppendState { + f.mu.RLock() + state := f.nativeAppends[filepath.Clean(nativePath)] + f.mu.RUnlock() + return state +} + +func (f *Filesystem) nativePathBusyLocked(name string) bool { + nativePath := filepath.Clean(nativePathFromRoot(f.nativeRoot, name)) + if state := f.nativeAppends[nativePath]; state != nil && state.HasPending() { + return true + } + for _, handle := range f.handles { + if filepath.Clean(handle.nativePath) == nativePath { + return true + } + } + return false +} + +func (f *Filesystem) sessionForPath(name string) (*vfs.Session, *sessionOwner, syscall.Errno) { + cleaned := cleanPath(name) + if f.canonical { + f.mu.RLock() + sessionID := f.paths[cleaned] + session := f.sessions[sessionID] + owner := f.owners[sessionID] + _, nativeFirst := f.nativeFirst[sessionID] + root := f.nativeRoot + _, retained := f.retained[cleaned] + f.mu.RUnlock() + if session == nil { + return nil, nil, syscall.ENOENT + } + if nativeFirst && root != "" && !retained { + if info, err := os.Stat(nativePathFromRoot(root, cleaned)); err == nil && !info.IsDir() { + return nil, nil, syscall.ENOENT + } + } + return session, owner, 0 + } + if cleaned == "/" || strings.Count(cleaned, "/") != 1 || !strings.HasSuffix(cleaned, ".jsonl") { + return nil, nil, syscall.ENOENT + } + sessionID := strings.TrimSuffix(strings.TrimPrefix(cleaned, "/"), ".jsonl") + f.mu.RLock() + session := f.sessions[sessionID] + owner := f.owners[sessionID] + loader := f.loader + f.mu.RUnlock() + if session != nil { + return session, owner, 0 + } + if loader == nil { + return nil, nil, syscall.ENOENT + } + f.loadMu.Lock() + defer f.loadMu.Unlock() + f.mu.RLock() + session = f.sessions[sessionID] + owner = f.owners[sessionID] + loader = f.loader + f.mu.RUnlock() + if session != nil { + return session, owner, 0 + } + if loader == nil { + return nil, nil, syscall.ENOENT + } + loaded, closer, err := loader(sessionID) + if err != nil { + return nil, nil, errnoFor(err) + } + if loaded == nil { + _ = newSessionOwner(closer).retire() + return nil, nil, syscall.EIO + } + loadedOwner := newSessionOwner(closer) + f.mu.Lock() + if session = f.sessions[sessionID]; session == nil { + f.sessions[sessionID] = loaded + f.owners[sessionID] = loadedOwner + session = loaded + owner = loadedOwner + loadedOwner = nil + } else { + owner = f.owners[sessionID] + } + f.mu.Unlock() + if err := loadedOwner.retire(); err != nil { + return nil, nil, errnoFor(err) + } + return session, owner, 0 +} + +func (f *Filesystem) acquireSessionForPath(name string) (*vfs.Session, *sessionOwner, syscall.Errno) { + for attempt := 0; attempt < 4; attempt++ { + session, owner, errno := f.sessionForPath(name) + if errno != 0 { + return nil, nil, errno + } + if owner != nil && owner.acquire() { + return session, owner, 0 + } + } + return nil, nil, syscall.EAGAIN +} + +func safeSessionID(sessionID string) bool { + return sessionID != "" && !strings.ContainsAny(sessionID, "/\\\x00") +} + +func canonicalSessionPath(name string) bool { + if name == "" || !strings.HasSuffix(name, ".jsonl") { + return false + } + return strings.HasPrefix(name, "/sessions/") || strings.HasPrefix(name, "/archived_sessions/") +} + +func fskitOpenUnlinkPath(name string) bool { + base := path.Base(name) + return canonicalNamespacePath(name) && strings.HasPrefix(base, ".nfs.") && len(base) > len(".nfs.") +} + +func nativeTransactionPath(name string) bool { + return canonicalSessionPath(name) && !strings.HasPrefix(path.Base(name), "._") +} + +func canonicalNamespacePath(name string) bool { + return name == "/" || name == "/sessions" || name == "/archived_sessions" || + strings.HasPrefix(name, "/sessions/") || strings.HasPrefix(name, "/archived_sessions/") +} + +func moveAppleDoubleSidecar(root string, oldPath string, newPath string) error { + if root == "" || oldPath == "" || newPath == "" || !canonicalSessionPath(oldPath) || !canonicalSessionPath(newPath) { + return nil + } + oldSidecar := filepath.Join(nativePathFromRoot(root, path.Dir(oldPath)), "._"+path.Base(oldPath)) + newSidecar := filepath.Join(nativePathFromRoot(root, path.Dir(newPath)), "._"+path.Base(newPath)) + if _, err := os.Lstat(oldSidecar); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return err + } + return os.Rename(oldSidecar, newSidecar) +} + +func managedXattrCarrier(root string, name string) string { + digest := sha256.Sum256([]byte(cleanPath(name))) + return filepath.Join(root, ".codexfold-xattrs", hex.EncodeToString(digest[:])) +} + +func moveManagedMetadata(root string, oldPath string, newPath string) error { + if err := moveAppleDoubleSidecar(root, oldPath, newPath); err != nil { + return err + } + return moveManagedXattrCarrier(root, oldPath, newPath) +} + +func moveManagedXattrCarrier(root string, oldPath string, newPath string) error { + if root == "" || oldPath == "" || newPath == "" || !canonicalSessionPath(oldPath) || !canonicalSessionPath(newPath) { + return nil + } + oldCarrier := managedXattrCarrier(root, oldPath) + newCarrier := managedXattrCarrier(root, newPath) + if _, err := os.Lstat(oldCarrier); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return err + } + return os.Rename(oldCarrier, newCarrier) +} + +func (f *Filesystem) ensureDirectoryChainLocked(directory string) { + chain := make([]string, 0, strings.Count(directory, "/")) + for directory != "." && directory != "/" && directory != "" { + chain = append(chain, directory) + directory = path.Dir(directory) + } + f.directories["/"] = struct{}{} + f.ensureDirectoryStateLocked("/", time.Now()) + now := time.Now() + for index := len(chain) - 1; index >= 0; index-- { + current := chain[index] + if _, exists := f.directories[current]; exists { + f.ensureDirectoryStateLocked(current, now) + continue + } + f.directories[current] = struct{}{} + f.ensureDirectoryStateLocked(current, now) + f.bumpDirectoryGenerationLocked(path.Dir(current), now) + } +} + +func (f *Filesystem) handle(handleID uint64) (*fileHandle, syscall.Errno) { + f.mu.RLock() + handle := f.handles[handleID] + f.mu.RUnlock() + if handle == nil { + return nil, syscall.EBADF + } + return handle, 0 +} + +func (f *Filesystem) HandlePath(handleID uint64) (string, syscall.Errno) { + f.mu.RLock() + handle := f.handles[handleID] + if handle == nil { + f.mu.RUnlock() + return "", syscall.EBADF + } + name := handle.path + f.mu.RUnlock() + return name, 0 +} + +func (f *Filesystem) nativePath(name string) (string, bool) { + f.mu.RLock() + root := f.nativeRoot + _, retained := f.retained[name] + f.mu.RUnlock() + if !f.canonical || root == "" || retained || name == "" || name == "/" || !canonicalNamespacePath(name) { + return "", false + } + return nativePathFromRoot(root, name), true +} + +func (f *Filesystem) nativeMetadataPath(name string) (string, bool) { + f.mu.RLock() + root := f.nativeRoot + f.mu.RUnlock() + if !f.canonical || root == "" || !canonicalNamespacePath(name) { + return "", false + } + if name == "/" { + return root, true + } + return nativePathFromRoot(root, name), true +} + +func (f *Filesystem) metadataPath(name string) (string, bool, syscall.Errno) { + cleaned := cleanPath(name) + if session, _, errno := f.sessionForPath(cleaned); errno == 0 { + return session.MetadataPath(), true, 0 + } + if nativePath, ok := f.nativeMetadataPath(cleaned); ok { + if _, err := os.Lstat(nativePath); err != nil { + return "", false, errnoFor(err) + } + return nativePath, false, 0 + } + return "", false, syscall.ENOENT +} + +func (f *Filesystem) xattrPath(name string, create bool) (string, bool, syscall.Errno) { + cleaned := cleanPath(name) + if _, _, errno := f.sessionForPath(cleaned); errno == 0 { + f.mu.RLock() + root := f.nativeRoot + f.mu.RUnlock() + if root == "" { + return "", true, syscall.ENOTSUP + } + carrier := managedXattrCarrier(root, cleaned) + if create { + if err := os.MkdirAll(filepath.Dir(carrier), 0o700); err != nil { + return "", true, errnoFor(err) + } + file, err := os.OpenFile(carrier, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return "", true, errnoFor(err) + } + if err := file.Close(); err != nil { + return "", true, errnoFor(err) + } + } + return carrier, true, 0 + } + if nativePath, ok := f.nativeMetadataPath(cleaned); ok { + if _, err := os.Lstat(nativePath); err != nil { + return "", false, errnoFor(err) + } + return nativePath, false, 0 + } + return "", false, syscall.ENOENT +} + +func (f *Filesystem) registerRetainedPathLocked(sessionID string, session *vfs.Session) { + for retained, currentID := range f.retained { + if currentID == sessionID { + delete(f.retained, retained) + } + } + if f.nativeRoot == "" || session == nil { + return + } + snapshot := filepath.Clean(session.State().NativeSnapshot.Path) + relative, err := filepath.Rel(f.nativeRoot, snapshot) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return + } + retained := cleanPath(filepath.ToSlash(relative)) + if canonicalSessionPath(retained) { + f.retained[retained] = sessionID + } +} + +func nativePathFromRoot(root string, name string) string { + return filepath.Join(root, filepath.FromSlash(strings.TrimPrefix(name, "/"))) +} + +func cleanPath(name string) string { + if name == "" || strings.ContainsRune(name, '\x00') || strings.Contains(name, "..") { + return "" + } + return path.Clean("/" + strings.TrimPrefix(name, "/")) +} + +func errnoFor(err error) syscall.Errno { + if err == nil { + return 0 + } + var errno syscall.Errno + if errors.As(err, &errno) { + return errno + } + switch { + case errors.Is(err, vfs.ErrWriterBusy): + return syscall.EBUSY + case errors.Is(err, errNativeAppendPending): + return syscall.EBUSY + case errors.Is(err, os.ErrNotExist): + return syscall.ENOENT + case errors.Is(err, os.ErrPermission): + return syscall.EACCES + case errors.Is(err, context.Canceled): + return syscall.EINTR + default: + return syscall.EIO + } +} diff --git a/internal/mountfs/filesystem_test.go b/internal/mountfs/filesystem_test.go new file mode 100644 index 0000000..c0344ad --- /dev/null +++ b/internal/mountfs/filesystem_test.go @@ -0,0 +1,1350 @@ +package mountfs + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + "testing" + "time" + + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/vfs" +) + +func TestFilesystemIOIdleRequiresNoActiveIOAndCompletedIdleWindow(t *testing.T) { + filesystem := New() + filesystem.lastIO.Store(time.Now().Add(-time.Second).UnixNano()) + if !filesystem.IOIdleFor(500 * time.Millisecond) { + t.Fatal("filesystem did not report an elapsed idle window") + } + endIO := filesystem.beginIO() + if filesystem.IOIdleFor(0) { + t.Fatal("filesystem reported idle while I/O was active") + } + endIO() + if filesystem.IOIdleFor(time.Second) { + t.Fatal("filesystem reported idle immediately after I/O") + } + if !filesystem.IOIdleFor(0) { + t.Fatal("filesystem did not report idle with a zero window") + } +} + +func TestFilesystemListsStatsReadsAndAppendsSession(t *testing.T) { + filesystem, source := mountFixture(t) + entries, errno := filesystem.ReadDir("/") + if errno != 0 || len(entries) != 1 || entries[0] != "session.jsonl" { + t.Fatalf("ReadDir = %#v errno=%v", entries, errno) + } + attribute, errno := filesystem.Getattr("/session.jsonl") + if errno != 0 || attribute.Mode&syscall.S_IFREG == 0 || attribute.Size != int64(len(source)) { + t.Fatalf("Getattr = %#v errno=%v", attribute, errno) + } + + readHandle, errno := filesystem.Open("/session.jsonl", os.O_RDONLY) + if errno != 0 { + t.Fatalf("Open read errno=%v", errno) + } + buffer := make([]byte, len(source)) + n, errno := filesystem.Read(readHandle, buffer, 0) + if errno != 0 || !bytes.Equal(buffer[:n], source) { + t.Fatalf("Read = %d errno=%v bytes=%q", n, errno, buffer[:n]) + } + if errno := filesystem.Release(readHandle); errno != 0 { + t.Fatalf("Release read errno=%v", errno) + } + + writeHandle, errno := filesystem.Open("/session.jsonl", os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("Open append errno=%v", errno) + } + if n, errno := filesystem.Write(writeHandle, []byte("-tail"), 0); errno != 0 || n != 5 { + t.Fatalf("Write append = %d errno=%v", n, errno) + } + if errno := filesystem.Fsync(writeHandle); errno != 0 { + t.Fatalf("Fsync errno=%v", errno) + } + _ = filesystem.Release(writeHandle) + + newHandle, errno := filesystem.Open("/session.jsonl", os.O_RDONLY) + if errno != 0 { + t.Fatalf("Open current errno=%v", errno) + } + current := make([]byte, len(source)+5) + n, errno = filesystem.Read(newHandle, current, 0) + if errno != 0 || !bytes.Equal(current[:n], append(append([]byte(nil), source...), []byte("-tail")...)) { + t.Fatalf("current read differs: n=%d errno=%v bytes=%q", n, errno, current[:n]) + } + _ = filesystem.Release(newHandle) +} + +func TestFilesystemRandomWriteTruncateAndWriterExclusion(t *testing.T) { + filesystem, source := mountFixture(t) + first, errno := filesystem.Open("/session.jsonl", os.O_RDWR) + if errno != 0 { + t.Fatalf("Open first writer errno=%v", errno) + } + if _, errno := filesystem.Open("/session.jsonl", os.O_WRONLY); errno != syscall.EBUSY { + t.Fatalf("second writer errno=%v, want EBUSY", errno) + } + if n, errno := filesystem.Write(first, []byte("PATCH"), 2); errno != 0 || n != 5 { + t.Fatalf("random Write = %d errno=%v", n, errno) + } + current := make([]byte, len(source)) + if n, errno := filesystem.Read(first, current, 0); errno != 0 || n != len(source) || string(current[2:7]) != "PATCH" { + t.Fatalf("read-after-write = %d errno=%v bytes=%q", n, errno, current) + } + if errno := filesystem.Truncate(first, int64(len(source)-3)); errno != 0 { + t.Fatalf("Truncate errno=%v", errno) + } + _ = filesystem.Release(first) + attribute, errno := filesystem.Getattr("/session.jsonl") + if errno != 0 || attribute.Size != int64(len(source)-3) { + t.Fatalf("truncated attribute=%#v errno=%v", attribute, errno) + } +} + +func TestFilesystemWriteAtVisibleEOFUsesDeltaWithoutCopyOnWrite(t *testing.T) { + source := []byte("first\nsecond\nthird\n") + session := mountSessionFixture(t, "session", source) + filesystem := New() + if err := filesystem.AddSession("session", session); err != nil { + t.Fatal(err) + } + handle, errno := filesystem.Open("/session.jsonl", os.O_RDWR) + if errno != 0 { + t.Fatalf("Open writer errno=%v", errno) + } + t.Cleanup(func() { _ = filesystem.Release(handle) }) + tail := []byte("tail\n") + if n, errno := filesystem.Write(handle, tail, int64(len(source))); errno != 0 || n != len(tail) { + t.Fatalf("Write at EOF = %d errno=%v", n, errno) + } + if state := session.State(); state.BackingPath != "" { + t.Fatalf("EOF write created copy-on-write backing %q", state.BackingPath) + } else if info, err := os.Stat(state.DeltaPath); err != nil || info.Size() != int64(len(tail)) { + t.Fatalf("delta after EOF write: info=%#v err=%v", info, err) + } + current := make([]byte, len(source)+len(tail)) + if n, errno := filesystem.Read(handle, current, 0); errno != 0 || n != len(current) { + t.Fatalf("Read after EOF write = %d errno=%v", n, errno) + } + want := append(append([]byte(nil), source...), tail...) + if !bytes.Equal(current, want) { + t.Fatalf("visible bytes differ: got=%q want=%q", current, want) + } +} + +func TestFilesystemStaleTailOffsetAppendsCompleteJSONLRecord(t *testing.T) { + source := []byte("{\"record\":0}\n") + session := mountSessionFixture(t, "session", source) + filesystem := New() + if err := filesystem.AddSession("session", session); err != nil { + t.Fatal(err) + } + handle, errno := filesystem.Open("/session.jsonl", os.O_RDWR) + if errno != 0 { + t.Fatalf("Open writer errno=%v", errno) + } + t.Cleanup(func() { _ = filesystem.Release(handle) }) + first := []byte("{\"record\":1}\n") + second := []byte("{\"record\":2}\n") + staleEOF := int64(len(source)) + if n, errno := filesystem.Write(handle, first, staleEOF); errno != 0 || n != len(first) { + t.Fatalf("first append = %d errno=%v", n, errno) + } + if n, errno := filesystem.Write(handle, second, staleEOF); errno != 0 || n != len(second) { + t.Fatalf("stale-offset append = %d errno=%v", n, errno) + } + if state := session.State(); state.BackingPath != "" { + t.Fatalf("stale JSONL tail offset created copy-on-write backing %q", state.BackingPath) + } + want := append(append(append([]byte(nil), source...), first...), second...) + current := make([]byte, len(want)) + if n, errno := filesystem.Read(handle, current, 0); errno != 0 || n != len(want) { + t.Fatalf("Read after stale-offset append = %d errno=%v", n, errno) + } + if !bytes.Equal(current, want) { + t.Fatalf("visible bytes differ: got=%q want=%q", current, want) + } +} + +func TestFilesystemStaleTailOffsetWithArbitraryBytesUsesCopyOnWrite(t *testing.T) { + source := []byte("{\"record\":0}\n") + session := mountSessionFixture(t, "session", source) + filesystem := New() + if err := filesystem.AddSession("session", session); err != nil { + t.Fatal(err) + } + handle, errno := filesystem.Open("/session.jsonl", os.O_RDWR) + if errno != 0 { + t.Fatalf("Open writer errno=%v", errno) + } + t.Cleanup(func() { _ = filesystem.Release(handle) }) + first := []byte("{\"record\":1}\n") + staleEOF := int64(len(source)) + if n, errno := filesystem.Write(handle, first, staleEOF); errno != 0 || n != len(first) { + t.Fatalf("first append = %d errno=%v", n, errno) + } + if n, errno := filesystem.Write(handle, []byte("PATCH"), staleEOF); errno != 0 || n != len("PATCH") { + t.Fatalf("random write = %d errno=%v", n, errno) + } + if state := session.State(); state.BackingPath == "" { + t.Fatal("arbitrary stale-offset write did not create copy-on-write backing") + } +} + +func TestNativePassthroughPreservesOutOfOrderAppendChunksByOffset(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := "/sessions/2026/07/16/rollout-repro.jsonl" + nativePath := nativePathFromRoot(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + large := append([]byte("{\"large\":\""), bytes.Repeat([]byte("x"), 40*1024)...) + large = append(large, []byte("\"}\n")...) + small := []byte("{\"record\":2}\n") + split := 32 * 1024 + + largeHandle, errno := filesystem.Open(route, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open large writer: %v", errno) + } + defer filesystem.Release(largeHandle) + smallHandle, errno := filesystem.Open(route, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open small writer: %v", errno) + } + defer filesystem.Release(smallHandle) + + baseOffset := int64(len(base)) + if n, errno := filesystem.Write(largeHandle, large[:split], baseOffset); errno != 0 || n != split { + t.Fatalf("write large prefix: n=%d errno=%v", n, errno) + } + if n, errno := filesystem.Write(smallHandle, small, baseOffset+int64(len(large))); errno != 0 || n != len(small) { + t.Fatalf("write later record: n=%d errno=%v", n, errno) + } + if n, errno := filesystem.Write(largeHandle, large[split:], baseOffset+int64(split)); errno != 0 || n != len(large)-split { + t.Fatalf("write large suffix: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(largeHandle); errno != 0 { + t.Fatalf("commit out-of-order chunks: %v", errno) + } + + got, err := os.ReadFile(nativePath) + if err != nil { + t.Fatal(err) + } + want := append(append(append([]byte(nil), base...), large...), small...) + if !bytes.Equal(got, want) { + t.Fatalf("native append chunks interleaved: got=%d bytes want=%d bytes", len(got), len(want)) + } +} + +func TestNativePassthroughRetryAtOverlappingOffsetIsIdempotent(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := "/sessions/2026/07/16/rollout-retry.jsonl" + nativePath := nativePathFromRoot(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + handle, errno := filesystem.Open(route, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open writer: %v", errno) + } + defer filesystem.Release(handle) + record := []byte("{\"payload\":{\"internal_chat_message_metadata_passthrough\":{\"turn_id\":\"turn\"}}}\n") + retry := record[len(record)-71:] + baseOffset := int64(len(base)) + if n, errno := filesystem.Write(handle, record, baseOffset); errno != 0 || n != len(record) { + t.Fatalf("write record: n=%d errno=%v", n, errno) + } + if n, errno := filesystem.Write(handle, retry, baseOffset+int64(len(record)-len(retry))); errno != 0 || n != len(retry) { + t.Fatalf("retry suffix: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("commit overlapping retry: %v", errno) + } + + got, err := os.ReadFile(nativePath) + if err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), base...), record...) + if !bytes.Equal(got, want) { + t.Fatalf("overlapping retry was appended: got=%q want=%q", got, want) + } +} + +func TestNativePassthroughStagesAppendUntilFsync(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := "/sessions/2026/07/16/rollout-staged.jsonl" + nativePath := nativePathFromRoot(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + writer, errno := filesystem.Open(route, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open writer: %v", errno) + } + defer filesystem.Release(writer) + record := []byte("{\"record\":1}\n") + if n, errno := filesystem.Write(writer, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage append: n=%d errno=%v", n, errno) + } + + backing, err := os.ReadFile(nativePath) + if err != nil || !bytes.Equal(backing, base) { + t.Fatalf("uncommitted bytes reached backing: got=%q err=%v", backing, err) + } + attribute, errno := filesystem.Getattr(route) + if errno != 0 || attribute.Size != int64(len(base)+len(record)) { + t.Fatalf("visible staged size=%d errno=%v", attribute.Size, errno) + } + reader, errno := filesystem.Open(route, os.O_RDONLY) + if errno != 0 { + t.Fatalf("open staged reader: %v", errno) + } + defer filesystem.Release(reader) + visible := make([]byte, len(base)+len(record)) + if n, errno := filesystem.Read(reader, visible, 0); errno != 0 || n != len(visible) { + t.Fatalf("read staged bytes: n=%d errno=%v", n, errno) + } + want := append(append([]byte(nil), base...), record...) + if !bytes.Equal(visible, want) { + t.Fatalf("staged visible bytes=%q want=%q", visible, want) + } + + if errno := filesystem.Fsync(writer); errno != 0 { + t.Fatalf("commit staged append: %v", errno) + } + backing, err = os.ReadFile(nativePath) + if err != nil || !bytes.Equal(backing, want) { + t.Fatalf("committed backing=%q err=%v", backing, err) + } +} + +func TestNativePassthroughInvalidAppendFailsClosed(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := "/sessions/2026/07/16/rollout-invalid.jsonl" + nativePath := nativePathFromRoot(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + writer, errno := filesystem.Open(route, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open writer: %v", errno) + } + defer filesystem.Release(writer) + if n, errno := filesystem.Write(writer, []byte("not-json\n"), int64(len(base))); errno != 0 || n != len("not-json\n") { + t.Fatalf("stage invalid append: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(writer); errno != syscall.EIO { + t.Fatalf("invalid append fsync errno=%v, want EIO", errno) + } + backing, err := os.ReadFile(nativePath) + if err != nil || !bytes.Equal(backing, base) { + t.Fatalf("invalid append changed backing: got=%q err=%v", backing, err) + } + attribute, errno := filesystem.Getattr(route) + if errno != 0 || attribute.Size != int64(len(base)) { + t.Fatalf("invalid append remained visible: size=%d errno=%v", attribute.Size, errno) + } + + record := []byte("{\"record\":1}\n") + if n, errno := filesystem.Write(writer, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("retry valid append: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(writer); errno != 0 { + t.Fatalf("commit valid retry: %v", errno) + } + want := append(append([]byte(nil), base...), record...) + backing, err = os.ReadFile(nativePath) + if err != nil || !bytes.Equal(backing, want) { + t.Fatalf("valid retry backing=%q err=%v", backing, err) + } +} + +func TestFilesystemPathTruncateUsesTheActiveWriter(t *testing.T) { + filesystem, source := mountFixture(t) + handle, errno := filesystem.Open("/session.jsonl", os.O_RDWR) + if errno != 0 { + t.Fatalf("Open writer errno=%v", errno) + } + t.Cleanup(func() { _ = filesystem.Release(handle) }) + wantSize := int64(len(source) - 3) + if errno := filesystem.TruncatePath("/session.jsonl", wantSize); errno != 0 { + t.Fatalf("TruncatePath with active writer errno=%v", errno) + } + attribute, errno := filesystem.Getattr("/session.jsonl") + if errno != 0 || attribute.Size != wantSize { + t.Fatalf("Getattr after path truncate = %#v errno=%v", attribute, errno) + } +} + +func TestFilesystemLoadsAMissingSessionOnceOnFirstAccess(t *testing.T) { + source := []byte("loaded") + session := mountSessionFixture(t, "loaded", source) + filesystem := New() + loads := 0 + filesystem.SetSessionLoader(func(sessionID string) (*vfs.Session, error) { + loads++ + if sessionID != "loaded" { + return nil, os.ErrNotExist + } + return session, nil + }) + for attempt := 0; attempt < 2; attempt++ { + attribute, errno := filesystem.Getattr("/loaded.jsonl") + if errno != 0 || attribute.Size != int64(len(source)) { + t.Fatalf("Getattr attempt %d = %#v errno=%v", attempt, attribute, errno) + } + } + if loads != 1 { + t.Fatalf("session loader calls = %d, want 1", loads) + } +} + +func TestFilesystemRejectsUnsafeAndManagementMutations(t *testing.T) { + filesystem, _ := mountFixture(t) + if _, errno := filesystem.Open("/../session.jsonl", os.O_RDONLY); errno != syscall.ENOENT { + t.Fatalf("unsafe path errno=%v", errno) + } + if errno := filesystem.Rename("/session.jsonl", "/other.jsonl"); errno != syscall.EPERM { + t.Fatalf("Rename errno=%v, want EPERM", errno) + } + if errno := filesystem.Unlink("/session.jsonl"); errno != syscall.EPERM { + t.Fatalf("Unlink errno=%v, want EPERM", errno) + } +} + +func TestCanonicalFilesystemMovesManagedSessionBetweenArchiveAndActivePaths(t *testing.T) { + source := []byte("canonical-session\n") + session := mountSessionFixture(t, "session", source) + filesystem := NewCanonical() + filename := "rollout-2026-07-12T14-28-28-session.jsonl" + archivedPath := "/archived_sessions/" + filename + if err := filesystem.AddSessionAt("session", archivedPath, session); err != nil { + t.Fatal(err) + } + for _, directory := range []string{"/sessions/2026", "/sessions/2026/07", "/sessions/2026/07/12"} { + if errno := filesystem.Mkdir(directory, 0o700); errno != 0 { + t.Fatalf("Mkdir %s errno=%v", directory, errno) + } + } + activePath := "/sessions/2026/07/12/" + filename + if errno := filesystem.Rename(archivedPath, activePath); errno != 0 { + t.Fatalf("Rename errno=%v", errno) + } + if _, errno := filesystem.Getattr(archivedPath); errno != syscall.ENOENT { + t.Fatalf("archived path errno=%v, want ENOENT", errno) + } + attribute, errno := filesystem.Getattr(activePath) + if errno != 0 || attribute.Mode&syscall.S_IFREG == 0 || attribute.Size != int64(len(source)) { + t.Fatalf("active Getattr = %#v errno=%v", attribute, errno) + } + entries, errno := filesystem.ReadDir("/sessions/2026/07/12") + if errno != 0 || len(entries) != 1 || entries[0] != filename { + t.Fatalf("active ReadDir = %#v errno=%v", entries, errno) + } + handle, errno := filesystem.Open(activePath, os.O_RDONLY) + if errno != 0 { + t.Fatalf("Open active errno=%v", errno) + } + defer filesystem.Release(handle) + got := make([]byte, len(source)) + if n, errno := filesystem.Read(handle, got, 0); errno != 0 || n != len(source) || !bytes.Equal(got, source) { + t.Fatalf("Read active = %d errno=%v bytes=%q", n, errno, got) + } +} + +func TestCanonicalFilesystemManagedSessionMasksRetainedSnapshotAtCurrentRoute(t *testing.T) { + root := t.TempDir() + filename := "rollout-2026-07-12T14-28-28-session.jsonl" + archivedPath := "/archived_sessions/" + filename + nativePath := filepath.Join(root, "archived_sessions", filename) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "sessions"), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("native-base\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + session := mountSessionWithNativeSnapshot(t, "session", base, nativePath) + writer, err := session.OpenWriter() + if err != nil { + t.Fatal(err) + } + tail := []byte("managed-tail\n") + if _, err := writer.Append(context.Background(), tail); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + if err := filesystem.AddSessionAt("session", archivedPath, session); err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), base...), tail...) + attribute, errno := filesystem.Getattr(archivedPath) + if errno != 0 || attribute.Size != int64(len(want)) { + t.Fatalf("managed Getattr = %#v errno=%v, want size %d", attribute, errno, len(want)) + } + handle, errno := filesystem.Open(archivedPath, os.O_RDONLY) + if errno != 0 { + t.Fatalf("managed Open errno=%v", errno) + } + defer filesystem.Release(handle) + got := make([]byte, len(want)) + if n, errno := filesystem.Read(handle, got, 0); errno != 0 || n != len(want) || !bytes.Equal(got, want) { + t.Fatalf("managed Read = %d errno=%v bytes=%q want=%q", n, errno, got, want) + } +} + +func TestCanonicalFilesystemNativePreferenceFallsBackToManagedWithoutPathLoss(t *testing.T) { + root := t.TempDir() + route := "/sessions/2026/07/14/rollout-retirement.jsonl" + nativePath := nativePathFromRoot(root, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + managedBytes := []byte("managed-current\n") + nativeBytes := []byte("native-current\n") + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + if err := filesystem.AddSessionAt("session", route, mountSessionFixture(t, "session", managedBytes)); err != nil { + t.Fatal(err) + } + read := func() []byte { + t.Helper() + handle, errno := filesystem.Open(route, os.O_RDONLY) + if errno != 0 { + t.Fatalf("Open errno=%v", errno) + } + defer filesystem.Release(handle) + attribute, errno := filesystem.Getattr(route) + if errno != 0 { + t.Fatalf("Getattr errno=%v", errno) + } + data := make([]byte, attribute.Size) + n, errno := filesystem.Read(handle, data, 0) + if errno != 0 || n != len(data) { + t.Fatalf("Read n=%d errno=%v size=%d", n, errno, len(data)) + } + return data + } + + if got := read(); !bytes.Equal(got, managedBytes) { + t.Fatalf("initial bytes = %q, want managed %q", got, managedBytes) + } + if err := filesystem.PreferNativeSession("session"); err != nil { + t.Fatal(err) + } + if got := read(); !bytes.Equal(got, nativeBytes) { + t.Fatalf("preferred bytes = %q, want native %q", got, nativeBytes) + } + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + if got := read(); !bytes.Equal(got, managedBytes) { + t.Fatalf("fallback bytes = %q, want managed %q", got, managedBytes) + } + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + if got := read(); !bytes.Equal(got, nativeBytes) { + t.Fatalf("restored native bytes = %q, want %q", got, nativeBytes) + } +} + +func TestCanonicalFilesystemHidesRetainedSnapshotAfterManagedRouteMoves(t *testing.T) { + root := t.TempDir() + filename := "rollout-2026-07-12T14-28-28-session.jsonl" + archivedPath := "/archived_sessions/" + filename + activePath := "/sessions/2026/07/12/" + filename + nativePath := filepath.Join(root, "archived_sessions", filename) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "sessions"), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("retained-base\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + session := mountSessionWithNativeSnapshot(t, "session", base, nativePath) + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + if err := filesystem.AddSessionAt("session", archivedPath, session); err != nil { + t.Fatal(err) + } + for _, directory := range []string{"/sessions/2026", "/sessions/2026/07", "/sessions/2026/07/12"} { + if errno := filesystem.Mkdir(directory, 0o700); errno != 0 { + t.Fatalf("Mkdir %s errno=%v", directory, errno) + } + } + if errno := filesystem.Rename(archivedPath, activePath); errno != 0 { + t.Fatalf("Rename errno=%v", errno) + } + if _, errno := filesystem.Getattr(archivedPath); errno != syscall.ENOENT { + t.Fatalf("retained archived Getattr errno=%v, want ENOENT", errno) + } + if _, errno := filesystem.Open(archivedPath, os.O_RDONLY); errno != syscall.ENOENT { + t.Fatalf("retained archived Open errno=%v, want ENOENT", errno) + } + entries, errno := filesystem.ReadDir("/archived_sessions") + if errno != 0 { + t.Fatalf("archived ReadDir errno=%v", errno) + } + for _, entry := range entries { + if entry == filename { + t.Fatalf("retained snapshot leaked into archived directory: %#v", entries) + } + } +} + +func TestCanonicalFilesystemMovesManagedSessionIntoExistingNativeDirectoryAfterRestart(t *testing.T) { + root := t.TempDir() + activeDirectory := filepath.Join(root, "sessions", "2026", "07", "12") + if err := os.MkdirAll(activeDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + + session := mountSessionFixture(t, "session", []byte("restart-route\n")) + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + filename := "rollout-2026-07-12T14-28-28-session.jsonl" + archivedPath := "/archived_sessions/" + filename + activePath := "/sessions/2026/07/12/" + filename + if err := filesystem.AddSessionAt("session", archivedPath, session); err != nil { + t.Fatal(err) + } + if errno := filesystem.Rename(archivedPath, activePath); errno != 0 { + t.Fatalf("Rename into existing native directory errno=%v", errno) + } + if _, errno := filesystem.Getattr(archivedPath); errno != syscall.ENOENT { + t.Fatalf("archived path errno=%v, want ENOENT", errno) + } + if _, errno := filesystem.Getattr(activePath); errno != 0 { + t.Fatalf("active path errno=%v", errno) + } + if len(filesystem.paths) != 1 || filesystem.paths[activePath] != "session" { + t.Fatalf("managed routes after rename = %#v", filesystem.paths) + } + for _, nativePath := range []string{ + nativePathFromRoot(root, archivedPath), + nativePathFromRoot(root, activePath), + } { + if _, err := os.Stat(nativePath); !os.IsNotExist(err) { + t.Fatalf("managed session was duplicated into native root: path=%s err=%v", nativePath, err) + } + } +} + +func TestCanonicalFilesystemKeepsEmptyNativeRootDisabled(t *testing.T) { + filesystem := NewCanonical() + filesystem.SetNativeRoot("") + if filesystem.nativeRoot != "" { + t.Fatalf("empty native root became %q", filesystem.nativeRoot) + } + if _, ok := filesystem.nativePath("/sessions/2026/07/12/rollout.jsonl"); ok { + t.Fatal("empty native root should not resolve a backing path") + } +} + +func TestCanonicalFilesystemHidesNativeFilesOutsideSessionNamespace(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "sessions"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "outside.txt"), []byte("outside"), 0o600); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + entries, errno := filesystem.ReadDir("/") + if errno != 0 || len(entries) != 2 || entries[0] != "archived_sessions" || entries[1] != "sessions" { + t.Fatalf("canonical root entries = %#v errno=%v", entries, errno) + } + if _, errno := filesystem.Getattr("/outside.txt"); errno != syscall.ENOENT { + t.Fatalf("outside Getattr errno=%v, want ENOENT", errno) + } + if _, errno := filesystem.Open("/outside.txt", os.O_RDONLY); errno != syscall.ENOENT { + t.Fatalf("outside Open errno=%v, want ENOENT", errno) + } + if errno := filesystem.Mkdir("/outside", 0o700); errno != syscall.EPERM { + t.Fatalf("outside Mkdir errno=%v, want EPERM", errno) + } +} + +func TestCanonicalFilesystemMovesAppleDoubleSidecarWithManagedRoute(t *testing.T) { + root := t.TempDir() + oldDirectory := filepath.Join(root, "archived_sessions") + newDirectory := filepath.Join(root, "sessions", "2026", "07", "12") + if err := os.MkdirAll(oldDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(newDirectory, 0o700); err != nil { + t.Fatal(err) + } + filename := "rollout-session.jsonl" + oldPath := "/archived_sessions/" + filename + newPath := "/sessions/2026/07/12/" + filename + oldSidecar := filepath.Join(oldDirectory, "._"+filename) + if err := os.WriteFile(oldSidecar, []byte("appledouble-metadata"), 0o600); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + if err := filesystem.AddSessionAt("session", oldPath, mountSessionFixture(t, "session", []byte("session\n"))); err != nil { + t.Fatal(err) + } + if errno := filesystem.MoveSessionAt("session", newPath); errno != nil { + t.Fatalf("MoveSessionAt errno=%v", errno) + } + if _, err := os.Stat(oldSidecar); !os.IsNotExist(err) { + t.Fatalf("old AppleDouble sidecar remained: %v", err) + } + newSidecar := filepath.Join(newDirectory, "._"+filename) + if got, err := os.ReadFile(newSidecar); err != nil || string(got) != "appledouble-metadata" { + t.Fatalf("new AppleDouble sidecar = %q err=%v", got, err) + } + entries, errno := filesystem.ReadDir(filepath.Dir(newPath)) + if errno != 0 || len(entries) != 2 || entries[0] != "._"+filename || entries[1] != filename { + t.Fatalf("session directory entries = %#v errno=%v", entries, errno) + } +} + +func TestCanonicalFilesystemUpsertMovesExistingSessionRoute(t *testing.T) { + session := mountSessionFixture(t, "session", []byte("route-update\n")) + filesystem := NewCanonical() + oldPath := "/archived_sessions/rollout-session.jsonl" + newPath := "/sessions/2026/07/12/rollout-session.jsonl" + if err := filesystem.AddSessionAt("session", oldPath, session); err != nil { + t.Fatal(err) + } + if err := filesystem.UpsertSessionAt("session", newPath, session); err != nil { + t.Fatal(err) + } + if _, errno := filesystem.Getattr(oldPath); errno != syscall.ENOENT { + t.Fatalf("old route errno=%v, want ENOENT", errno) + } + if _, errno := filesystem.Getattr(newPath); errno != 0 { + t.Fatalf("new route errno=%v", errno) + } +} + +func TestCanonicalFilesystemRemoveSessionRevealsNativeFile(t *testing.T) { + root := t.TempDir() + route := "/archived_sessions/rollout-session.jsonl" + nativePath := nativePathFromRoot(root, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(nativePath, []byte("native\n"), 0o600); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + if err := filesystem.AddSessionAt("session", route, mountSessionFixture(t, "managed", []byte("managed\n"))); err != nil { + t.Fatal(err) + } + if err := filesystem.RemoveSession("session"); err != nil { + t.Fatal(err) + } + handle, errno := filesystem.Open(route, os.O_RDONLY) + if errno != 0 { + t.Fatalf("open native after removal errno=%v", errno) + } + defer filesystem.Release(handle) + got := make([]byte, len("native\n")) + if n, errno := filesystem.Read(handle, got, 0); errno != 0 || n != len(got) || string(got) != "native\n" { + t.Fatalf("native after removal = %q n=%d errno=%v", got, n, errno) + } +} + +func TestCanonicalFilesystemPassesThroughNativeSessionFiles(t *testing.T) { + root := t.TempDir() + nativeDirectory := filepath.Join(root, "sessions", "2026", "07", "12") + if err := os.MkdirAll(nativeDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + nativePath := filepath.Join(nativeDirectory, "native.jsonl") + source := []byte("native-session\n") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + entries, errno := filesystem.ReadDir("/sessions/2026/07/12") + if errno != 0 || len(entries) != 1 || entries[0] != "native.jsonl" { + t.Fatalf("native ReadDir = %#v errno=%v", entries, errno) + } + handle, errno := filesystem.Open("/sessions/2026/07/12/native.jsonl", os.O_RDONLY) + if errno != 0 { + t.Fatalf("native Open errno=%v", errno) + } + got := make([]byte, len(source)) + if n, errno := filesystem.Read(handle, got, 0); errno != 0 || n != len(source) || !bytes.Equal(got, source) { + t.Fatalf("native Read = %d errno=%v bytes=%q", n, errno, got) + } + if errno := filesystem.Release(handle); errno != 0 { + t.Fatalf("native Release errno=%v", errno) + } + + createdPath := "/sessions/2026/07/12/created.jsonl" + created, errno := filesystem.Open(createdPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL) + if errno != 0 { + t.Fatalf("native create Open errno=%v", errno) + } + createdBytes := []byte("{\"created\":\"session\"}\n") + if n, errno := filesystem.Write(created, createdBytes, 0); errno != 0 || n != len(createdBytes) { + t.Fatalf("native create Write = %d errno=%v", n, errno) + } + if errno := filesystem.Release(created); errno != 0 { + t.Fatalf("native create Release errno=%v", errno) + } + if got, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(strings.TrimPrefix(createdPath, "/")))); err != nil || !bytes.Equal(got, createdBytes) { + t.Fatalf("native created bytes = %q err=%v", got, err) + } + + renamedPath := "/archived_sessions/created.jsonl" + if errno := filesystem.Rename(createdPath, renamedPath); errno != 0 { + t.Fatalf("native Rename errno=%v", errno) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(strings.TrimPrefix(createdPath, "/")))); !os.IsNotExist(err) { + t.Fatalf("native source remained after rename: %v", err) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(strings.TrimPrefix(renamedPath, "/")))); err != nil { + t.Fatalf("native destination missing after rename: %v", err) + } + if errno := filesystem.Unlink(renamedPath); errno != 0 { + t.Fatalf("native Unlink errno=%v", errno) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(strings.TrimPrefix(renamedPath, "/")))); !os.IsNotExist(err) { + t.Fatalf("native destination remained after unlink: %v", err) + } +} + +func TestCanonicalFilesystemNativeStreamReadFallsBackForVirtualAndPendingFiles(t *testing.T) { + root := t.TempDir() + pathName := "/sessions/2026/07/12/stream.jsonl" + nativePath := nativePathFromRoot(root, pathName) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(nativePath, []byte("native-content\n"), 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + nativeHandle, errno := filesystem.Open(pathName, os.O_RDONLY) + if errno != 0 { + t.Fatalf("open native handle errno=%v", errno) + } + defer filesystem.Release(nativeHandle) + var streamed []byte + handled, n, err := filesystem.StreamNativeRead(nativeHandle, 7, 64, func(file *os.File, offset int64, length int) (int, error) { + streamed = make([]byte, length) + read, err := file.ReadAt(streamed, offset) + streamed = streamed[:read] + return read, err + }) + if !handled || err != nil || n != len(streamed) || string(streamed) != "content\n" { + t.Fatalf("native stream handled=%t n=%d err=%v bytes=%q", handled, n, err, streamed) + } + + writer, errno := filesystem.Open(pathName, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open append handle errno=%v", errno) + } + pending := []byte("{\"pending\":true}\n") + if written, errno := filesystem.Write(writer, pending, int64(len("native-content\n"))); errno != 0 || written != len(pending) { + t.Fatalf("stage append written=%d errno=%v", written, errno) + } + handled, _, err = filesystem.StreamNativeRead(nativeHandle, 0, 64, func(_ *os.File, _ int64, _ int) (int, error) { + t.Fatal("pending append unexpectedly used native stream") + return 0, nil + }) + if handled || err != nil { + t.Fatalf("pending append stream handled=%t err=%v, want fallback", handled, err) + } + var pendingStream []byte + pendingTotals := make(map[int]struct{}) + n, err = filesystem.StreamBufferedRead(nativeHandle, 0, 64, 4, func(total int, chunk []byte) error { + pendingTotals[total] = struct{}{} + pendingStream = append(pendingStream, chunk...) + return nil + }) + wantPending := append([]byte("native-content\n"), pending...) + if err != nil || n != len(wantPending) || !bytes.Equal(pendingStream, wantPending) || len(pendingTotals) != 1 { + t.Fatalf("pending buffered stream n=%d err=%v totals=%v bytes=%q", n, err, pendingTotals, pendingStream) + } + if errno := filesystem.Release(writer); errno != 0 { + t.Fatalf("release append handle errno=%v", errno) + } + + virtualPath := "/sessions/2026/07/12/virtual.jsonl" + if err := filesystem.AddSessionAt("virtual", virtualPath, mountSessionFixture(t, "virtual", []byte("virtual\n"))); err != nil { + t.Fatal(err) + } + virtualHandle, errno := filesystem.Open(virtualPath, os.O_RDONLY) + if errno != 0 { + t.Fatalf("open virtual handle errno=%v", errno) + } + defer filesystem.Release(virtualHandle) + handled, _, err = filesystem.StreamNativeRead(virtualHandle, 0, 64, func(_ *os.File, _ int64, _ int) (int, error) { + t.Fatal("virtual session unexpectedly used native stream") + return 0, nil + }) + if handled || err != nil { + t.Fatalf("virtual stream handled=%t err=%v, want fallback", handled, err) + } + var virtualStream []byte + var virtualTotals []int + n, err = filesystem.StreamBufferedRead(virtualHandle, 1, 64, 3, func(total int, chunk []byte) error { + virtualTotals = append(virtualTotals, total) + virtualStream = append(virtualStream, chunk...) + return nil + }) + if err != nil || n != len("irtual\n") || string(virtualStream) != "irtual\n" || !slices.Equal(virtualTotals, []int{7, 7, 7}) { + t.Fatalf("virtual buffered stream n=%d err=%v totals=%v bytes=%q", n, err, virtualTotals, virtualStream) + } +} + +func TestCanonicalFilesystemSupportsFSKitOpenUnlinkStaging(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "sessions", "2026", "07", "12") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + + original := "/sessions/2026/07/12/open.jsonl" + hidden := "/sessions/2026/07/12/.nfs.20051026.83fd" + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(filepath.Join(directory, "open.jsonl"), base, 0o600); err != nil { + t.Fatal(err) + } + handle, errno := filesystem.Open(original, os.O_RDWR) + if errno != 0 { + t.Fatalf("open errno=%v", errno) + } + if errno := filesystem.Rename(original, hidden); errno != 0 { + t.Fatalf("open-unlink rename errno=%v", errno) + } + if got, errno := filesystem.HandlePath(handle); errno != 0 || got != hidden { + t.Fatalf("handle path = %q errno=%v, want %q", got, errno, hidden) + } + appended := []byte("{\"record\":1}\n") + if n, errno := filesystem.Write(handle, appended, int64(len(base))); errno != 0 || n != len(appended) { + t.Fatalf("write after hidden rename = %d errno=%v", n, errno) + } + if errno := filesystem.Release(handle); errno != 0 { + t.Fatalf("release errno=%v", errno) + } + hiddenNative := filepath.Join(directory, filepath.Base(hidden)) + want := append(append([]byte(nil), base...), appended...) + if got, err := os.ReadFile(hiddenNative); err != nil || !bytes.Equal(got, want) { + t.Fatalf("hidden bytes = %q err=%v, want %q", got, err, want) + } + if errno := filesystem.Unlink(hidden); errno != 0 { + t.Fatalf("unlink hidden errno=%v", errno) + } + if _, err := os.Stat(hiddenNative); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("hidden file remained: %v", err) + } + + second := "/sessions/2026/07/12/second.jsonl" + if err := os.WriteFile(filepath.Join(directory, "second.jsonl"), base, 0o600); err != nil { + t.Fatal(err) + } + if errno := filesystem.Rename(second, "/sessions/2026/07/12/.not-nfs"); errno != syscall.EPERM { + t.Fatalf("non-FSKit hidden rename errno=%v, want EPERM", errno) + } + if errno := filesystem.Rename(second, "/archived_sessions/.nfs.20051026.83fd"); errno != syscall.EPERM { + t.Fatalf("cross-directory open-unlink rename errno=%v, want EPERM", errno) + } +} + +func TestCanonicalFilesystemRenamesOpenNativeReader(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "sessions", "2026", "07", "12") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + + original := "/sessions/2026/07/12/open-reader.jsonl" + renamed := "/sessions/2026/07/12/renamed-reader.jsonl" + want := []byte("{\"record\":0}\n") + if err := os.WriteFile(filepath.Join(directory, "open-reader.jsonl"), want, 0o600); err != nil { + t.Fatal(err) + } + handle, errno := filesystem.Open(original, os.O_RDONLY) + if errno != 0 { + t.Fatalf("open errno=%v", errno) + } + defer filesystem.Release(handle) + + if errno := filesystem.Rename(original, renamed); errno != 0 { + t.Fatalf("rename open reader errno=%v", errno) + } + if got, errno := filesystem.HandlePath(handle); errno != 0 || got != renamed { + t.Fatalf("handle path = %q errno=%v, want %q", got, errno, renamed) + } + buffer := make([]byte, len(want)) + if n, errno := filesystem.Read(handle, buffer, 0); errno != 0 || n != len(want) || !bytes.Equal(buffer, want) { + t.Fatalf("read after rename = %q n=%d errno=%v, want %q", buffer, n, errno, want) + } + if _, err := os.Stat(filepath.Join(directory, "open-reader.jsonl")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("source remained after rename: %v", err) + } + if got, err := os.ReadFile(filepath.Join(directory, "renamed-reader.jsonl")); err != nil || !bytes.Equal(got, want) { + t.Fatalf("renamed bytes = %q err=%v, want %q", got, err, want) + } +} + +func TestFilesystemUpsertChangesNewOpensWithoutInvalidatingExistingHandles(t *testing.T) { + first := mountSessionFixture(t, "first-session", []byte("first")) + second := mountSessionFixture(t, "second-session", []byte("second")) + filesystem := New() + if err := filesystem.AddSession("session", first); err != nil { + t.Fatal(err) + } + oldHandle, errno := filesystem.Open("/session.jsonl", os.O_RDONLY) + if errno != 0 { + t.Fatalf("open old handle: %v", errno) + } + if err := filesystem.UpsertSession("session", second); err != nil { + t.Fatalf("UpsertSession: %v", err) + } + newHandle, errno := filesystem.Open("/session.jsonl", os.O_RDONLY) + if errno != 0 { + t.Fatalf("open new handle: %v", errno) + } + oldBytes := make([]byte, 5) + if n, errno := filesystem.Read(oldHandle, oldBytes, 0); errno != 0 || n != 5 || string(oldBytes) != "first" { + t.Fatalf("old handle changed: n=%d errno=%v bytes=%q", n, errno, oldBytes) + } + newBytes := make([]byte, 6) + if n, errno := filesystem.Read(newHandle, newBytes, 0); errno != 0 || n != 6 || string(newBytes) != "second" { + t.Fatalf("new handle did not use replacement: n=%d errno=%v bytes=%q", n, errno, newBytes) + } + _ = filesystem.Release(oldHandle) + _ = filesystem.Release(newHandle) +} + +func TestFilesystemOwnedGenerationClosesAfterItsLastHandle(t *testing.T) { + first := mountSessionFixture(t, "first-session", []byte("first")) + second := mountSessionFixture(t, "second-session", []byte("second")) + firstCloser := &countingCloser{} + secondCloser := &countingCloser{} + filesystem := New() + if err := filesystem.AddSessionOwned("session", first, firstCloser); err != nil { + t.Fatal(err) + } + oldHandle, errno := filesystem.Open("/session.jsonl", os.O_RDONLY) + if errno != 0 { + t.Fatalf("open old handle: %v", errno) + } + if err := filesystem.UpsertSessionOwned("session", second, secondCloser); err != nil { + t.Fatalf("upsert owned generation: %v", err) + } + if firstCloser.calls != 0 { + t.Fatal("old generation closed while its read handle remained open") + } + newHandle, errno := filesystem.Open("/session.jsonl", os.O_RDONLY) + if errno != 0 { + t.Fatalf("open new handle: %v", errno) + } + oldBytes := make([]byte, 5) + if n, errno := filesystem.Read(oldHandle, oldBytes, 0); errno != 0 || n != len(oldBytes) || string(oldBytes) != "first" { + t.Fatalf("old generation read: n=%d errno=%v bytes=%q", n, errno, oldBytes) + } + newBytes := make([]byte, 6) + if n, errno := filesystem.Read(newHandle, newBytes, 0); errno != 0 || n != len(newBytes) || string(newBytes) != "second" { + t.Fatalf("new generation read: n=%d errno=%v bytes=%q", n, errno, newBytes) + } + if err := filesystem.RemoveSession("session"); err != nil { + t.Fatalf("remove current generation: %v", err) + } + if secondCloser.calls != 0 { + t.Fatal("current generation closed while its read handle remained open") + } + if errno := filesystem.Release(oldHandle); errno != 0 { + t.Fatalf("release old handle: %v", errno) + } + if firstCloser.calls != 1 { + t.Fatalf("old generation close calls=%d, want 1", firstCloser.calls) + } + if errno := filesystem.Release(newHandle); errno != 0 { + t.Fatalf("release new handle: %v", errno) + } + if secondCloser.calls != 1 { + t.Fatalf("new generation close calls=%d, want 1", secondCloser.calls) + } +} + +func TestFilesystemOwnedGenerationClosesImmediatelyWithoutHandles(t *testing.T) { + firstCloser := &countingCloser{} + secondCloser := &countingCloser{} + filesystem := New() + if err := filesystem.AddSessionOwned("session", mountSessionFixture(t, "first-session", []byte("first")), firstCloser); err != nil { + t.Fatal(err) + } + if err := filesystem.UpsertSessionOwned("session", mountSessionFixture(t, "second-session", []byte("second")), secondCloser); err != nil { + t.Fatal(err) + } + if firstCloser.calls != 1 { + t.Fatalf("replaced generation close calls=%d, want 1", firstCloser.calls) + } + if err := filesystem.CloseSessions(); err != nil { + t.Fatal(err) + } + if secondCloser.calls != 1 { + t.Fatalf("current generation close calls=%d, want 1", secondCloser.calls) + } +} + +func TestFilesystemRejectedOwnedSessionClosesIncomingOwner(t *testing.T) { + closer := &countingCloser{} + filesystem := New() + err := filesystem.AddSessionOwned("", mountSessionFixture(t, "session", []byte("session")), closer) + if err == nil { + t.Fatal("invalid owned session was accepted") + } + if closer.calls != 1 { + t.Fatalf("rejected owner close calls=%d, want 1", closer.calls) + } +} + +type countingCloser struct { + calls int +} + +func (c *countingCloser) Close() error { + c.calls++ + return nil +} + +func TestMountWithoutFuseBuildReturnsPrerequisiteError(t *testing.T) { + if Available() { + t.Skip("FUSE-enabled builds are covered by the gated real mount test") + } + err := Mount(context.Background(), HostOptions{MountPoint: t.TempDir(), Filesystem: New()}) + if !errors.Is(err, ErrPrerequisite) { + t.Fatalf("Mount error = %v, want ErrPrerequisite", err) + } +} + +func TestCanonicalSyntheticDirectoryAttributesStayStableUntilContentsChange(t *testing.T) { + filesystem := NewCanonical() + first, errno := filesystem.Getattr("/sessions") + if errno != 0 { + t.Fatalf("first Getattr errno=%v", errno) + } + second, errno := filesystem.Getattr("/sessions") + if errno != 0 { + t.Fatalf("second Getattr errno=%v", errno) + } + if first.ObjectID != second.ObjectID || !first.ModTime.Equal(second.ModTime) || !first.ChangeTime.Equal(second.ChangeTime) { + t.Fatalf("unchanged synthetic directory attributes drifted: first=%#v second=%#v", first, second) + } + + archived, errno := filesystem.Getattr("/archived_sessions") + if errno != 0 { + t.Fatalf("archived Getattr errno=%v", errno) + } + filesystem.bumpDirectoryGeneration("/sessions") + changed, errno := filesystem.Getattr("/sessions") + if errno != 0 { + t.Fatalf("changed Getattr errno=%v", errno) + } + if changed.ObjectID != first.ObjectID { + t.Fatalf("directory content change replaced stable object identity %q with %q", first.ObjectID, changed.ObjectID) + } + if changed.DirectoryGeneration <= first.DirectoryGeneration || !changed.ModTime.After(first.ModTime) || !changed.ChangeTime.After(first.ChangeTime) { + t.Fatalf("directory content generation did not advance attributes: first=%#v changed=%#v", first, changed) + } + archivedAfter, errno := filesystem.Getattr("/archived_sessions") + if errno != 0 { + t.Fatalf("archived Getattr after unrelated change errno=%v", errno) + } + if archivedAfter.ObjectID != archived.ObjectID { + t.Fatalf("unrelated directory identity changed from %q to %q", archived.ObjectID, archivedAfter.ObjectID) + } +} + +func TestCanonicalNativeDirectoryObjectIdentityUsesExplicitContentGeneration(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "sessions", "2026", "07", "22") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(root, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + first, errno := filesystem.Getattr("/sessions/2026/07/22") + if errno != 0 { + t.Fatalf("first Getattr errno=%v", errno) + } + if err := os.WriteFile(filepath.Join(directory, "created.jsonl"), []byte("{}\n"), 0o600); err != nil { + t.Fatal(err) + } + beforeNotification, errno := filesystem.Getattr("/sessions/2026/07/22") + if errno != 0 { + t.Fatalf("pre-notification Getattr errno=%v", errno) + } + if beforeNotification.ObjectID != first.ObjectID { + t.Fatalf("native metadata alone changed object ID from %q to %q", first.ObjectID, beforeNotification.ObjectID) + } + filesystem.bumpDirectoryGeneration("/sessions/2026/07/22") + afterNotification, errno := filesystem.Getattr("/sessions/2026/07/22") + if errno != 0 { + t.Fatalf("post-notification Getattr errno=%v", errno) + } + if afterNotification.ObjectID != first.ObjectID { + t.Fatalf("explicit directory content generation replaced object ID %q with %q", first.ObjectID, afterNotification.ObjectID) + } + if afterNotification.DirectoryGeneration <= first.DirectoryGeneration { + t.Fatalf("directory generation did not advance: before=%d after=%d", first.DirectoryGeneration, afterNotification.DirectoryGeneration) + } +} + +type mountReader map[string][]byte + +func (r mountReader) ReadAt(_ context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { + data := r[ref.SHA256] + if offset >= int64(len(data)) { + return 0, io.EOF + } + n := copy(destination, data[offset:]) + if n < len(destination) { + return n, io.EOF + } + return n, nil +} + +func mountFixture(t *testing.T) (*Filesystem, []byte) { + t.Helper() + root := t.TempDir() + source := []byte("first\nsecond\nthird\n") + digest := sha256.Sum256(source) + hexDigest := hex.EncodeToString(digest[:]) + nativePath := filepath.Join(root, "native.jsonl") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatalf("write native: %v", err) + } + manifest := fold.Manifest{Version: fold.ManifestVersion, Kind: fold.ManifestKind, Session: fold.ManifestSession{ID: "session", RolloutPath: nativePath}, Source: fold.ManifestSource{Bytes: int64(len(source)), SHA256: hexDigest}, Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: hexDigest, RawBytes: int64(len(source))}}}} + session, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: mountReader{hexDigest: source}, NativeSnapshot: vfs.NativeFile{Path: nativePath, Bytes: int64(len(source)), SHA256: hexDigest}}) + if err != nil { + t.Fatalf("OpenSession: %v", err) + } + filesystem := New() + if err := filesystem.AddSession("session", session); err != nil { + t.Fatalf("AddSession: %v", err) + } + return filesystem, source +} + +func mountSessionFixture(t *testing.T, sessionID string, source []byte) *vfs.Session { + t.Helper() + root := t.TempDir() + digest := sha256.Sum256(source) + hexDigest := hex.EncodeToString(digest[:]) + nativePath := filepath.Join(root, "native.jsonl") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatal(err) + } + manifest := fold.Manifest{Version: fold.ManifestVersion, Kind: fold.ManifestKind, Session: fold.ManifestSession{ID: sessionID, RolloutPath: nativePath}, Source: fold.ManifestSource{Bytes: int64(len(source)), SHA256: hexDigest}, Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: hexDigest, RawBytes: int64(len(source))}}}} + session, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: mountReader{hexDigest: source}, NativeSnapshot: vfs.NativeFile{Path: nativePath, Bytes: int64(len(source)), SHA256: hexDigest}}) + if err != nil { + t.Fatal(err) + } + return session +} + +func mountSessionWithNativeSnapshot(t *testing.T, sessionID string, source []byte, nativePath string) *vfs.Session { + t.Helper() + root := t.TempDir() + digest := sha256.Sum256(source) + hexDigest := hex.EncodeToString(digest[:]) + manifest := fold.Manifest{Version: fold.ManifestVersion, Kind: fold.ManifestKind, Session: fold.ManifestSession{ID: sessionID, RolloutPath: nativePath}, Source: fold.ManifestSource{Bytes: int64(len(source)), SHA256: hexDigest}, Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: hexDigest, RawBytes: int64(len(source))}}}} + session, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: mountReader{hexDigest: source}, NativeSnapshot: vfs.NativeFile{Path: nativePath, Bytes: int64(len(source)), SHA256: hexDigest}}) + if err != nil { + t.Fatal(err) + } + return session +} diff --git a/internal/mountfs/fuse_integration_linux_test.go b/internal/mountfs/fuse_integration_linux_test.go new file mode 100644 index 0000000..a4a476f --- /dev/null +++ b/internal/mountfs/fuse_integration_linux_test.go @@ -0,0 +1,405 @@ +//go:build linux && fuse && fuse3 && cgo + +package mountfs + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "sync" + "testing" + "time" +) + +func TestRealFuse3ManagedReadWriteAndRestart(t *testing.T) { + requireRealFuse3(t) + root := t.TempDir() + source := []byte("{\"record\":0}\n") + managed := mountSessionFixture(t, "linux-managed", source) + filesystem := New() + if err := filesystem.AddSession("linux-managed", managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealFuse3Mount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, "linux-managed.jsonl") + if got, err := os.ReadFile(target); err != nil || !bytes.Equal(got, source) { + t.Fatalf("initial managed read = %q err=%v", got, err) + } + + file, err := os.OpenFile(target, os.O_RDWR|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + tail := []byte("{\"record\":1}\n") + if _, err := file.Write(tail); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), source...), tail...) + + file, err = os.OpenFile(target, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteAt([]byte("PATCH"), 2); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Truncate(int64(len(want) - 2)); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + copy(want[2:], []byte("PATCH")) + want = want[:len(want)-2] + if got, err := os.ReadFile(target); err != nil || !bytes.Equal(got, want) { + t.Fatalf("mutated managed read = %q err=%v", got, err) + } + if managed.State().BackingPath == "" { + t.Fatal("random write did not enter copy-on-write backing") + } + + stopMount() + waitForRealFuse3Unmount(t, mountPoint) + assertRealFuse3BackingSealed(t, mountPoint) + stopRemount := startRealFuse3Mount(t, mountPoint, filesystem) + if got, err := os.ReadFile(target); err != nil || !bytes.Equal(got, want) { + t.Fatalf("remounted managed read = %q err=%v", got, err) + } + stopRemount() + waitForRealFuse3Unmount(t, mountPoint) + assertRealFuse3BackingSealed(t, mountPoint) +} + +func TestRealFuse3CanonicalArchiveUnarchiveRename(t *testing.T) { + requireRealFuse3(t) + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + activeDirectory := filepath.Join(nativeRoot, "sessions", "2026", "07", "16") + archivedDirectory := filepath.Join(nativeRoot, "archived_sessions") + for _, directory := range []string{activeDirectory, archivedDirectory} { + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + } + source := []byte("{\"canonical\":true}\n") + managed := mountSessionFixture(t, "linux-canonical", source) + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + filename := "rollout-linux-canonical.jsonl" + if err := filesystem.AddSessionAt("linux-canonical", "/archived_sessions/"+filename, managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealFuse3Mount(t, mountPoint, filesystem) + archivedPath := filepath.Join(mountPoint, "archived_sessions", filename) + activePath := filepath.Join(mountPoint, "sessions", "2026", "07", "16", filename) + if err := os.Rename(archivedPath, activePath); err != nil { + t.Fatalf("unarchive rename: %v", err) + } + if got, err := os.ReadFile(activePath); err != nil || !bytes.Equal(got, source) { + t.Fatalf("active managed read = %q err=%v", got, err) + } + if _, err := os.Stat(archivedPath); !os.IsNotExist(err) { + t.Fatalf("archived route remained after unarchive: %v", err) + } + if err := os.Rename(activePath, archivedPath); err != nil { + t.Fatalf("archive rename: %v", err) + } + if got, err := os.ReadFile(archivedPath); err != nil || !bytes.Equal(got, source) { + t.Fatalf("restored archived read = %q err=%v", got, err) + } + nativePath := filepath.Join(nativeRoot, "archived_sessions", filename) + nativeBytes := []byte("{\"native_fallback\":true}\n") + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + if err := filesystem.PreferNativeSession("linux-canonical"); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(archivedPath); err != nil || !bytes.Equal(got, nativeBytes) { + t.Fatalf("preferred native read = %q err=%v", got, err) + } + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(archivedPath); err != nil || !bytes.Equal(got, source) { + t.Fatalf("managed fallback read = %q err=%v", got, err) + } + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + if err := filesystem.RemoveSession("linux-canonical"); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(archivedPath); err != nil || !bytes.Equal(got, nativeBytes) { + t.Fatalf("native read after managed removal = %q err=%v", got, err) + } + stopMount() + waitForRealFuse3Unmount(t, mountPoint) + assertRealFuse3BackingSealed(t, mountPoint) +} + +func TestRealFuse3HostCrashUnmountsAndRestarts(t *testing.T) { + requireRealFuse3(t) + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("sessions", "2026", "07", "16", "rollout-crash.jsonl") + nativePath := filepath.Join(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"crash_recovery\":true}\n") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + + for attempt := 1; attempt <= 2; attempt++ { + process, done, output := startRealFuse3Helper(t, mountPoint, nativeRoot) + waitForRealFuse3ProcessMount(t, mountPoint, done, output) + mountedPath := filepath.Join(mountPoint, route) + if got, err := os.ReadFile(mountedPath); err != nil || !bytes.Equal(got, source) { + t.Fatalf("attempt %d mounted read = %q err=%v", attempt, got, err) + } + if err := process.Kill(); err != nil { + t.Fatal(err) + } + if err := <-done; err == nil { + t.Fatalf("attempt %d killed helper exited successfully", attempt) + } + if attempt == 2 { + if err := recoverStaleMount(mountPoint); err != nil { + t.Fatal(err) + } + waitForRealFuse3Unmount(t, mountPoint) + assertRealFuse3BackingSealed(t, mountPoint) + } + } +} + +func TestRealFuse3ReadAndFsyncPerformance(t *testing.T) { + requireRealFuse3(t) + root := t.TempDir() + line := []byte("{\"payload\":\"0123456789abcdef0123456789abcdef0123456789abcdef\"}\n") + source := bytes.Repeat(line, (16<<20)/len(line)+1) + source = source[:16<<20] + managed := mountSessionFixture(t, "linux-performance", source) + filesystem := New() + if err := filesystem.AddSession("linux-performance", managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealFuse3Mount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, "linux-performance.jsonl") + + readStart := time.Now() + file, err := os.Open(target) + if err != nil { + t.Fatal(err) + } + readBytes, copyErr := io.Copy(io.Discard, file) + closeErr := file.Close() + if copyErr != nil || closeErr != nil || readBytes != int64(len(source)) { + t.Fatalf("mounted performance read bytes=%d copy=%v close=%v", readBytes, copyErr, closeErr) + } + readDuration := time.Since(readStart) + readMiBPerSecond := float64(readBytes) / (1024 * 1024) / readDuration.Seconds() + if readMiBPerSecond < 25 { + t.Fatalf("mounted read throughput %.2f MiB/s is below the 25 MiB/s safety floor", readMiBPerSecond) + } + + file, err = os.OpenFile(target, os.O_RDWR|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + durations := make([]time.Duration, 0, 50) + for index := range 50 { + started := time.Now() + if _, err := fmt.Fprintf(file, "{\"append\":%d}\n", index); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + t.Fatal(err) + } + durations = append(durations, time.Since(started)) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + p95 := durations[(len(durations)*95+99)/100-1] + if p95 > 250*time.Millisecond { + t.Fatalf("mounted append+fsync p95 %s exceeds the 250ms safety ceiling", p95) + } + t.Logf("FUSE3 read=%.2f MiB/s append_fsync_p95=%s", readMiBPerSecond, p95) + stopMount() + waitForRealFuse3Unmount(t, mountPoint) + assertRealFuse3BackingSealed(t, mountPoint) +} + +func TestRealFuse3CrashHelper(t *testing.T) { + if os.Getenv("CODEXFOLD_FUSE3_CRASH_HELPER") != "1" { + t.Skip("helper process") + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(os.Getenv("CODEXFOLD_FUSE3_NATIVE_ROOT")) + if err := Mount(context.Background(), HostOptions{ + MountPoint: os.Getenv("CODEXFOLD_FUSE3_MOUNT_POINT"), Filesystem: filesystem, Foreground: true, + }); err != nil { + t.Fatal(err) + } +} + +func requireRealFuse3(t *testing.T) { + t.Helper() + if os.Getenv("CODEXFOLD_RUN_FUSE3_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE3_TEST=1 to run the real Linux FUSE3 adapter test") + } +} + +func startRealFuse3Mount(t *testing.T, mountPoint string, filesystem *Filesystem) func() { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + mountDone := make(chan error, 1) + go func() { + mountDone <- Mount(ctx, HostOptions{MountPoint: mountPoint, Filesystem: filesystem, Foreground: true}) + }() + var stopOnce sync.Once + stop := func() { + stopOnce.Do(func() { + cancel() + select { + case err := <-mountDone: + if err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("FUSE3 mount shutdown: %v", err) + } + case <-time.After(10 * time.Second): + t.Error("FUSE3 mount did not stop after cancellation") + } + }) + } + t.Cleanup(stop) + deadline := time.Now().Add(20 * time.Second) + var lastProbeErr error + for time.Now().Before(deadline) { + if err := probeRealFuse3Mount(mountPoint); err == nil { + return stop + } else { + lastProbeErr = err + } + select { + case err := <-mountDone: + t.Fatalf("FUSE3 mount exited before health: %v", err) + case <-time.After(100 * time.Millisecond): + } + } + t.Fatalf("FUSE3 mount did not become healthy: %v", lastProbeErr) + return stop +} + +func waitForRealFuse3Unmount(t *testing.T, mountPoint string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if !linuxFuseMountVisible(mountPoint) { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("FUSE3 mount remained active after shutdown") +} + +func probeRealFuse3Mount(mountPoint string) error { + identity, err := os.ReadFile(filepath.Join(mountPoint, ".codexfold-health")) + if err != nil { + return err + } + if len(identity) < 16 { + return fmt.Errorf("mount identity is too short: %d", len(identity)) + } + return nil +} + +func assertRealFuse3BackingSealed(t *testing.T, mountPoint string) { + t.Helper() + info, err := os.Stat(mountPoint) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm()&0o200 != 0 { + t.Fatalf("unmounted FUSE3 backing remained writable: mode=%#o", info.Mode().Perm()) + } +} + +func startRealFuse3Helper(t *testing.T, mountPoint string, nativeRoot string) (*os.Process, <-chan error, *bytes.Buffer) { + t.Helper() + command := exec.Command(os.Args[0], "-test.run=^TestRealFuse3CrashHelper$", "-test.v") + command.Env = append(os.Environ(), + "CODEXFOLD_FUSE3_CRASH_HELPER=1", + "CODEXFOLD_FUSE3_MOUNT_POINT="+mountPoint, + "CODEXFOLD_FUSE3_NATIVE_ROOT="+nativeRoot, + ) + var output bytes.Buffer + command.Stdout = &output + command.Stderr = &output + if err := command.Start(); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- command.Wait() }() + return command.Process, done, &output +} + +func waitForRealFuse3ProcessMount(t *testing.T, mountPoint string, done <-chan error, output *bytes.Buffer) { + t.Helper() + deadline := time.Now().Add(20 * time.Second) + var lastProbeErr error + for time.Now().Before(deadline) { + if err := probeRealFuse3Mount(mountPoint); err == nil { + return + } else { + lastProbeErr = err + } + select { + case err := <-done: + t.Fatalf("FUSE3 helper exited before health: %v output=%s", err, output.String()) + case <-time.After(100 * time.Millisecond): + } + } + t.Fatalf("FUSE3 helper did not become healthy: %v output=%s", lastProbeErr, output.String()) +} diff --git a/internal/mountfs/fuse_integration_test.go b/internal/mountfs/fuse_integration_test.go new file mode 100644 index 0000000..0b2c0ec --- /dev/null +++ b/internal/mountfs/fuse_integration_test.go @@ -0,0 +1,974 @@ +//go:build darwin && fuse && cgo + +package mountfs + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/service" + "github.com/samekind/codexfold/internal/vfs" + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/unix" +) + +func TestOperationTraceRecordsWriteShapeWithoutPath(t *testing.T) { + var recorded []string + filesystem := &fuseFilesystem{recorder: func(operation string) { + recorded = append(recorded, operation) + }} + privatePath := "/sessions/private-session.jsonl" + filesystem.recordOpen("open", privatePath, 0x9, os.O_WRONLY|os.O_APPEND, 17, 0) + filesystem.recordIO("write", privatePath, 17, 1234, 89, 89) + filesystem.recordIO("read", privatePath, 17, 1200, 64, 64) + filesystem.recordHandleResult("flush", privatePath, 17, 0) + + joined := strings.Join(recorded, "\n") + for _, field := range []string{ + "open kind=session flags=0x9 translated=0x9 handle=17 result=0", + "write kind=session handle=17 offset=1234 bytes=89 result=89", + "read kind=session handle=17 offset=1200 bytes=64 result=64", + "flush kind=session handle=17 result=0", + } { + if !strings.Contains(joined, field) { + t.Fatalf("operation trace missing %q: %s", field, joined) + } + } + if strings.Contains(joined, privatePath) || strings.Contains(joined, "private-session") { + t.Fatalf("operation trace exposed a session path: %s", joined) + } +} + +func TestOpenExUsesDirectIOOnlyForWritableSessions(t *testing.T) { + source := []byte("{\"record\":0}\n") + managed := mountSessionFixture(t, "direct-io", source) + core := New() + if err := core.AddSession("direct-io", managed); err != nil { + t.Fatal(err) + } + filesystem := &fuseFilesystem{core: core} + + readOnly := fuse.FileInfo_t{Flags: fuse.O_RDONLY} + if result := filesystem.OpenEx("/direct-io.jsonl", &readOnly); result != 0 { + t.Fatalf("read-only OpenEx result=%d", result) + } + if readOnly.DirectIo { + t.Fatal("read-only session unexpectedly enabled direct I/O") + } + if errno := core.Release(readOnly.Fh); errno != 0 { + t.Fatalf("release read-only handle errno=%v", errno) + } + + writable := fuse.FileInfo_t{Flags: fuse.O_RDWR} + if result := filesystem.OpenEx("/direct-io.jsonl", &writable); result != 0 { + t.Fatalf("writable OpenEx result=%d", result) + } + if !writable.DirectIo { + t.Fatal("writable session did not enable direct I/O") + } + if errno := core.Release(writable.Fh); errno != 0 { + t.Fatalf("release writable handle errno=%v", errno) + } +} + +func TestFuseStatfsFallsBackToMountParentWithoutNativeRoot(t *testing.T) { + filesystem := &fuseFilesystem{core: New(), statRoot: t.TempDir()} + var stat fuse.Statfs_t + if result := filesystem.Statfs("/", &stat); result != 0 { + t.Fatalf("Statfs result=%d", result) + } + if stat.Bsize == 0 || stat.Blocks == 0 { + t.Fatalf("Statfs returned no backing capacity: %#v", stat) + } +} + +func TestRealFuseMountNativeFileOperations(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + source := []byte("first\nsecond\nthird\n") + digest := sha256.Sum256(source) + digestHex := hex.EncodeToString(digest[:]) + nativePath := filepath.Join(root, "native.jsonl") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatal(err) + } + manifest := fold.Manifest{ + Version: fold.ManifestVersion, Kind: fold.ManifestKind, + Session: fold.ManifestSession{ID: "fixture", RolloutPath: nativePath, Archived: true}, + Source: fold.ManifestSource{Bytes: int64(len(source)), SHA256: digestHex}, + Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: digestHex, RawBytes: int64(len(source))}}}, + } + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{ + Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, + Reader: fuseFixtureReader{digestHex: source}, NativeSnapshot: vfs.NativeFile{Path: nativePath, Bytes: int64(len(source)), SHA256: digestHex}, + }) + if err != nil { + t.Fatal(err) + } + filesystem := New() + if err := filesystem.AddSession("fixture", managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealMount(t, mountPoint, filesystem) + identity, err := os.ReadFile(filepath.Join(mountPoint, ".codexfold-health")) + if err != nil || len(identity) < 16 { + t.Fatalf("mount identity file is unavailable: size=%d err=%v", len(identity), err) + } + target := filepath.Join(mountPoint, "fixture.jsonl") + entries, err := os.ReadDir(mountPoint) + if err != nil || len(entries) != 1 || entries[0].Name() != "fixture.jsonl" { + t.Fatalf("native directory listing = %#v err=%v", entries, err) + } + initialInfo, err := os.Stat(target) + if err != nil || initialInfo.Size() != int64(len(source)) || initialInfo.Mode().Perm() != 0o600 { + t.Fatalf("initial native stat = %#v err=%v", initialInfo, err) + } + read, err := os.ReadFile(target) + if err != nil || string(read) != string(source) { + t.Fatalf("native read differs: %q err=%v", read, err) + } + hotSource := []byte("hot\n") + hotSession := mountSessionFixture(t, "hot", hotSource) + var hotAvailable atomic.Bool + filesystem.SetSessionLoader(func(sessionID string) (*vfs.Session, error) { + if sessionID != "hot" || !hotAvailable.Load() { + return nil, os.ErrNotExist + } + return hotSession, nil + }) + hotAvailable.Store(true) + hotTarget := filepath.Join(mountPoint, "hot.jsonl") + waitForRealFile(t, hotTarget, hotSource) + appendFile, err := os.OpenFile(target, os.O_RDWR|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + tail := []byte("tail\n") + if _, err := appendFile.Write(tail); err != nil { + _ = appendFile.Close() + t.Fatal(err) + } + if err := appendFile.Sync(); err != nil { + _ = appendFile.Close() + t.Fatal(err) + } + if _, err := unix.FcntlInt(appendFile.Fd(), unix.F_FULLFSYNC, 0); err != nil { + _ = appendFile.Close() + t.Fatalf("F_FULLFSYNC: %v", err) + } + if err := appendFile.Close(); err != nil { + t.Fatal(err) + } + read, err = os.ReadFile(target) + want := append(append([]byte(nil), source...), tail...) + if err != nil || string(read) != string(want) { + t.Fatalf("append read differs: %q err=%v", read, err) + } + randomFile, err := os.OpenFile(target, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := randomFile.WriteAt([]byte("PATCH"), 2); err != nil { + _ = randomFile.Close() + t.Fatal(err) + } + if err := randomFile.Truncate(int64(len(want) - 3)); err != nil { + _ = randomFile.Close() + t.Fatal(err) + } + if err := randomFile.Sync(); err != nil { + _ = randomFile.Close() + t.Fatal(err) + } + if err := randomFile.Close(); err != nil { + t.Fatal(err) + } + info, err := os.Stat(target) + if err != nil || info.Size() != int64(len(want)-3) { + t.Fatalf("stat after truncate: %#v err=%v", info, err) + } + if err := os.Rename(target, filepath.Join(mountPoint, "renamed.jsonl")); err == nil { + t.Fatal("rename should fail closed until a real Codex trace requires it") + } + mutated := append([]byte(nil), want...) + copy(mutated[2:], []byte("PATCH")) + mutated = mutated[:len(mutated)-3] + read, err = os.ReadFile(target) + if err != nil || !bytes.Equal(read, mutated) { + t.Fatalf("random-write/truncate read differs: %q err=%v", read, err) + } + stopMount() + waitForRealUnmount(t, mountPoint) + + stopRemount := startRealMount(t, mountPoint, filesystem) + read, err = os.ReadFile(target) + if err != nil || !bytes.Equal(read, mutated) { + t.Fatalf("remount read differs: %q err=%v", read, err) + } + stopRemount() + waitForRealUnmount(t, mountPoint) +} + +func TestRealFuseTReadAndFsyncPerformance(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + line := []byte("{\"payload\":\"0123456789abcdef0123456789abcdef0123456789abcdef\"}\n") + source := bytes.Repeat(line, (32<<20)/len(line)+1) + source = source[:32<<20] + nativeReadPath := filepath.Join(root, "native-read.jsonl") + if err := os.WriteFile(nativeReadPath, source, 0o600); err != nil { + t.Fatal(err) + } + + managed := mountSessionFixture(t, "darwin-performance", source) + filesystem := New() + if err := filesystem.AddSession("darwin-performance", managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, "darwin-performance.jsonl") + + nativeRead := bestSequentialRead(t, nativeReadPath, int64(len(source)), 3) + fuseRead := bestSequentialRead(t, target, int64(len(source)), 3) + ratio := fuseRead / nativeRead + if fuseRead < 1024 || ratio < 0.25 { + t.Fatalf("FUSE-T read %.2f MiB/s is %.1f%% of APFS %.2f MiB/s", fuseRead, ratio*100, nativeRead) + } + + nativeAppendPath := filepath.Join(root, "native-append.jsonl") + if err := os.WriteFile(nativeAppendPath, []byte("{\"record\":0}\n"), 0o600); err != nil { + t.Fatal(err) + } + nativeP95 := appendFsyncP95(t, nativeAppendPath, 100) + fuseP95 := appendFsyncP95(t, target, 100) + ceiling := 50 * time.Millisecond + if relative := nativeP95 * 5; relative > ceiling { + ceiling = relative + } + if fuseP95 > ceiling { + t.Fatalf("FUSE-T append+fsync p95 %s exceeds ceiling %s; APFS p95=%s", fuseP95, ceiling, nativeP95) + } + t.Logf("FUSE-T read=%.2f MiB/s APFS=%.2f MiB/s ratio=%.1f%% append_fsync_p95=%s APFS_p95=%s", fuseRead, nativeRead, ratio*100, fuseP95, nativeP95) + + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func bestSequentialRead(t *testing.T, path string, size int64, rounds int) float64 { + t.Helper() + best := float64(0) + for range rounds { + started := time.Now() + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + readBytes, copyErr := io.Copy(io.Discard, file) + closeErr := file.Close() + if copyErr != nil || closeErr != nil || readBytes != size { + t.Fatalf("sequential read %q bytes=%d copy=%v close=%v", path, readBytes, copyErr, closeErr) + } + throughput := float64(readBytes) / (1024 * 1024) / time.Since(started).Seconds() + if throughput > best { + best = throughput + } + } + return best +} + +func appendFsyncP95(t *testing.T, path string, rounds int) time.Duration { + t.Helper() + file, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + durations := make([]time.Duration, 0, rounds) + for index := range rounds { + started := time.Now() + if _, err := fmt.Fprintf(file, "{\"append\":%d}\n", index); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + t.Fatal(err) + } + durations = append(durations, time.Since(started)) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + return durations[(len(durations)*95+99)/100-1] +} + +func TestRealFuseManagedStaleTailOffsetsPreserveJSONL(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + source := []byte("{\"record\":0}\n") + managed := mountSessionFixture(t, "stale-tail", source) + filesystem := New() + if err := filesystem.AddSession("stale-tail", managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + var traceMu sync.Mutex + var trace []string + options := HostOptions{ + MountPoint: mountPoint, + Filesystem: filesystem, + Foreground: true, + OperationRecorder: func(operation string) { + traceMu.Lock() + defer traceMu.Unlock() + trace = append(trace, operation) + }, + } + stopMount := startRealMountWithOptions(t, options) + var mountStat unix.Statfs_t + if err := unix.Statfs(mountPoint, &mountStat); err != nil { + t.Fatal(err) + } + if mountStat.Flags&unix.MNT_SYNCHRONOUS == 0 { + t.Fatal("real FUSE-T mount was reported healthy before synchronous I/O was enabled") + } + target := filepath.Join(mountPoint, "stale-tail.jsonl") + file, err := os.OpenFile(target, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + first := []byte("{\"record\":1}\n") + second := []byte("{\"record\":2}\n") + staleEOF := int64(len(source)) + if _, err := file.WriteAt(first, staleEOF); err != nil { + _ = file.Close() + t.Fatal(err) + } + if _, err := file.WriteAt(second, staleEOF); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + want := append(append(append([]byte(nil), source...), first...), second...) + got, err := os.ReadFile(target) + if err != nil || !bytes.Equal(got, want) { + traceMu.Lock() + defer traceMu.Unlock() + t.Fatalf("stale-tail visible bytes differ: got=%q want=%q err=%v trace=%q", got, want, err, trace) + } + if state := managed.State(); state.BackingPath != "" { + t.Fatalf("stale-tail writes created backing %q", state.BackingPath) + } + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func TestRealFuseCanonicalNativeToManagedCutover(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("archived_sessions", "rollout-cutover.jsonl") + nativePath := filepath.Join(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + source := []byte("{\"cutover\":true}\n") + if err := os.WriteFile(nativePath, source, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, route) + + managed := mountSessionFixture(t, "cutover", source) + if err := filesystem.UpsertSessionAt("cutover", "/"+filepath.ToSlash(route), managed); err != nil { + t.Fatal(err) + } + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + waitForRealFile(t, target, source) + + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func TestRealFuseCanonicalManagedRemovalRevealsNative(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("sessions", "2026", "07", "14", "rollout-rollback.jsonl") + nativePath := filepath.Join(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + managedBytes := []byte("{\"managed\":true}\n") + nativeBytes := []byte("{\"native\":true}\n") + managed := mountSessionFixture(t, "rollback", managedBytes) + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + if err := filesystem.AddSessionAt("rollback", "/"+filepath.ToSlash(route), managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, route) + waitForRealFile(t, target, managedBytes) + + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + if err := filesystem.RemoveSession("rollback"); err != nil { + t.Fatal(err) + } + waitForRealFileTransition(t, target, nativeBytes) + + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func TestRealFuseCanonicalManagedRemovalCanBeReaddedAtSamePath(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("sessions", "2026", "07", "14", "rollout-republish.jsonl") + nativePath := filepath.Join(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + firstManagedBytes := []byte("{\"generation\":1}\n") + nativeBytes := []byte("{\"native\":true}\n") + digest := sha256.Sum256(firstManagedBytes) + digestHex := hex.EncodeToString(digest[:]) + managedRoot := filepath.Join(root, "managed") + managedNativePath := filepath.Join(root, "managed-native.jsonl") + if err := os.WriteFile(managedNativePath, firstManagedBytes, 0o600); err != nil { + t.Fatal(err) + } + manifest := fold.Manifest{ + Version: fold.ManifestVersion, Kind: fold.ManifestKind, + Session: fold.ManifestSession{ID: "republish", RolloutPath: managedNativePath}, + Source: fold.ManifestSource{Bytes: int64(len(firstManagedBytes)), SHA256: digestHex}, + Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: digestHex, RawBytes: int64(len(firstManagedBytes))}}}, + } + managedOptions := vfs.SessionOptions{ + Root: managedRoot, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, + Reader: fuseFixtureReader{digestHex: firstManagedBytes}, + NativeSnapshot: vfs.NativeFile{ + Path: managedNativePath, Bytes: int64(len(firstManagedBytes)), SHA256: digestHex, + }, + } + firstManaged, err := vfs.OpenSession(context.Background(), managedOptions) + if err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + if err := filesystem.AddSessionAt("republish", "/"+filepath.ToSlash(route), firstManaged); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + canonicalHome := filepath.Join(root, "home") + if err := os.MkdirAll(canonicalHome, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(mountPoint, "sessions"), filepath.Join(canonicalHome, "sessions")); err != nil { + t.Fatal(err) + } + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(canonicalHome, route) + waitForRealFile(t, target, firstManagedBytes) + + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + stateDirectory := filepath.Dir(firstManaged.State().DeltaPath) + retiredStateDirectory := stateDirectory + ".retired" + if err := os.Rename(stateDirectory, retiredStateDirectory); err != nil { + t.Fatal(err) + } + if _, err := os.ReadFile(target); err == nil { + t.Fatal("managed read unexpectedly succeeded after its state directory was retired") + } + if err := filesystem.RemoveSession("republish"); err != nil { + t.Fatal(err) + } + waitForRealFileTransition(t, target, nativeBytes) + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + waitForRealFileMissing(t, target) + if err := os.Rename(retiredStateDirectory, stateDirectory); err != nil { + t.Fatal(err) + } + republished, err := vfs.RepublishSessionState(filepath.Join(stateDirectory, "state.json")) + if err != nil { + t.Fatal(err) + } + if republished.Generation != 2 { + t.Fatalf("republished generation = %d, want 2", republished.Generation) + } + secondManaged, err := vfs.OpenSession(context.Background(), managedOptions) + if err != nil { + t.Fatal(err) + } + + if err := filesystem.UpsertSessionAt("republish", "/"+filepath.ToSlash(route), secondManaged); err != nil { + t.Fatal(err) + } + waitForRealFile(t, target, firstManagedBytes) + + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func TestRealFuseCanonicalNativePreferenceNeverLosesPath(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("sessions", "2026", "07", "14", "rollout-native-preference.jsonl") + nativePath := filepath.Join(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + managedBytes := []byte("{\"managed\":true}\n") + nativeBytes := []byte("{\"native\":true}\n") + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + if err := filesystem.AddSessionAt("native-preference", "/"+filepath.ToSlash(route), mountSessionFixture(t, "native-preference", managedBytes)); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, route) + waitForRealFile(t, target, managedBytes) + + if err := filesystem.PreferNativeSession("native-preference"); err != nil { + t.Fatal(err) + } + waitForRealFileTransition(t, target, nativeBytes) + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + waitForRealFileTransition(t, target, managedBytes) + if err := os.WriteFile(nativePath, nativeBytes, 0o600); err != nil { + t.Fatal(err) + } + waitForRealFileTransition(t, target, nativeBytes) + + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func TestRealFuseCanonicalNativeAppendTransactionSurvivesRestart(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("sessions", "2026", "07", "16", "rollout-native-transaction.jsonl") + nativePath := filepath.Join(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(mountPoint, route) + var traceMu sync.Mutex + var trace []string + start := func() func() { + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + if err := filesystem.RecoverNativeAppendTransactions(); err != nil { + t.Fatal(err) + } + return startRealMountWithOptions(t, HostOptions{ + MountPoint: mountPoint, Filesystem: filesystem, Foreground: true, + OperationRecorder: func(operation string) { + traceMu.Lock() + trace = append(trace, operation) + traceMu.Unlock() + }, + }) + } + + stopMount := start() + large := append([]byte("{\"large\":\""), bytes.Repeat([]byte("x"), 1<<20)...) + large = append(large, []byte("\"}\n")...) + file, err := os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if n, err := file.Write(large); err != nil || n != len(large) { + _ = file.Close() + t.Fatalf("write large append: n=%d err=%v", n, err) + } + want := append(append([]byte(nil), base...), large...) + backingAfterWrite, err := os.ReadFile(nativePath) + if err != nil || (!bytes.Equal(backingAfterWrite, base) && !bytes.Equal(backingAfterWrite, want)) { + _ = file.Close() + traceMu.Lock() + currentTrace := strings.Join(trace, "\n") + traceMu.Unlock() + t.Fatalf("backing exposed a partial transaction: bytes=%d err=%v trace=%s", len(backingAfterWrite), err, currentTrace) + } + visible, err := os.ReadFile(target) + if err != nil || !bytes.Equal(visible, want) { + _ = file.Close() + t.Fatalf("pending mounted view: bytes=%d err=%v want=%d", len(visible), err, len(want)) + } + if err := file.Sync(); err != nil { + _ = file.Close() + t.Fatalf("fsync large append: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("close large append: %v", err) + } + assertNativeBytes(t, nativePath, want) + + stopMount() + waitForRealUnmount(t, mountPoint) + stopMount = start() + waitForRealFile(t, target, want) + + afterRestart := []byte("{\"after_restart\":true}\n") + file, err = os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if n, err := file.Write(afterRestart); err != nil || n != len(afterRestart) { + _ = file.Close() + t.Fatalf("write after restart: n=%d err=%v", n, err) + } + if err := file.Close(); err != nil { + t.Fatalf("release commit after restart: %v", err) + } + want = append(want, afterRestart...) + assertNativeBytes(t, nativePath, want) + if !completeJSONL(want) { + t.Fatal("real FUSE append produced invalid JSONL") + } + + traceMu.Lock() + joined := strings.Join(trace, "\n") + traceMu.Unlock() + for _, marker := range []string{"open kind=session", "write kind=session", "fsync", "flush", "release"} { + if !strings.Contains(joined, marker) { + t.Fatalf("real append trace missing %q: %s", marker, joined) + } + } + if writes := strings.Count(joined, "write kind=session"); writes < 30 { + t.Fatalf("large append was not exercised as split FUSE writes: writes=%d", writes) + } + for _, entry := range strings.Split(joined, "\n") { + if strings.Contains(entry, "kind=session") && strings.Contains(entry, "result=-") { + t.Fatalf("session operation failed in real append trace: %s", entry) + } + } + for _, entry := range strings.Split(joined, "\n") { + if strings.Contains(entry, "kind=appledouble") && strings.Contains(entry, "result=-5") { + t.Fatalf("AppleDouble metadata was routed through JSONL validation: %s", entry) + } + } + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func TestRealFuseMountCanonicalManagedRename(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + source := []byte("canonical-session\n") + managed := mountSessionFixture(t, "fixture", source) + nativeRoot := filepath.Join(root, "native") + nativeActiveDirectory := filepath.Join(nativeRoot, "sessions", "2026", "07", "12") + if err := os.MkdirAll(nativeActiveDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(nativeRoot, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + filename := "rollout-2026-07-12T14-28-28-fixture.jsonl" + archivedPath := "/archived_sessions/" + filename + if err := filesystem.AddSessionAt("fixture", archivedPath, managed); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + var recordedMu sync.Mutex + var recorded []string + stopMount := startRealMountWithOptions(t, HostOptions{ + MountPoint: mountPoint, Filesystem: filesystem, Foreground: true, + OperationRecorder: func(operation string) { + recordedMu.Lock() + recorded = append(recorded, operation) + recordedMu.Unlock() + }, + }) + archivedTarget := filepath.Join(mountPoint, "archived_sessions", filename) + activeTarget := filepath.Join(mountPoint, "sessions", "2026", "07", "12", filename) + attributeName := "com.codexfold.test" + attributeValue := []byte("persistent-metadata") + if err := unix.Setxattr(archivedTarget, attributeName, attributeValue, 0); err != nil { + t.Fatalf("set managed xattr: %v", err) + } + archivedSidecar := filepath.Join(nativeRoot, "archived_sessions", "._"+filename) + activeSidecar := filepath.Join(nativeActiveDirectory, "._"+filename) + sidecar, err := os.ReadFile(archivedSidecar) + if err != nil || !bytes.Contains(sidecar, []byte(attributeName)) || !bytes.Contains(sidecar, attributeValue) { + t.Fatalf("AppleDouble sidecar did not preserve xattr: bytes=%d err=%v", len(sidecar), err) + } + if err := os.Rename(archivedTarget, activeTarget); err != nil { + t.Fatalf("rename canonical managed session: %v", err) + } + if _, err := os.Stat(archivedTarget); !os.IsNotExist(err) { + t.Fatalf("archived path remained after rename: %v", err) + } + got, err := os.ReadFile(activeTarget) + if err != nil || !bytes.Equal(got, source) { + t.Fatalf("active managed bytes differ: got=%q err=%v", got, err) + } + if _, err := os.Stat(archivedSidecar); !os.IsNotExist(err) { + t.Fatalf("archived AppleDouble sidecar remained after rename: %v", err) + } + movedSidecar, err := os.ReadFile(activeSidecar) + if err != nil || !bytes.Contains(movedSidecar, []byte(attributeName)) || !bytes.Contains(movedSidecar, attributeValue) { + t.Fatalf("active AppleDouble sidecar lost xattr: bytes=%d err=%v", len(movedSidecar), err) + } + if err := os.Rename(activeTarget, archivedTarget); err != nil { + t.Fatalf("rename canonical managed session back: %v", err) + } + if _, err := os.Stat(activeTarget); !os.IsNotExist(err) { + t.Fatalf("active path remained after reverse rename: %v", err) + } + if _, err := os.Stat(activeSidecar); !os.IsNotExist(err) { + t.Fatalf("active AppleDouble sidecar remained after reverse rename: %v", err) + } + restoredSidecar, err := os.ReadFile(archivedSidecar) + if err != nil || !bytes.Contains(restoredSidecar, []byte(attributeName)) || !bytes.Contains(restoredSidecar, attributeValue) { + t.Fatalf("restored AppleDouble sidecar lost xattr: bytes=%d err=%v", len(restoredSidecar), err) + } + recordedMu.Lock() + joined := strings.Join(recorded, ",") + recordedMu.Unlock() + for _, operation := range []string{"getattr", "rename", "open", "read", "release"} { + if !strings.Contains(joined, operation) { + t.Fatalf("operation trace missing %q: %s", operation, joined) + } + } + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func startRealMount(t *testing.T, mountPoint string, filesystem *Filesystem) func() { + return startRealMountWithOptions(t, HostOptions{MountPoint: mountPoint, Filesystem: filesystem, Foreground: true}) +} + +func startRealMountWithOptions(t *testing.T, options HostOptions) func() { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + mountDone := make(chan error, 1) + go func() { + mountDone <- Mount(ctx, options) + }() + var stopOnce sync.Once + stopMount := func() { + stopOnce.Do(func() { + cancel() + select { + case err := <-mountDone: + if err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("mount shutdown: %v", err) + } + case <-time.After(10 * time.Second): + t.Error("mount did not stop after cancellation") + } + }) + } + t.Cleanup(stopMount) + waitForRealMount(t, options.MountPoint, mountDone) + return stopMount +} + +func waitForRealMount(t *testing.T, mountPoint string, mountDone <-chan error) { + t.Helper() + deadline := time.Now().Add(20 * time.Second) + var lastProbeErr error + for time.Now().Before(deadline) { + if err := service.ProbeMount(mountPoint); err == nil { + return + } else { + lastProbeErr = err + } + select { + case err := <-mountDone: + t.Fatalf("mount exited before becoming healthy: %v", err) + case <-time.After(100 * time.Millisecond): + } + } + t.Fatalf("FUSE mount did not become healthy: %v", lastProbeErr) +} + +func waitForRealUnmount(t *testing.T, mountPoint string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + var stat unix.Statfs_t + if err := unix.Statfs(mountPoint, &stat); err != nil || !sameRealMountPath(unix.ByteSliceToString(stat.Mntonname[:]), mountPoint) { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("FUSE mount remained active after shutdown") +} + +func sameRealMountPath(left string, right string) bool { + canonical := func(path string) string { + resolved, err := filepath.EvalSymlinks(path) + if err == nil { + return filepath.Clean(resolved) + } + return filepath.Clean(path) + } + return canonical(left) == canonical(right) +} + +func waitForRealFile(t *testing.T, path string, want []byte) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(path) + if err == nil { + if !bytes.Equal(data, want) { + t.Fatalf("hot-loaded file differs: got=%q want=%q", data, want) + } + return + } + if !os.IsNotExist(err) { + t.Fatalf("read hot-loaded file: %v", err) + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("hot-loaded file did not become visible") +} + +func waitForRealFileTransition(t *testing.T, path string, want []byte) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(path) + if err == nil && bytes.Equal(data, want) { + return + } + if err != nil && !os.IsNotExist(err) { + t.Fatalf("read transitioned file: %v", err) + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("managed file did not transition to native bytes") +} + +func waitForRealFileMissing(t *testing.T, path string) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(path); os.IsNotExist(err) { + return + } else if err != nil { + t.Fatalf("stat transitioned file: %v", err) + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("canonical file did not become absent") +} + +type fuseFixtureReader map[string][]byte + +func (r fuseFixtureReader) ReadAt(_ context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { + data := r[ref.SHA256] + if offset >= int64(len(data)) { + return 0, io.EOF + } + n := copy(destination, data[offset:]) + if n < len(destination) { + return n, io.EOF + } + return n, nil +} diff --git a/internal/mountfs/fuse_provider_darwin.go b/internal/mountfs/fuse_provider_darwin.go new file mode 100644 index 0000000..3963243 --- /dev/null +++ b/internal/mountfs/fuse_provider_darwin.go @@ -0,0 +1,45 @@ +//go:build darwin && fuse && cgo + +package mountfs + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +const darwinFUSEtLibraryPath = "/usr/local/lib/libfuse-t.dylib" + +var darwinHigherPriorityFUSELibraries = []string{ + "/usr/local/lib/libfuse.2.dylib", + "/usr/local/lib/libosxfuse.2.dylib", +} + +func validateFuseProvider() error { + return validateDarwinFUSEProviderPaths(darwinFUSEtLibraryPath, darwinHigherPriorityFUSELibraries) +} + +func validateDarwinFUSEProviderPaths(fuseTPath string, higherPriority []string) error { + for _, path := range higherPriority { + if _, err := os.Stat(path); err == nil { + return fmt.Errorf("unsupported macOS FUSE library %q would take precedence over FUSE-T", path) + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect competing macOS FUSE library %q: %w", path, err) + } + } + + resolved, err := filepath.EvalSymlinks(fuseTPath) + if err != nil { + return fmt.Errorf("FUSE-T library %q is unavailable: %w", fuseTPath, err) + } + info, err := os.Stat(resolved) + if err != nil { + return fmt.Errorf("inspect FUSE-T library %q: %w", resolved, err) + } + if !info.Mode().IsRegular() || !strings.Contains(filepath.Base(resolved), "libfuse-t") { + return fmt.Errorf("FUSE-T library resolves to an unexpected file %q", resolved) + } + return nil +} diff --git a/internal/mountfs/fuse_provider_darwin_test.go b/internal/mountfs/fuse_provider_darwin_test.go new file mode 100644 index 0000000..e36c713 --- /dev/null +++ b/internal/mountfs/fuse_provider_darwin_test.go @@ -0,0 +1,41 @@ +//go:build darwin && fuse && cgo + +package mountfs + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestValidateDarwinFUSEProviderRequiresFUSEt(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "libfuse-t-1.2.7.dylib") + if err := os.WriteFile(target, []byte("fixture"), 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "libfuse-t.dylib") + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + + if err := validateDarwinFUSEProviderPaths(link, nil); err != nil { + t.Fatalf("valid FUSE-T layout rejected: %v", err) + } + + competitor := filepath.Join(root, "libfuse.2.dylib") + if err := os.WriteFile(competitor, []byte("fixture"), 0o600); err != nil { + t.Fatal(err) + } + if err := validateDarwinFUSEProviderPaths(link, []string{competitor}); err == nil || !strings.Contains(err.Error(), "take precedence") { + t.Fatalf("competing FUSE library was not rejected: %v", err) + } +} + +func TestValidateDarwinFUSEProviderRejectsMissingFUSEt(t *testing.T) { + err := validateDarwinFUSEProviderPaths(filepath.Join(t.TempDir(), "libfuse-t.dylib"), nil) + if err == nil || !strings.Contains(err.Error(), "unavailable") { + t.Fatalf("missing FUSE-T library was not rejected: %v", err) + } +} diff --git a/internal/mountfs/fuse_provider_other.go b/internal/mountfs/fuse_provider_other.go new file mode 100644 index 0000000..5f06e01 --- /dev/null +++ b/internal/mountfs/fuse_provider_other.go @@ -0,0 +1,5 @@ +//go:build (linux && fuse && fuse3 && cgo) || (windows && winfsp) + +package mountfs + +func validateFuseProvider() error { return nil } diff --git a/internal/mountfs/host.go b/internal/mountfs/host.go new file mode 100644 index 0000000..ad54a5c --- /dev/null +++ b/internal/mountfs/host.go @@ -0,0 +1,64 @@ +package mountfs + +import ( + "context" + "errors" + "fmt" + "os" +) + +var ErrPrerequisite = errors.New("FUSE host prerequisite is unavailable in this build") + +type HostOptions struct { + MountPoint string + Filesystem *Filesystem + Foreground bool + OperationRecorder func(string) + BuildSHA256 string +} + +func Mount(ctx context.Context, options HostOptions) error { + if options.MountPoint == "" || options.Filesystem == nil { + return errors.New("mount point and filesystem are required") + } + if !Available() { + return ErrPrerequisite + } + if err := prepareMountPoint(options.MountPoint); err != nil { + return err + } + return mountHost(ctx, options) +} + +func prepareMountPoint(path string) error { + if err := recoverStaleMount(path); err != nil { + return fmt.Errorf("recover stale mount: %w", err) + } + info, err := os.Lstat(path) + if os.IsNotExist(err) { + if err := os.MkdirAll(path, 0o700); err != nil { + return fmt.Errorf("create mount backing directory: %w", err) + } + info, err = os.Lstat(path) + } + if err != nil { + return fmt.Errorf("inspect mount backing directory: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 { + return errors.New("mount backing path must not be a symlink") + } + if !info.IsDir() { + return errors.New("mount backing path is not a directory") + } + entries, err := os.ReadDir(path) + if err != nil { + return fmt.Errorf("inspect mount backing contents: %w", err) + } + if len(entries) != 0 { + return errors.New("mount backing directory must be empty") + } + if err := os.Chmod(path, 0o500); err != nil { + return fmt.Errorf("seal mount backing directory: %w", err) + } + return nil +} diff --git a/internal/mountfs/host_cgofuse.go b/internal/mountfs/host_cgofuse.go new file mode 100644 index 0000000..5cfb77f --- /dev/null +++ b/internal/mountfs/host_cgofuse.go @@ -0,0 +1,583 @@ +//go:build (darwin && fuse && cgo) || (linux && fuse && fuse3 && cgo) || (windows && winfsp) + +package mountfs + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "syscall" + "time" + + "github.com/samekind/codexfold/internal/buildid" + "github.com/samekind/codexfold/internal/mountid" + "github.com/winfsp/cgofuse/fuse" +) + +type fuseFilesystem struct { + fuse.FileSystemBase + core *Filesystem + recorder func(string) + mountIdentity []byte + statRoot string + mountReady atomic.Bool +} + +const healthHandle = ^uint64(0) - 1 + +func Available() bool { return true } + +func (f *fuseFilesystem) Getattr(name string, stat *fuse.Stat_t, _ uint64) int { + if cleanPath(name) == "/"+mountid.Path { + if !f.mountReady.Load() { + f.recordResult("getattr", name, -int(syscall.ENOENT)) + return -int(syscall.ENOENT) + } + stat.Mode = syscall.S_IFREG | 0o400 + stat.Size = int64(len(f.mountIdentity)) + stat.Nlink = 1 + stat.Blksize = 4096 + stat.Blocks = (stat.Size + 511) / 512 + stat.Uid, stat.Gid, _ = fuse.Getcontext() + f.recordResult("getattr", name, 0) + return 0 + } + attribute, errno := f.core.Getattr(name) + if errno != 0 { + f.recordResult("getattr", name, -int(errno)) + return -int(errno) + } + stat.Mode = attribute.Mode + stat.Size = attribute.Size + stat.Nlink = 1 + stat.Blksize = 4096 + stat.Blocks = (attribute.Size + 511) / 512 + stat.Mtim = fuse.NewTimespec(attribute.ModTime) + stat.Ctim = stat.Mtim + stat.Atim = stat.Mtim + stat.Uid, stat.Gid, _ = fuse.Getcontext() + f.recordResult("getattr", name, 0) + return 0 +} + +func (f *fuseFilesystem) Statfs(name string, stat *fuse.Statfs_t) int { + f.core.mu.RLock() + root := f.core.nativeRoot + f.core.mu.RUnlock() + if root == "" { + root = f.statRoot + } + result := populateFilesystemStat(root, stat) + f.recordResult("statfs", name, result) + return result +} + +func (f *fuseFilesystem) Mknod(name string, _ uint32, _ uint64) int { + result := -int(syscall.ENOSYS) + f.recordResult("mknod", name, result) + return result +} + +func (f *fuseFilesystem) Opendir(name string) (int, uint64) { + f.record("opendir") + if _, errno := f.core.ReadDir(name); errno != 0 { + return -int(errno), ^uint64(0) + } + return 0, 0 +} + +func (f *fuseFilesystem) Readdir(name string, fill func(string, *fuse.Stat_t, int64) bool, _ int64, _ uint64) int { + f.record("readdir") + entries, errno := f.core.ReadDir(name) + if errno != 0 { + return -int(errno) + } + fill(".", nil, 0) + fill("..", nil, 0) + for _, entry := range entries { + if !fill(entry, nil, 0) { + break + } + } + return 0 +} + +func (f *fuseFilesystem) Open(name string, flags int) (int, uint64) { + if cleanPath(name) == "/"+mountid.Path { + if !f.mountReady.Load() { + return -int(syscall.ENOENT), ^uint64(0) + } + if flags&fuse.O_ACCMODE != fuse.O_RDONLY { + return -int(syscall.EPERM), ^uint64(0) + } + return 0, healthHandle + } + translated := translateOpenFlags(flags) + handle, errno := f.core.Open(name, translated) + if errno == syscall.EBUSY && writableSession(name, flags) { + deadline := time.Now().Add(250 * time.Millisecond) + for errno == syscall.EBUSY && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + handle, errno = f.core.Open(name, translated) + } + } + if errno != 0 { + result := -int(errno) + f.recordOpen("open", name, flags, translated, handle, result) + return result, ^uint64(0) + } + f.recordOpen("open", name, flags, translated, handle, 0) + return 0, handle +} + +func (f *fuseFilesystem) Create(name string, flags int, _ uint32) (int, uint64) { + translated := translateOpenFlags(flags) | os.O_CREATE + handle, errno := f.core.Open(name, translated) + if errno != 0 { + result := -int(errno) + f.recordOpen("create", name, flags, translated, handle, result) + return result, ^uint64(0) + } + f.recordOpen("create", name, flags, translated, handle, 0) + return 0, handle +} + +func (f *fuseFilesystem) OpenEx(name string, info *fuse.FileInfo_t) int { + result, handle := f.Open(name, info.Flags) + if result == 0 { + info.Fh = handle + info.DirectIo = writableSession(name, info.Flags) + f.record(fmt.Sprintf("open_config kind=%s handle=%d direct_io=%t", operationKind(name), handle, info.DirectIo)) + } + return result +} + +func (f *fuseFilesystem) CreateEx(name string, _ uint32, info *fuse.FileInfo_t) int { + result, handle := f.Create(name, info.Flags, 0o600) + if result == 0 { + info.Fh = handle + info.DirectIo = writableSession(name, info.Flags) + f.record(fmt.Sprintf("create_config kind=%s handle=%d direct_io=%t", operationKind(name), handle, info.DirectIo)) + } + return result +} + +func (f *fuseFilesystem) Read(name string, destination []byte, offset int64, handle uint64) int { + if handle == healthHandle { + if offset < 0 || offset >= int64(len(f.mountIdentity)) { + f.recordIO("read", name, handle, offset, len(destination), 0) + return 0 + } + n := copy(destination, f.mountIdentity[offset:]) + f.recordIO("read", name, handle, offset, len(destination), n) + return n + } + n, errno := f.core.Read(handle, destination, offset) + if errno != 0 { + result := -int(errno) + f.recordIO("read", name, handle, offset, len(destination), result) + return result + } + f.recordIO("read", name, handle, offset, len(destination), n) + return n +} + +func (f *fuseFilesystem) Write(name string, data []byte, offset int64, handle uint64) int { + n, errno := f.core.Write(handle, data, offset) + if errno != 0 { + result := -int(errno) + f.recordIO("write", name, handle, offset, len(data), result) + return result + } + f.recordIO("write", name, handle, offset, len(data), n) + return n +} + +func (f *fuseFilesystem) Truncate(name string, size int64, handle uint64) int { + var errno syscall.Errno + if handle == 0 || handle == ^uint64(0) { + errno = f.core.TruncatePath(name, size) + } else { + errno = f.core.Truncate(handle, size) + } + result := -int(errno) + f.record(fmt.Sprintf("truncate kind=%s handle=%d size=%d result=%d", operationKind(name), handle, size, result)) + return result +} + +func (f *fuseFilesystem) Flush(name string, handle uint64) int { + if handle == healthHandle { + f.recordHandleResult("flush", name, handle, 0) + return 0 + } + result := -int(f.core.Flush(handle)) + f.recordHandleResult("flush", name, handle, result) + return result +} + +func (f *fuseFilesystem) Fsync(name string, dataOnly bool, handle uint64) int { + if handle == healthHandle { + f.record(fmt.Sprintf("fsync kind=%s handle=%d datasync=%t result=0", operationKind(name), handle, dataOnly)) + return 0 + } + result := -int(f.core.Fsync(handle)) + f.record(fmt.Sprintf("fsync kind=%s handle=%d datasync=%t result=%d", operationKind(name), handle, dataOnly, result)) + return result +} + +func (f *fuseFilesystem) Release(name string, handle uint64) int { + if handle == healthHandle { + f.recordHandleResult("release", name, handle, 0) + return 0 + } + result := -int(f.core.Release(handle)) + f.recordHandleResult("release", name, handle, result) + return result +} + +func (f *fuseFilesystem) Mkdir(name string, mode uint32) int { + result := -int(f.core.Mkdir(name, mode)) + f.recordResult("mkdir", name, result) + return result +} + +func (f *fuseFilesystem) Rmdir(name string) int { + result := -int(syscall.ENOSYS) + f.recordResult("rmdir", name, result) + return result +} + +func (f *fuseFilesystem) Link(oldName string, _ string) int { + result := -int(syscall.ENOSYS) + f.recordResult("link", oldName, result) + return result +} + +func (f *fuseFilesystem) Symlink(_ string, newName string) int { + result := -int(syscall.ENOSYS) + f.recordResult("symlink", newName, result) + return result +} + +func (f *fuseFilesystem) Readlink(name string) (int, string) { + result := -int(syscall.ENOSYS) + f.recordResult("readlink", name, result) + return result, "" +} + +func (f *fuseFilesystem) Rename(oldName string, newName string) int { + result := -int(f.core.Rename(oldName, newName)) + f.recordResult("rename", oldName, result) + return result +} + +func (f *fuseFilesystem) Unlink(name string) int { + result := -int(f.core.Unlink(name)) + f.recordResult("unlink", name, result) + return result +} + +func (f *fuseFilesystem) Access(name string, _ uint32) int { + f.record("access") + if cleanPath(name) == "/"+mountid.Path { + return 0 + } + _, errno := f.core.Getattr(name) + return -int(errno) +} + +func (f *fuseFilesystem) Chmod(name string, mode uint32) int { + path, managed, errc := f.metadataPath(name) + if errc != 0 { + f.recordResult("chmod", name, errc) + return errc + } + result := 0 + if !managed { + result = unixResult(os.Chmod(path, os.FileMode(mode)&os.ModePerm)) + } + f.recordResult("chmod", name, result) + return result +} + +func (f *fuseFilesystem) Chown(name string, uid uint32, gid uint32) int { + path, managed, errc := f.metadataPath(name) + if errc != 0 { + f.recordResult("chown", name, errc) + return errc + } + result := 0 + if !managed { + result = unixResult(os.Chown(path, int(uid), int(gid))) + } + f.recordResult("chown", name, result) + return result +} + +func (f *fuseFilesystem) Utimens(name string, times []fuse.Timespec) int { + path, managed, errc := f.metadataPath(name) + if errc != 0 { + f.recordResult("utimens", name, errc) + return errc + } + result := 0 + if !managed { + if len(times) != 2 { + result = -int(syscall.EINVAL) + } else { + result = setFileTimes(path, times) + } + } + f.recordResult("utimens", name, result) + return result +} + +func (f *fuseFilesystem) Setxattr(name string, attribute string, value []byte, flags int) int { + f.record("setxattr") + path, errc := f.xattrPath(name, true) + if errc != 0 { + return errc + } + return setExtendedAttribute(path, attribute, value, flags) +} + +func (f *fuseFilesystem) Getxattr(name string, attribute string) (int, []byte) { + f.record("getxattr") + path, errc := f.xattrPath(name, false) + if errc != 0 { + return errc, nil + } + return getExtendedAttribute(path, attribute) +} + +func (f *fuseFilesystem) Listxattr(name string, fill func(string) bool) int { + f.record("listxattr") + path, errc := f.xattrPath(name, false) + if errc != 0 { + return errc + } + result, attributes := listExtendedAttributes(path) + if result != 0 { + return result + } + for _, attribute := range attributes { + if attribute != "" && !fill(attribute) { + break + } + } + return 0 +} + +func (f *fuseFilesystem) Removexattr(name string, attribute string) int { + f.record("removexattr") + path, errc := f.xattrPath(name, false) + if errc != 0 { + return errc + } + return removeExtendedAttribute(path, attribute) +} + +func (f *fuseFilesystem) xattrPath(name string, create bool) (string, int) { + cleaned := cleanPath(name) + if _, _, errno := f.core.sessionForPath(cleaned); errno == 0 { + f.core.mu.RLock() + root := f.core.nativeRoot + f.core.mu.RUnlock() + if root == "" { + return "", -int(syscall.ENOTSUP) + } + carrier := managedXattrCarrier(root, cleaned) + if create { + if err := os.MkdirAll(filepath.Dir(carrier), 0o700); err != nil { + return "", unixResult(err) + } + file, err := os.OpenFile(carrier, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return "", unixResult(err) + } + if err := file.Close(); err != nil { + return "", unixResult(err) + } + } + return carrier, 0 + } + if native, ok := f.core.nativePath(cleaned); ok { + return native, 0 + } + return "", -int(syscall.ENOENT) +} + +func (f *fuseFilesystem) metadataPath(name string) (string, bool, int) { + cleaned := cleanPath(name) + if _, _, errno := f.core.sessionForPath(cleaned); errno == 0 { + return "", true, 0 + } + if native, ok := f.core.nativePath(cleaned); ok { + return native, false, 0 + } + return "", false, -int(syscall.ENOENT) +} + +func unixResult(err error) int { + if err == nil { + return 0 + } + var errno syscall.Errno + if errors.As(err, &errno) { + return -int(errno) + } + return -int(syscall.EIO) +} + +func (f *fuseFilesystem) record(operation string) { + if f.recorder != nil { + f.recorder(operation) + } +} + +func (f *fuseFilesystem) recordResult(operation string, name string, result int) { + f.record(fmt.Sprintf("%s kind=%s result=%d", operation, operationKind(name), result)) +} + +func (f *fuseFilesystem) recordOpen(operation string, name string, flags int, translated int, handle uint64, result int) { + f.record(fmt.Sprintf("%s kind=%s flags=%#x translated=%#x handle=%d result=%d", operation, operationKind(name), flags, translated, handle, result)) +} + +func (f *fuseFilesystem) recordIO(operation string, name string, handle uint64, offset int64, bytes int, result int) { + f.record(fmt.Sprintf("%s kind=%s handle=%d offset=%d bytes=%d result=%d", operation, operationKind(name), handle, offset, bytes, result)) +} + +func (f *fuseFilesystem) recordHandleResult(operation string, name string, handle uint64, result int) { + f.record(fmt.Sprintf("%s kind=%s handle=%d result=%d", operation, operationKind(name), handle, result)) +} + +func operationKind(name string) string { + kind := "other" + base := filepath.Base(name) + if strings.HasPrefix(base, "._") { + kind = "appledouble" + } else if strings.HasSuffix(base, ".jsonl") { + kind = "session" + } + return kind +} + +func writableSession(name string, flags int) bool { + return operationKind(name) == "session" && flags&fuse.O_ACCMODE != fuse.O_RDONLY +} + +func translateOpenFlags(flags int) int { + translated := os.O_RDONLY + switch flags & fuse.O_ACCMODE { + case fuse.O_WRONLY: + translated = os.O_WRONLY + case fuse.O_RDWR: + translated = os.O_RDWR + } + if flags&fuse.O_APPEND != 0 { + translated |= os.O_APPEND + } + if flags&fuse.O_TRUNC != 0 { + translated |= os.O_TRUNC + } + if flags&fuse.O_CREAT != 0 { + translated |= os.O_CREATE + } + if flags&fuse.O_EXCL != 0 { + translated |= os.O_EXCL + } + return translated +} + +func mountHost(ctx context.Context, options HostOptions) (result error) { + defer func() { + if recovered := recover(); recovered != nil { + result = fmt.Errorf("%w: %v", ErrPrerequisite, recovered) + } + }() + if err := validateFuseProvider(); err != nil { + return fmt.Errorf("validate selected FUSE provider: %w", err) + } + buildSHA256 := options.BuildSHA256 + var err error + if buildSHA256 == "" { + buildSHA256, err = buildid.CurrentSHA256() + if err != nil { + return fmt.Errorf("hash mounted executable: %w", err) + } + } + identity, err := mountid.New(buildSHA256) + if err != nil { + return fmt.Errorf("generate mount identity: %w", err) + } + filesystem := &fuseFilesystem{ + core: options.Filesystem, + recorder: options.OperationRecorder, + mountIdentity: []byte(identity), + statRoot: filepath.Dir(options.MountPoint), + } + host := fuse.NewFileSystemHost(filesystem) + backing, err := prepareMountedBacking(options.MountPoint) + if err != nil { + return fmt.Errorf("prepare mount backing permissions: %w", err) + } + backingClosed := false + defer func() { + if !backingClosed { + _ = backing.Close() + } + }() + arguments := []string{"-o", "fsname=codexfold", "-o", "default_permissions", "-o", "attr_timeout=0", "-o", "entry_timeout=0", "-o", "negative_timeout=0"} + if options.Foreground { + arguments = append(arguments, "-f") + } + if runtime.GOOS == "darwin" { + arguments = append(arguments, "-o", "backend=nfs", "-o", "volname=CodexFold") + } + done := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + _ = host.Unmount() + case <-done: + } + }() + policyContext, cancelPolicy := context.WithCancel(ctx) + policyDone := make(chan error, 1) + go func() { + err := configureMountedFilesystem(policyContext, options.MountPoint) + if err == nil { + err = backing.Seal() + } + if err == nil { + filesystem.mountReady.Store(true) + } else { + _ = host.Unmount() + } + policyDone <- err + }() + mounted := host.Mount(options.MountPoint, arguments) + cancelPolicy() + policyErr := <-policyDone + backingErr := backing.Close() + backingClosed = true + close(done) + if err := ctx.Err(); err != nil { + return err + } + if !mounted { + return errors.New("FUSE host exited without mounting") + } + if policyErr != nil { + return fmt.Errorf("configure mounted filesystem: %w", policyErr) + } + if backingErr != nil { + return fmt.Errorf("seal unmounted backing directory: %w", backingErr) + } + return ctx.Err() +} diff --git a/internal/mountfs/host_platform_posix.go b/internal/mountfs/host_platform_posix.go new file mode 100644 index 0000000..c28e612 --- /dev/null +++ b/internal/mountfs/host_platform_posix.go @@ -0,0 +1,56 @@ +//go:build (darwin && fuse && cgo) || (linux && fuse && fuse3 && cgo) + +package mountfs + +import ( + "bytes" + + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/unix" +) + +func setFileTimes(path string, times []fuse.Timespec) int { + values := []unix.Timespec{{Sec: times[0].Sec, Nsec: times[0].Nsec}, {Sec: times[1].Sec, Nsec: times[1].Nsec}} + return unixResult(unix.UtimesNanoAt(unix.AT_FDCWD, path, values, 0)) +} + +func setExtendedAttribute(path string, attribute string, value []byte, flags int) int { + return unixResult(unix.Setxattr(path, attribute, value, flags)) +} + +func getExtendedAttribute(path string, attribute string) (int, []byte) { + size, err := unix.Getxattr(path, attribute, nil) + if err != nil { + return unixResult(err), nil + } + value := make([]byte, size) + n, err := unix.Getxattr(path, attribute, value) + if err != nil { + return unixResult(err), nil + } + return 0, value[:n] +} + +func listExtendedAttributes(path string) (int, []string) { + size, err := unix.Listxattr(path, nil) + if err != nil { + return unixResult(err), nil + } + buffer := make([]byte, size) + n, err := unix.Listxattr(path, buffer) + if err != nil { + return unixResult(err), nil + } + parts := bytes.Split(bytes.TrimRight(buffer[:n], "\x00"), []byte{0}) + attributes := make([]string, 0, len(parts)) + for _, part := range parts { + if len(part) != 0 { + attributes = append(attributes, string(part)) + } + } + return 0, attributes +} + +func removeExtendedAttribute(path string, attribute string) int { + return unixResult(unix.Removexattr(path, attribute)) +} diff --git a/internal/mountfs/host_platform_windows.go b/internal/mountfs/host_platform_windows.go new file mode 100644 index 0000000..330a020 --- /dev/null +++ b/internal/mountfs/host_platform_windows.go @@ -0,0 +1,53 @@ +//go:build windows && winfsp + +package mountfs + +import ( + "os" + "syscall" + "time" + + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/windows" +) + +func populateFilesystemStat(path string, stat *fuse.Statfs_t) int { + pointer, err := windows.UTF16PtrFromString(path) + if err != nil { + return unixResult(err) + } + var available, total, free uint64 + if err := windows.GetDiskFreeSpaceEx(pointer, &available, &total, &free); err != nil { + return unixResult(err) + } + const blockSize = uint64(4096) + stat.Bsize = blockSize + stat.Frsize = blockSize + stat.Blocks = total / blockSize + stat.Bfree = free / blockSize + stat.Bavail = available / blockSize + stat.Namemax = 255 + return 0 +} + +func setFileTimes(path string, times []fuse.Timespec) int { + atime := time.Unix(times[0].Sec, times[0].Nsec) + mtime := time.Unix(times[1].Sec, times[1].Nsec) + return unixResult(os.Chtimes(path, atime, mtime)) +} + +func setExtendedAttribute(string, string, []byte, int) int { + return -int(syscall.ENOSYS) +} + +func getExtendedAttribute(string, string) (int, []byte) { + return -int(syscall.ENOSYS), nil +} + +func listExtendedAttributes(string) (int, []string) { + return -int(syscall.ENOSYS), nil +} + +func removeExtendedAttribute(string, string) int { + return -int(syscall.ENOSYS) +} diff --git a/internal/mountfs/host_safety_test.go b/internal/mountfs/host_safety_test.go new file mode 100644 index 0000000..497df42 --- /dev/null +++ b/internal/mountfs/host_safety_test.go @@ -0,0 +1,72 @@ +package mountfs + +import ( + "os" + "path/filepath" + "testing" +) + +func TestPrepareMountPointRejectsOrdinaryFiles(t *testing.T) { + mountPoint := filepath.Join(t.TempDir(), "mount") + if err := os.MkdirAll(filepath.Join(mountPoint, "sessions"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(mountPoint, "sessions", "stale.jsonl"), []byte("stale\n"), 0o600); err != nil { + t.Fatal(err) + } + + if err := prepareMountPoint(mountPoint); err == nil { + t.Fatal("non-empty ordinary directory must not be accepted as a mount backing directory") + } +} + +func TestPrepareMountPointCreatesAndSealsMissingBackingDirectory(t *testing.T) { + mountPoint := filepath.Join(t.TempDir(), "missing", "mount") + if err := prepareMountPoint(mountPoint); err != nil { + t.Fatal(err) + } + info, err := os.Stat(mountPoint) + if err != nil { + t.Fatal(err) + } + if !info.IsDir() || info.Mode().Perm() != 0o500 { + t.Fatalf("created mount backing mode=%#o directory=%t", info.Mode().Perm(), info.IsDir()) + } +} + +func TestPrepareMountPointSealsEmptyBackingDirectory(t *testing.T) { + mountPoint := filepath.Join(t.TempDir(), "mount") + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + + if err := prepareMountPoint(mountPoint); err != nil { + t.Fatal(err) + } + info, err := os.Stat(mountPoint) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm()&0o200 != 0 { + t.Fatalf("unmounted backing directory remained writable: mode=%#o", info.Mode().Perm()) + } + if err := os.Mkdir(filepath.Join(mountPoint, "sessions"), 0o700); err == nil { + t.Fatal("sealed unmounted backing directory accepted a namespace write") + } +} + +func TestPrepareMountPointRejectsSymlink(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "target") + if err := os.MkdirAll(target, 0o700); err != nil { + t.Fatal(err) + } + mountPoint := filepath.Join(root, "mount") + if err := os.Symlink(target, mountPoint); err != nil { + t.Fatal(err) + } + + if err := prepareMountPoint(mountPoint); err == nil { + t.Fatal("mount backing path must not be a symlink") + } +} diff --git a/internal/mountfs/host_statfs_darwin.go b/internal/mountfs/host_statfs_darwin.go new file mode 100644 index 0000000..730d9a0 --- /dev/null +++ b/internal/mountfs/host_statfs_darwin.go @@ -0,0 +1,30 @@ +//go:build darwin && fuse && cgo + +package mountfs + +import ( + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/unix" +) + +func populateFilesystemStat(path string, stat *fuse.Statfs_t) int { + var source unix.Statfs_t + result := unixResult(unix.Statfs(path, &source)) + if result != 0 { + return result + } + stat.Bsize = uint64(source.Bsize) + if source.Iosize > 0 { + stat.Frsize = uint64(source.Iosize) + } else { + stat.Frsize = uint64(source.Bsize) + } + stat.Blocks = source.Blocks + stat.Bfree = source.Bfree + stat.Bavail = source.Bavail + stat.Files = source.Files + stat.Ffree = source.Ffree + stat.Favail = source.Ffree + stat.Namemax = 255 + return 0 +} diff --git a/internal/mountfs/host_statfs_linux.go b/internal/mountfs/host_statfs_linux.go new file mode 100644 index 0000000..f98dde8 --- /dev/null +++ b/internal/mountfs/host_statfs_linux.go @@ -0,0 +1,26 @@ +//go:build linux && fuse && fuse3 && cgo + +package mountfs + +import ( + "github.com/winfsp/cgofuse/fuse" + "golang.org/x/sys/unix" +) + +func populateFilesystemStat(path string, stat *fuse.Statfs_t) int { + var source unix.Statfs_t + result := unixResult(unix.Statfs(path, &source)) + if result != 0 { + return result + } + stat.Bsize = uint64(source.Bsize) + stat.Frsize = uint64(source.Bsize) + stat.Blocks = source.Blocks + stat.Bfree = source.Bfree + stat.Bavail = source.Bavail + stat.Files = source.Files + stat.Ffree = source.Ffree + stat.Favail = source.Ffree + stat.Namemax = 255 + return 0 +} diff --git a/internal/mountfs/host_stub.go b/internal/mountfs/host_stub.go new file mode 100644 index 0000000..7fc0b95 --- /dev/null +++ b/internal/mountfs/host_stub.go @@ -0,0 +1,9 @@ +//go:build (!darwin && !linux && !windows) || (darwin && (!fuse || !cgo)) || (linux && (!fuse || !fuse3 || !cgo)) || (windows && !winfsp) + +package mountfs + +import "context" + +func Available() bool { return false } + +func mountHost(context.Context, HostOptions) error { return ErrPrerequisite } diff --git a/internal/mountfs/mount_backing_linux.go b/internal/mountfs/mount_backing_linux.go new file mode 100644 index 0000000..dfdf145 --- /dev/null +++ b/internal/mountfs/mount_backing_linux.go @@ -0,0 +1,60 @@ +//go:build linux && fuse && fuse3 && cgo + +package mountfs + +import ( + "errors" + "os" + "sync" +) + +type linuxMountBacking struct { + mu sync.Mutex + directory *os.File + sealed bool + closed bool +} + +func prepareMountedBacking(path string) (*linuxMountBacking, error) { + directory, err := os.Open(path) + if err != nil { + return nil, err + } + if err := directory.Chmod(0o700); err != nil { + _ = directory.Close() + return nil, err + } + return &linuxMountBacking{directory: directory}, nil +} + +func (backing *linuxMountBacking) Seal() error { + backing.mu.Lock() + defer backing.mu.Unlock() + if backing.closed { + return errors.New("mount backing guard is already closed") + } + if backing.sealed { + return nil + } + if err := backing.directory.Chmod(0o500); err != nil { + return err + } + backing.sealed = true + return nil +} + +func (backing *linuxMountBacking) Close() error { + backing.mu.Lock() + defer backing.mu.Unlock() + if backing.closed { + return nil + } + var sealErr error + if !backing.sealed { + sealErr = backing.directory.Chmod(0o500) + backing.sealed = sealErr == nil + } + closeErr := backing.directory.Close() + backing.closed = true + return errors.Join(sealErr, closeErr) +} diff --git a/internal/mountfs/mount_backing_other.go b/internal/mountfs/mount_backing_other.go new file mode 100644 index 0000000..6a7df4a --- /dev/null +++ b/internal/mountfs/mount_backing_other.go @@ -0,0 +1,12 @@ +//go:build (darwin && fuse && cgo) || (windows && winfsp) + +package mountfs + +type noOpMountBacking struct{} + +func prepareMountedBacking(string) (*noOpMountBacking, error) { + return &noOpMountBacking{}, nil +} + +func (*noOpMountBacking) Seal() error { return nil } +func (*noOpMountBacking) Close() error { return nil } diff --git a/internal/mountfs/mount_linux.go b/internal/mountfs/mount_linux.go new file mode 100644 index 0000000..a4470ae --- /dev/null +++ b/internal/mountfs/mount_linux.go @@ -0,0 +1,89 @@ +//go:build linux + +package mountfs + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" +) + +type linuxMountRecord struct { + Filesystem string + Source string +} + +func recoverStaleMount(mountPoint string) error { + record, mounted := findLinuxMount(mountPoint) + if !mounted { + return nil + } + if !strings.HasPrefix(record.Filesystem, "fuse") || !strings.Contains(strings.ToLower(record.Source), "codexfold") { + return fmt.Errorf("mount point is already used by %s source %s", record.Filesystem, record.Source) + } + _, healthErr := os.ReadFile(filepath.Join(mountPoint, ".codexfold-health")) + if healthErr == nil { + return errors.New("a healthy CodexFold mount is already active") + } + if !errors.Is(healthErr, syscall.ENOTCONN) && !errors.Is(healthErr, syscall.EIO) { + return fmt.Errorf("CodexFold mount is not proven stale: %w", healthErr) + } + fusermount, err := exec.LookPath("fusermount3") + if err != nil { + return fmt.Errorf("locate fusermount3 for stale mount recovery: %w", err) + } + if output, err := exec.Command(fusermount, "-uz", mountPoint).CombinedOutput(); err != nil { + return fmt.Errorf("unmount stale CodexFold FUSE3 mount: %w: %s", err, strings.TrimSpace(string(output))) + } + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, exists := findLinuxMount(mountPoint); !exists { + return os.Chmod(mountPoint, 0o500) + } + time.Sleep(25 * time.Millisecond) + } + return errors.New("stale CodexFold FUSE3 mount remained after fusermount3") +} + +func linuxFuseMountVisible(mountPoint string) bool { + record, mounted := findLinuxMount(mountPoint) + return mounted && strings.HasPrefix(record.Filesystem, "fuse") +} + +func findLinuxMount(mountPoint string) (linuxMountRecord, bool) { + data, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + return linuxMountRecord{}, false + } + want := filepath.Clean(mountPoint) + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) < 7 { + continue + } + separator := -1 + for index := 6; index < len(fields); index++ { + if fields[index] == "-" { + separator = index + break + } + } + if separator < 0 || separator+2 >= len(fields) { + continue + } + mountedAt := unescapeLinuxMountField(fields[4]) + if filepath.Clean(mountedAt) == want { + return linuxMountRecord{Filesystem: fields[separator+1], Source: unescapeLinuxMountField(fields[separator+2])}, true + } + } + return linuxMountRecord{}, false +} + +func unescapeLinuxMountField(value string) string { + return strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`).Replace(value) +} diff --git a/internal/mountfs/mount_policy_darwin.go b/internal/mountfs/mount_policy_darwin.go new file mode 100644 index 0000000..cbc5359 --- /dev/null +++ b/internal/mountfs/mount_policy_darwin.go @@ -0,0 +1,51 @@ +//go:build darwin && fuse && cgo + +package mountfs + +import ( + "bytes" + "context" + "fmt" + "os/exec" + "time" + + "golang.org/x/sys/unix" +) + +func configureMountedFilesystem(ctx context.Context, mountPoint string) error { + deadline, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + + var lastErr error + for { + var stat unix.Statfs_t + if err := unix.Statfs(mountPoint, &stat); err == nil && statfsType(stat) == "nfs" { + output, err := exec.CommandContext(deadline, "/sbin/mount", "-u", "-o", "sync", mountPoint).CombinedOutput() + if err == nil { + if err := unix.Statfs(mountPoint, &stat); err == nil && stat.Flags&unix.MNT_SYNCHRONOUS != 0 { + return nil + } + lastErr = fmt.Errorf("NFS mount did not report synchronous I/O") + } else if deadline.Err() == nil { + lastErr = fmt.Errorf("update NFS mount: %w: %s", err, bytes.TrimSpace(output)) + } + } + + select { + case <-deadline.Done(): + if lastErr != nil { + return lastErr + } + return deadline.Err() + case <-time.After(25 * time.Millisecond): + } + } +} + +func statfsType(stat unix.Statfs_t) string { + length := bytes.IndexByte(stat.Fstypename[:], 0) + if length < 0 { + length = len(stat.Fstypename) + } + return string(stat.Fstypename[:length]) +} diff --git a/internal/mountfs/mount_policy_linux.go b/internal/mountfs/mount_policy_linux.go new file mode 100644 index 0000000..4684716 --- /dev/null +++ b/internal/mountfs/mount_policy_linux.go @@ -0,0 +1,24 @@ +//go:build linux && fuse && fuse3 && cgo + +package mountfs + +import ( + "context" + "errors" + "time" +) + +func configureMountedFilesystem(ctx context.Context, mountPoint string) error { + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + for { + if linuxFuseMountVisible(mountPoint) { + return nil + } + select { + case <-ctx.Done(): + return errors.New("FUSE3 mount did not become visible before cancellation") + case <-ticker.C: + } + } +} diff --git a/internal/mountfs/mount_policy_windows.go b/internal/mountfs/mount_policy_windows.go new file mode 100644 index 0000000..6c6d9d0 --- /dev/null +++ b/internal/mountfs/mount_policy_windows.go @@ -0,0 +1,7 @@ +//go:build windows && winfsp + +package mountfs + +import "context" + +func configureMountedFilesystem(context.Context, string) error { return nil } diff --git a/internal/mountfs/mount_stale_other.go b/internal/mountfs/mount_stale_other.go new file mode 100644 index 0000000..b49d91d --- /dev/null +++ b/internal/mountfs/mount_stale_other.go @@ -0,0 +1,5 @@ +//go:build !linux + +package mountfs + +func recoverStaleMount(string) error { return nil } diff --git a/internal/mountfs/native_append.go b/internal/mountfs/native_append.go new file mode 100644 index 0000000..f264bea --- /dev/null +++ b/internal/mountfs/native_append.go @@ -0,0 +1,672 @@ +package mountfs + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "unicode/utf8" +) + +const ( + nativeAppendJournalVersion = 1 + maxNativeAppendPendingBytes = 256 << 20 +) + +var ( + errNativeAppendGap = errors.New("native append transaction contains an unfilled offset gap") + errNativeAppendPending = errors.New("native append transaction has pending writes") + errNativeReadStale = errors.New("native read source changed outside the append state") +) + +// nativeAppendJournalCheckpoint is nil in production. Integration tests use it +// in a subprocess to terminate exactly after the recovery journal is durable. +var nativeAppendJournalCheckpoint func(nativeAppendJournal, []byte) + +type nativeAppendSegment struct { + offset int64 + data []byte +} + +type nativeAppendState struct { + mu sync.RWMutex + path string + journalRoot string + baseSize int64 + visibleEnd int64 + segments []nativeAppendSegment +} + +type nativeAppendJournal struct { + Version int `json:"version"` + TargetPath string `json:"target_path"` + BaseSize int64 `json:"base_size"` + FinalSize int64 `json:"final_size"` + TailSHA256 string `json:"tail_sha256"` +} + +func newNativeAppendState(path string, journalRoot string) (*nativeAppendState, error) { + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, errors.New("native append target is not a regular file") + } + return &nativeAppendState{ + path: filepath.Clean(path), journalRoot: filepath.Clean(journalRoot), + baseSize: info.Size(), visibleEnd: info.Size(), + }, nil +} + +func (s *nativeAppendState) Stage(data []byte, offset int64) (int, error) { + if offset < 0 { + return 0, errors.New("negative native append offset") + } + if len(data) == 0 { + return 0, nil + } + originalBytes := len(data) + s.mu.Lock() + defer s.mu.Unlock() + + end, overflow := addInt64(offset, int64(len(data))) + if overflow { + return 0, errors.New("native append offset overflow") + } + if offset < s.baseSize { + committedEnd := min(end, s.baseSize) + committed := make([]byte, committedEnd-offset) + file, err := os.Open(s.path) + if err != nil { + return 0, err + } + _, readErr := file.ReadAt(committed, offset) + closeErr := file.Close() + if readErr != nil && !errors.Is(readErr, io.EOF) { + return 0, readErr + } + if closeErr != nil { + return 0, closeErr + } + overlap := int(committedEnd - offset) + if !bytes.Equal(committed, data[:overlap]) { + s.clearPendingLocked() + return 0, errors.New("native append conflicts with committed bytes") + } + data = data[overlap:] + offset = committedEnd + } + if len(data) == 0 { + return originalBytes, nil + } + end, overflow = addInt64(offset, int64(len(data))) + if overflow || end-s.baseSize > maxNativeAppendPendingBytes { + s.clearPendingLocked() + return 0, errors.New("native append transaction exceeds the pending byte limit") + } + if err := s.stageSegmentLocked(offset, data); err != nil { + s.clearPendingLocked() + return 0, err + } + if end > s.visibleEnd { + s.visibleEnd = end + } + return originalBytes, nil +} + +func (s *nativeAppendState) stageSegmentLocked(offset int64, data []byte) error { + segments := append(append([]nativeAppendSegment(nil), s.segments...), nativeAppendSegment{ + offset: offset, + data: append([]byte(nil), data...), + }) + sort.SliceStable(segments, func(i, j int) bool { return segments[i].offset < segments[j].offset }) + normalized := make([]nativeAppendSegment, 0, len(segments)) + for _, segment := range segments { + if len(normalized) == 0 { + normalized = append(normalized, segment) + continue + } + last := &normalized[len(normalized)-1] + lastEnd := last.offset + int64(len(last.data)) + segmentEnd := segment.offset + int64(len(segment.data)) + if segment.offset > lastEnd { + normalized = append(normalized, segment) + continue + } + overlapEnd := min(lastEnd, segmentEnd) + if overlapEnd > segment.offset { + lastStart := segment.offset - last.offset + overlap := overlapEnd - segment.offset + if !bytes.Equal(last.data[int(lastStart):int(lastStart+overlap)], segment.data[:int(overlap)]) { + return errors.New("native append segments contain conflicting overlap") + } + } + if segmentEnd > lastEnd { + last.data = append(last.data, segment.data[int(lastEnd-segment.offset):]...) + } + } + s.segments = normalized + return nil +} + +func (s *nativeAppendState) ReadAt(file *os.File, destination []byte, offset int64) (int, error) { + if offset < 0 { + return 0, errors.New("negative native read offset") + } + s.mu.RLock() + defer s.mu.RUnlock() + visibleEnd := s.contiguousEndLocked() + if offset >= visibleEnd { + return 0, io.EOF + } + limit := len(destination) + if remaining := visibleEnd - offset; int64(limit) > remaining { + limit = int(remaining) + } + visible := destination[:limit] + clear(visible) + if offset < s.baseSize { + committedBytes := limit + if remaining := s.baseSize - offset; int64(committedBytes) > remaining { + committedBytes = int(remaining) + } + n, err := file.ReadAt(visible[:committedBytes], offset) + if err != nil && !errors.Is(err, io.EOF) { + return n, err + } + } + for _, segment := range s.segments { + segmentEnd := segment.offset + int64(len(segment.data)) + readEnd := offset + int64(limit) + start := max(offset, segment.offset) + end := min(readEnd, segmentEnd) + if start >= end { + continue + } + copy(visible[start-offset:end-offset], segment.data[start-segment.offset:end-segment.offset]) + } + if limit < len(destination) { + return limit, io.EOF + } + return limit, nil +} + +// StreamRead holds the append-state read lock while the caller streams a +// stable range from the backing descriptor. This prevents append commit, +// truncate, and managed positional writes from changing the visible range +// halfway through a response. +func (s *nativeAppendState) StreamRead(file *os.File, offset int64, length int, callback func(*os.File, int64, int) (int, error)) (int, error) { + if file == nil || callback == nil || offset < 0 || length < 0 { + return 0, errors.New("invalid native stream read") + } + s.mu.RLock() + defer s.mu.RUnlock() + if len(s.segments) != 0 { + return 0, errNativeAppendPending + } + info, err := file.Stat() + if err != nil { + return 0, err + } + if !info.Mode().IsRegular() { + return 0, errors.New("native stream source is not a regular file") + } + if info.Size() != s.baseSize { + return 0, errNativeReadStale + } + if offset >= s.baseSize || length == 0 { + return callback(file, offset, 0) + } + if remaining := s.baseSize - offset; int64(length) > remaining { + length = int(remaining) + } + written, err := callback(file, offset, length) + if written < 0 || written > length { + return written, errors.New("native stream callback returned an invalid byte count") + } + return written, err +} + +// WriteAt serializes managed positional writes with StreamRead. Without this +// lock a sendfile response could race a writer that has already passed the +// pending-append check. +func (s *nativeAppendState) WriteAt(file *os.File, data []byte, offset int64) (int, error) { + if file == nil || offset < 0 { + return 0, errors.New("invalid native positional write") + } + s.mu.Lock() + defer s.mu.Unlock() + if len(s.segments) != 0 { + return 0, errNativeAppendPending + } + n, err := file.WriteAt(data, offset) + if err != nil { + return n, err + } + info, err := file.Stat() + if err != nil { + return n, err + } + s.baseSize = info.Size() + s.visibleEnd = info.Size() + return n, nil +} + +func (s *nativeAppendState) VisibleSize() int64 { + s.mu.RLock() + defer s.mu.RUnlock() + return s.contiguousEndLocked() +} + +func (s *nativeAppendState) VisibleSizeForBacking(backingSize int64) int64 { + s.mu.RLock() + defer s.mu.RUnlock() + if len(s.segments) == 0 { + return backingSize + } + return s.contiguousEndLocked() +} + +func (s *nativeAppendState) HasPending() bool { + s.mu.RLock() + defer s.mu.RUnlock() + return len(s.segments) != 0 +} + +func (s *nativeAppendState) Commit() error { + return s.commit(true) +} + +func (s *nativeAppendState) CommitAvailable() error { + return s.commit(false) +} + +func (s *nativeAppendState) commit(strict bool) error { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.segments) == 0 { + file, err := os.OpenFile(s.path, os.O_RDWR, 0) + if err != nil { + return err + } + syncErr := file.Sync() + closeErr := file.Close() + return errors.Join(syncErr, closeErr) + } + + tail, err := s.assembleLocked() + if err != nil { + if !strict && errors.Is(err, errNativeAppendGap) { + return nil + } + s.clearPendingLocked() + return err + } + if tail[len(tail)-1] != '\n' { + if !strict { + return nil + } + s.clearPendingLocked() + return errors.New("native append transaction ends with an incomplete JSONL record") + } + if !utf8.Valid(tail) || !completeJSONL(tail) { + s.clearPendingLocked() + return errors.New("native append transaction is not complete valid JSONL") + } + if err := commitNativeAppend(s.path, s.journalRoot, s.baseSize, tail); err != nil { + s.clearPendingLocked() + return err + } + s.baseSize += int64(len(tail)) + s.clearPendingLocked() + return nil +} + +func (s *nativeAppendState) Truncate(size int64) error { + if size < 0 { + return errors.New("negative native truncate size") + } + s.mu.Lock() + defer s.mu.Unlock() + if len(s.segments) != 0 { + return errors.New("cannot truncate a native append transaction with pending writes") + } + if err := os.Truncate(s.path, size); err != nil { + return err + } + file, err := os.OpenFile(s.path, os.O_RDWR, 0) + if err != nil { + return err + } + syncErr := file.Sync() + closeErr := file.Close() + if syncErr != nil || closeErr != nil { + return errors.Join(syncErr, closeErr) + } + s.baseSize = size + s.visibleEnd = size + return nil +} + +func (s *nativeAppendState) Refresh() error { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.segments) != 0 { + return errors.New("cannot refresh native append state with pending writes") + } + info, err := os.Stat(s.path) + if err != nil { + return err + } + s.baseSize = info.Size() + s.visibleEnd = info.Size() + return nil +} + +func (s *nativeAppendState) RefreshIfIdle() error { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.segments) != 0 { + return nil + } + info, err := os.Stat(s.path) + if err != nil { + return err + } + s.baseSize = info.Size() + s.visibleEnd = info.Size() + return nil +} + +func (s *nativeAppendState) Relocate(newPath string) error { + if !filepath.IsAbs(newPath) { + return errors.New("absolute native append relocation path is required") + } + s.mu.Lock() + defer s.mu.Unlock() + newPath = filepath.Clean(newPath) + if err := os.Rename(s.path, newPath); err != nil { + return err + } + s.path = newPath + return nil +} + +func (s *nativeAppendState) assembleLocked() ([]byte, error) { + if s.visibleEnd < s.baseSize || s.visibleEnd-s.baseSize > maxNativeAppendPendingBytes { + return nil, errors.New("native append transaction has an invalid visible range") + } + tail := make([]byte, s.visibleEnd-s.baseSize) + covered := make([]byte, len(tail)) + segments := append([]nativeAppendSegment(nil), s.segments...) + sort.SliceStable(segments, func(i, j int) bool { return segments[i].offset < segments[j].offset }) + for _, segment := range segments { + start := segment.offset - s.baseSize + if start < 0 || start+int64(len(segment.data)) > int64(len(tail)) { + return nil, errors.New("native append segment lies outside the transaction range") + } + for index, value := range segment.data { + position := int(start) + index + if covered[position] != 0 && tail[position] != value { + return nil, errors.New("native append segments contain conflicting overlap") + } + tail[position] = value + covered[position] = 1 + } + } + if bytes.IndexByte(covered, 0) >= 0 { + return nil, errNativeAppendGap + } + return tail, nil +} + +func (s *nativeAppendState) contiguousEndLocked() int64 { + end := s.baseSize + for _, segment := range s.segments { + if segment.offset > end { + break + } + segmentEnd := segment.offset + int64(len(segment.data)) + if segmentEnd > end { + end = segmentEnd + } + } + return end +} + +func (s *nativeAppendState) clearPendingLocked() { + s.segments = nil + s.visibleEnd = s.baseSize +} + +func commitNativeAppend(path string, journalRoot string, baseSize int64, tail []byte) error { + info, err := os.Stat(path) + if err != nil { + return err + } + if info.Size() != baseSize { + return fmt.Errorf("native append backing changed: size=%d expected=%d", info.Size(), baseSize) + } + digest := sha256.Sum256(tail) + record := nativeAppendJournal{ + Version: nativeAppendJournalVersion, TargetPath: filepath.Clean(path), BaseSize: baseSize, + FinalSize: baseSize + int64(len(tail)), TailSHA256: hex.EncodeToString(digest[:]), + } + journalPath, err := writeNativeAppendJournal(journalRoot, record) + if err != nil { + return err + } + if nativeAppendJournalCheckpoint != nil { + nativeAppendJournalCheckpoint(record, tail) + } + rollback := func(commitErr error) error { + truncateErr := os.Truncate(path, baseSize) + syncErr := syncNativePath(path) + if truncateErr != nil || syncErr != nil { + // Leave the journal in place so startup recovery can finish rollback. + return errors.Join(commitErr, truncateErr, syncErr) + } + return errors.Join(commitErr, removeNativeAppendJournal(journalPath)) + } + + file, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + return rollback(err) + } + n, writeErr := file.WriteAt(tail, baseSize) + if writeErr == nil && n != len(tail) { + writeErr = io.ErrShortWrite + } + syncErr := file.Sync() + closeErr := file.Close() + if writeErr != nil || syncErr != nil || closeErr != nil { + return rollback(errors.Join(writeErr, syncErr, closeErr)) + } + verified, err := hashNativeAppendTail(path, baseSize, int64(len(tail))) + if err != nil || verified != record.TailSHA256 { + return rollback(fmt.Errorf("verify native append transaction: digest=%s expected=%s err=%w", verified, record.TailSHA256, err)) + } + return removeNativeAppendJournal(journalPath) +} + +func writeNativeAppendJournal(root string, record nativeAppendJournal) (string, error) { + if root == "" || !filepath.IsAbs(root) { + return "", errors.New("absolute native append journal root is required") + } + if err := os.MkdirAll(root, 0o700); err != nil { + return "", err + } + digest := sha256.Sum256([]byte(record.TargetPath)) + finalPath := filepath.Join(root, hex.EncodeToString(digest[:])+".json") + data, err := json.Marshal(record) + if err != nil { + return "", err + } + temporary, err := os.CreateTemp(root, ".native-append-*.tmp") + if err != nil { + return "", err + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return "", err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return "", err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return "", err + } + if err := temporary.Close(); err != nil { + return "", err + } + if err := os.Rename(temporaryPath, finalPath); err != nil { + return "", err + } + if err := syncDirectory(root); err != nil { + return "", err + } + return finalPath, nil +} + +func removeNativeAppendJournal(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return syncDirectory(filepath.Dir(path)) +} + +func recoverNativeAppendTransactions(nativeRoot string, journalRoot string) error { + entries, err := os.ReadDir(journalRoot) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + for _, entry := range entries { + if entry.IsDir() { + return fmt.Errorf("native append journal contains an unexpected directory %q", entry.Name()) + } + path := filepath.Join(journalRoot, entry.Name()) + if strings.HasPrefix(entry.Name(), ".native-append-") && strings.HasSuffix(entry.Name(), ".tmp") { + if err := os.Remove(path); err != nil { + return err + } + continue + } + if !strings.HasSuffix(entry.Name(), ".json") { + return fmt.Errorf("native append journal contains an unexpected file %q", entry.Name()) + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + var record nativeAppendJournal + if err := json.Unmarshal(data, &record); err != nil { + return fmt.Errorf("decode native append journal %s: %w", entry.Name(), err) + } + if err := validateNativeAppendJournal(nativeRoot, record); err != nil { + return fmt.Errorf("validate native append journal %s: %w", entry.Name(), err) + } + info, err := os.Stat(record.TargetPath) + if err != nil { + return err + } + committed := false + if info.Size() == record.FinalSize { + digest, hashErr := hashNativeAppendTail(record.TargetPath, record.BaseSize, record.FinalSize-record.BaseSize) + committed = hashErr == nil && digest == record.TailSHA256 + } + if !committed { + if info.Size() < record.BaseSize { + return fmt.Errorf("native append target is shorter than its rollback size: %s", record.TargetPath) + } + if err := os.Truncate(record.TargetPath, record.BaseSize); err != nil { + return err + } + if err := syncNativePath(record.TargetPath); err != nil { + return err + } + } + if err := os.Remove(path); err != nil { + return err + } + } + return syncDirectory(journalRoot) +} + +func validateNativeAppendJournal(nativeRoot string, record nativeAppendJournal) error { + if record.Version != nativeAppendJournalVersion || record.BaseSize < 0 || record.FinalSize < record.BaseSize || + record.FinalSize-record.BaseSize > maxNativeAppendPendingBytes { + return errors.New("native append journal metadata is invalid") + } + target := filepath.Clean(record.TargetPath) + relative, err := filepath.Rel(filepath.Clean(nativeRoot), target) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return errors.New("native append journal target is outside the native root") + } + if !nativeTransactionPath("/" + filepath.ToSlash(relative)) { + return errors.New("native append journal target is not a canonical session path") + } + digest, err := hex.DecodeString(record.TailSHA256) + if err != nil || len(digest) != sha256.Size { + return errors.New("native append journal digest is invalid") + } + return nil +} + +func hashNativeAppendTail(path string, offset int64, size int64) (string, error) { + if offset < 0 || size < 0 { + return "", errors.New("invalid native append hash range") + } + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hasher := sha256.New() + if _, err := io.CopyN(hasher, io.NewSectionReader(file, offset, size), size); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func syncNativePath(path string) error { + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return err + } + syncErr := file.Sync() + closeErr := file.Close() + return errors.Join(syncErr, closeErr) +} + +func syncDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + syncErr := directory.Sync() + closeErr := directory.Close() + return errors.Join(syncErr, closeErr) +} + +func addInt64(left int64, right int64) (int64, bool) { + if right > 0 && left > int64(^uint64(0)>>1)-right { + return 0, true + } + return left + right, false +} diff --git a/internal/mountfs/native_append_real_integration_test.go b/internal/mountfs/native_append_real_integration_test.go new file mode 100644 index 0000000..0e8e1b1 --- /dev/null +++ b/internal/mountfs/native_append_real_integration_test.go @@ -0,0 +1,368 @@ +//go:build darwin && fuse && cgo + +package mountfs + +import ( + "bufio" + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "testing" +) + +const ( + nativeAppendCrashHelperEnv = "CODEXFOLD_NATIVE_APPEND_CRASH_HELPER" + nativeAppendCrashRootEnv = "CODEXFOLD_NATIVE_APPEND_CRASH_ROOT" + realCodexTraceBaseBytes = 51836 + realCodexTraceFinalBytes = 57255 +) + +type codexWriteReplayOperation struct { + kind string + offset int64 + size int + flags int +} + +func TestRealFuseReplaysSanitizedCodexWriteTraceAgainstAPFS(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + route := filepath.Join("sessions", "2026", "07", "16", "rollout-trace-replay.jsonl") + nativeRoot := filepath.Join(root, "native") + nativePath := filepath.Join(nativeRoot, route) + referencePath := filepath.Join(root, "apfs-reference.jsonl") + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + + base := exactJSONLRecord(t, realCodexTraceBaseBytes, 'b') + tail := exactJSONLRecord(t, realCodexTraceFinalBytes-realCodexTraceBaseBytes, 't') + want := append(append([]byte(nil), base...), tail...) + for _, target := range []string{referencePath, nativePath} { + if err := os.WriteFile(target, base, 0o600); err != nil { + t.Fatal(err) + } + } + operations := loadCodexWriteReplay(t, filepath.Join("testdata", "codex-real-resume-write.trace")) + + var recordedMu sync.Mutex + var recorded []string + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + stopMount := startRealMountWithOptions(t, HostOptions{ + MountPoint: mountPoint, + Filesystem: filesystem, + Foreground: true, + OperationRecorder: func(operation string) { + recordedMu.Lock() + defer recordedMu.Unlock() + recorded = append(recorded, operation) + }, + }) + target := filepath.Join(mountPoint, route) + waitForRealFile(t, target, base) + + replayCodexWriteOperations(t, referencePath, want, operations) + replayCodexWriteOperations(t, target, want, operations) + assertNativeBytes(t, nativePath, want) + assertNativeBytes(t, referencePath, want) + visible, err := os.ReadFile(target) + if err != nil || !bytes.Equal(visible, want) { + t.Fatalf("mounted replay bytes=%d err=%v want=%d", len(visible), err, len(want)) + } + if !completeJSONL(want) { + t.Fatal("trace replay fixture did not produce valid JSONL") + } + + recordedMu.Lock() + joined := strings.Join(recorded, "\n") + recordedMu.Unlock() + for _, operation := range operations { + if operation.kind != "write" { + continue + } + marker := fmt.Sprintf("offset=%d bytes=%d result=%d", operation.offset, operation.size, operation.size) + if !strings.Contains(joined, marker) { + t.Fatalf("real FUSE trace did not replay %q: %s", marker, joined) + } + } + for _, marker := range []string{"open kind=session flags=0x2", "fsync kind=session", "flush kind=session", "release kind=session"} { + if !strings.Contains(joined, marker) { + t.Fatalf("real FUSE replay trace missing %q: %s", marker, joined) + } + } + for _, entry := range strings.Split(joined, "\n") { + if strings.Contains(entry, "kind=session") && strings.Contains(entry, "result=-") { + t.Fatalf("real FUSE replay operation failed: %s", entry) + } + } + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func TestRealFuseNativeAppendSIGKILLRecovery(t *testing.T) { + if os.Getenv(nativeAppendCrashHelperEnv) == "1" { + runNativeAppendCrashHelper(t) + return + } + if os.Getenv("CODEXFOLD_RUN_FUSE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_FUSE_TEST=1 to run the real FUSE-T adapter test") + } + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := filepath.Join("sessions", "2026", "07", "16", "rollout-sigkill-recovery.jsonl") + nativePath := filepath.Join(nativeRoot, route) + mountPoint := filepath.Join(root, "mount") + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":\"before-crash\"}\n") + crashTail := exactJSONLRecord(t, 128<<10, 'c') + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + + command := exec.Command(os.Args[0], "-test.run=^TestRealFuseNativeAppendSIGKILLRecovery$", "-test.v") + command.Env = append(os.Environ(), nativeAppendCrashHelperEnv+"=1", nativeAppendCrashRootEnv+"="+root) + helperLogPath := filepath.Join(root, "crash-helper.log") + helperLog, err := os.Create(helperLogPath) + if err != nil { + t.Fatal(err) + } + command.Stdout, command.Stderr = helperLog, helperLog + runErr := command.Run() + closeErr := helperLog.Close() + output, readErr := os.ReadFile(helperLogPath) + if closeErr != nil || readErr != nil { + t.Fatalf("read crash helper log: close=%v read=%v", closeErr, readErr) + } + if runErr == nil { + t.Fatalf("crash helper exited normally: %s", output) + } + exitError, ok := runErr.(*exec.ExitError) + if !ok { + t.Fatalf("crash helper did not return an exit status: %v: %s", runErr, output) + } + waitStatus, ok := exitError.Sys().(syscall.WaitStatus) + if !ok || !waitStatus.Signaled() || waitStatus.Signal() != syscall.SIGKILL { + t.Fatalf("crash helper was not SIGKILLed: status=%v err=%v: %s", exitError.Sys(), runErr, output) + } + + info, err := os.Stat(nativePath) + if err != nil { + t.Fatal(err) + } + if info.Size() <= int64(len(base)) || info.Size() >= int64(len(base)+len(crashTail)) { + t.Fatalf("crash did not leave a partial backing write: size=%d base=%d final=%d", info.Size(), len(base), len(base)+len(crashTail)) + } + journalRoot := filepath.Join(nativeRoot, ".codexfold-native-journal") + entries, err := os.ReadDir(journalRoot) + if err != nil || len(entries) != 1 { + t.Fatalf("durable recovery journal missing after SIGKILL: entries=%d err=%v", len(entries), err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + if err := filesystem.RecoverNativeAppendTransactions(); err != nil { + t.Fatalf("recover SIGKILL transaction: %v", err) + } + assertNativeBytes(t, nativePath, base) + assertEmptyJournal(t, journalRoot) + + stopMount := startRealMount(t, mountPoint, filesystem) + target := filepath.Join(mountPoint, route) + waitForRealFile(t, target, base) + afterRecovery := []byte("{\"record\":\"after-recovery\"}\n") + file, err := os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if n, writeErr := file.Write(afterRecovery); writeErr != nil || n != len(afterRecovery) { + _ = file.Close() + t.Fatalf("append after SIGKILL recovery: n=%d err=%v", n, writeErr) + } + if err := file.Close(); err != nil { + t.Fatalf("close after SIGKILL recovery: %v", err) + } + want := append(append([]byte(nil), base...), afterRecovery...) + assertNativeBytes(t, nativePath, want) + if !completeJSONL(want) { + t.Fatal("post-recovery append produced invalid JSONL") + } + stopMount() + waitForRealUnmount(t, mountPoint) +} + +func runNativeAppendCrashHelper(t *testing.T) { + root := os.Getenv(nativeAppendCrashRootEnv) + if root == "" { + t.Fatal("crash helper root is missing") + } + nativeRoot := filepath.Join(root, "native") + nativePath := filepath.Join(nativeRoot, "sessions", "2026", "07", "16", "rollout-sigkill-recovery.jsonl") + journalRoot := filepath.Join(nativeRoot, ".codexfold-native-journal") + tail := exactJSONLRecord(t, 128<<10, 'c') + nativeAppendJournalCheckpoint = func(record nativeAppendJournal, committedTail []byte) { + if !bytes.Equal(committedTail, tail) { + panic("checkpoint received an unexpected append tail") + } + file, err := os.OpenFile(record.TargetPath, os.O_WRONLY, 0) + if err != nil { + panic(err) + } + partial := committedTail[:len(committedTail)/2] + if n, err := file.WriteAt(partial, record.BaseSize); err != nil || n != len(partial) { + panic(fmt.Sprintf("partial crash write n=%d err=%v", n, err)) + } + if err := file.Sync(); err != nil { + panic(err) + } + if err := file.Close(); err != nil { + panic(err) + } + if err := syscall.Kill(os.Getpid(), syscall.SIGKILL); err != nil { + panic(err) + } + select {} + } + info, err := os.Stat(nativePath) + if err != nil { + t.Fatal(err) + } + if err := commitNativeAppend(nativePath, journalRoot, info.Size(), tail); err != nil { + t.Fatalf("crash checkpoint was not reached: %v", err) + } + t.Fatal("crash checkpoint was not reached") +} + +func loadCodexWriteReplay(t *testing.T, path string) []codexWriteReplayOperation { + t.Helper() + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + var operations []codexWriteReplayOperation + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + operation := codexWriteReplayOperation{kind: fields[0]} + switch operation.kind { + case "open": + if len(fields) != 2 { + t.Fatalf("invalid open replay operation: %q", line) + } + flags, err := strconv.ParseInt(fields[1], 0, 32) + if err != nil { + t.Fatal(err) + } + operation.flags = int(flags) + case "write": + if len(fields) != 3 { + t.Fatalf("invalid write replay operation: %q", line) + } + offset, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + t.Fatal(err) + } + size, err := strconv.Atoi(fields[2]) + if err != nil { + t.Fatal(err) + } + operation.offset, operation.size = offset, size + case "fsync", "flush", "release": + if len(fields) != 1 { + t.Fatalf("invalid %s replay operation: %q", operation.kind, line) + } + default: + t.Fatalf("unknown replay operation: %q", line) + } + operations = append(operations, operation) + } + if err := scanner.Err(); err != nil { + t.Fatal(err) + } + return operations +} + +func replayCodexWriteOperations(t *testing.T, path string, final []byte, operations []codexWriteReplayOperation) { + t.Helper() + var file *os.File + for _, operation := range operations { + switch operation.kind { + case "open": + if file != nil { + t.Fatal("replay opened an already-open file") + } + var err error + file, err = os.OpenFile(path, operation.flags, 0) + if err != nil { + t.Fatal(err) + } + case "write": + if file == nil || operation.offset < 0 || operation.size < 0 || operation.offset+int64(operation.size) > int64(len(final)) { + t.Fatalf("invalid replay write: offset=%d size=%d final=%d", operation.offset, operation.size, len(final)) + } + chunk := final[operation.offset : operation.offset+int64(operation.size)] + if n, err := file.WriteAt(chunk, operation.offset); err != nil || n != len(chunk) { + t.Fatalf("replay write offset=%d size=%d: n=%d err=%v", operation.offset, operation.size, n, err) + } + case "fsync": + if file == nil { + t.Fatal("replay fsync without an open file") + } + if err := file.Sync(); err != nil { + t.Fatal(err) + } + case "flush": + // FUSE emits flush during close; there is no separate portable os.File call. + case "release": + if file == nil { + t.Fatal("replay release without an open file") + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + file = nil + } + } + if file != nil { + _ = file.Close() + t.Fatal("replay trace ended with an open file") + } +} + +func exactJSONLRecord(t *testing.T, size int, fill byte) []byte { + t.Helper() + prefix := []byte("{\"payload\":\"") + suffix := []byte("\"}\n") + if size < len(prefix)+len(suffix) { + t.Fatalf("JSONL record size %d is too small", size) + } + record := append([]byte(nil), prefix...) + record = append(record, bytes.Repeat([]byte{fill}, size-len(prefix)-len(suffix))...) + record = append(record, suffix...) + if len(record) != size || !completeJSONL(record) { + t.Fatalf("invalid exact JSONL record: size=%d want=%d", len(record), size) + } + return record +} diff --git a/internal/mountfs/native_append_test.go b/internal/mountfs/native_append_test.go new file mode 100644 index 0000000..674a3e7 --- /dev/null +++ b/internal/mountfs/native_append_test.go @@ -0,0 +1,437 @@ +package mountfs + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "sync" + "syscall" + "testing" +) + +func TestNativeAppendGapFailsClosedAndCanRetry(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "gap") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + + record := []byte("{\"record\":1}\n") + if n, errno := filesystem.Write(handle, record, int64(len(base)+5)); errno != 0 || n != len(record) { + t.Fatalf("stage gapped append: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("intermediate gapped fsync errno=%v", errno) + } + if errno := filesystem.Flush(handle); errno != syscall.EIO { + t.Fatalf("final gapped flush errno=%v, want EIO", errno) + } + assertNativeBytes(t, nativePath, base) + + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage retry: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("valid retry fsync: %v", errno) + } + assertNativeBytes(t, nativePath, append(append([]byte(nil), base...), record...)) +} + +func TestNativeGetattrDoesNotHideExternalGrowthBehindIdleAppendState(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "external-growth") + handle, errno := filesystem.Open(route, os.O_RDONLY) + if errno != 0 { + t.Fatalf("open native reader: %v", errno) + } + defer filesystem.Release(handle) + + tail := []byte("{\"record\":1}\n") + file, err := os.OpenFile(nativePath, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.Write(tail); err != nil { + file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + + attribute, errno := filesystem.Getattr(route) + want := int64(len(base) + len(tail)) + if errno != 0 || attribute.Size != want { + t.Fatalf("Getattr after external growth size=%d errno=%v, want %d", attribute.Size, errno, want) + } +} + +func TestNativeAppendIntermediateFsyncKeepsPartialRecord(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "partial-fsync") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + record := append([]byte("{\"payload\":\""), bytes.Repeat([]byte("x"), 96*1024)...) + record = append(record, []byte("\"}\n")...) + split := 32 * 1024 + if n, errno := filesystem.Write(handle, record[:split], int64(len(base))); errno != 0 || n != split { + t.Fatalf("stage partial record: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("intermediate fsync: %v", errno) + } + assertNativeBytes(t, nativePath, base) + if n, errno := filesystem.Write(handle, record[split:], int64(len(base)+split)); errno != 0 || n != len(record)-split { + t.Fatalf("finish record: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("complete fsync: %v", errno) + } + assertNativeBytes(t, nativePath, append(append([]byte(nil), base...), record...)) +} + +func TestNativeAppendEarlyHandleReleaseDoesNotDiscardOtherWriter(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "multi-handle-release") + largeHandle := openNativeAppendTestHandle(t, filesystem, route) + smallHandle := openNativeAppendTestHandle(t, filesystem, route) + large := append([]byte("{\"large\":\""), bytes.Repeat([]byte("x"), 96*1024)...) + large = append(large, []byte("\"}\n")...) + small := []byte("{\"small\":true}\n") + split := 32 * 1024 + if n, errno := filesystem.Write(largeHandle, large[:split], int64(len(base))); errno != 0 || n != split { + t.Fatalf("stage large prefix: n=%d errno=%v", n, errno) + } + if n, errno := filesystem.Write(smallHandle, small, int64(len(base)+len(large))); errno != 0 || n != len(small) { + t.Fatalf("stage later record: n=%d errno=%v", n, errno) + } + if errno := filesystem.Release(smallHandle); errno != 0 { + t.Fatalf("release later writer: %v", errno) + } + assertNativeBytes(t, nativePath, base) + if n, errno := filesystem.Write(largeHandle, large[split:], int64(len(base)+split)); errno != 0 || n != len(large)-split { + t.Fatalf("finish large record: n=%d errno=%v", n, errno) + } + if errno := filesystem.Release(largeHandle); errno != 0 { + t.Fatalf("release final writer: %v", errno) + } + want := append(append(append([]byte(nil), base...), large...), small...) + assertNativeBytes(t, nativePath, want) +} + +func TestNativeAppendConflictingOverlapFailsImmediately(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "conflict") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + + record := []byte("{\"record\":1}\n") + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage record: n=%d errno=%v", n, errno) + } + if n, errno := filesystem.Write(handle, []byte("X"), int64(len(base)+2)); errno != syscall.EIO || n != 0 { + t.Fatalf("conflicting overlap: n=%d errno=%v, want n=0 EIO", n, errno) + } + attribute, errno := filesystem.Getattr(route) + if errno != 0 || attribute.Size != int64(len(base)) { + t.Fatalf("conflict remained visible: size=%d errno=%v", attribute.Size, errno) + } + assertNativeBytes(t, nativePath, base) + + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage valid retry: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("valid retry fsync: %v", errno) + } +} + +func TestNativeAppendIdenticalRetriesStayDeduplicated(t *testing.T) { + filesystem, route, _, base := nativeAppendTestFilesystem(t, "deduplicated") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + record := []byte("{\"record\":1}\n") + for index := 0; index < 10_000; index++ { + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("retry %d: n=%d errno=%v", index, n, errno) + } + } + state := filesystem.handles[handle].nativeAppend + state.mu.Lock() + defer state.mu.Unlock() + if len(state.segments) != 1 || len(state.segments[0].data) != len(record) { + t.Fatalf("duplicate retries retained: segments=%d bytes=%d", len(state.segments), len(state.segments[0].data)) + } +} + +func TestNativeAppendFlushCommits(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "flush") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + record := []byte("{\"flush\":true}\n") + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage append: n=%d errno=%v", n, errno) + } + if errno := filesystem.Flush(handle); errno != 0 { + t.Fatalf("flush: %v", errno) + } + assertNativeBytes(t, nativePath, append(append([]byte(nil), base...), record...)) +} + +func TestNativeAppendReleaseCommits(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "release") + handle := openNativeAppendTestHandle(t, filesystem, route) + record := []byte("{\"release\":true}\n") + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage append: n=%d errno=%v", n, errno) + } + if errno := filesystem.Release(handle); errno != 0 { + t.Fatalf("release: %v", errno) + } + assertNativeBytes(t, nativePath, append(append([]byte(nil), base...), record...)) +} + +func TestNativeAppendTruncateFailsWhilePending(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "truncate") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + record := []byte("{\"pending\":true}\n") + if n, errno := filesystem.Write(handle, record, int64(len(base))); errno != 0 || n != len(record) { + t.Fatalf("stage append: n=%d errno=%v", n, errno) + } + if errno := filesystem.Truncate(handle, 0); errno != syscall.EIO { + t.Fatalf("handle truncate errno=%v, want EIO", errno) + } + if errno := filesystem.TruncatePath(route, 0); errno != syscall.EIO { + t.Fatalf("path truncate errno=%v, want EIO", errno) + } + assertNativeBytes(t, nativePath, base) + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("commit after rejected truncate: %v", errno) + } +} + +func TestNativeAppendRejectsInvalidUTF8(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, "utf8") + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + invalid := []byte{'{', '"', 'x', '"', ':', '"', 0xff, '"', '}', '\n'} + if n, errno := filesystem.Write(handle, invalid, int64(len(base))); errno != 0 || n != len(invalid) { + t.Fatalf("stage invalid UTF-8: n=%d errno=%v", n, errno) + } + if errno := filesystem.Fsync(handle); errno != syscall.EIO { + t.Fatalf("invalid UTF-8 fsync errno=%v, want EIO", errno) + } + assertNativeBytes(t, nativePath, base) +} + +func TestNativeAppendHandlesLargeOutOfOrderRecords(t *testing.T) { + for _, payloadBytes := range []int{32*1024 + 1, 64*1024 + 1, 1 << 20} { + t.Run(fmt.Sprintf("payload-%d", payloadBytes), func(t *testing.T) { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, fmt.Sprintf("large-%d", payloadBytes)) + handle := openNativeAppendTestHandle(t, filesystem, route) + defer filesystem.Release(handle) + record := append([]byte("{\"payload\":\""), bytes.Repeat([]byte("x"), payloadBytes)...) + record = append(record, []byte("\"}\n")...) + const chunkBytes = 31 * 1024 + for end := len(record); end > 0; { + start := max(0, end-chunkBytes) + offset := int64(len(base) + start) + if n, errno := filesystem.Write(handle, record[start:end], offset); errno != 0 || n != end-start { + t.Fatalf("stage chunk [%d:%d]: n=%d errno=%v", start, end, n, errno) + } + end = start + } + if errno := filesystem.Fsync(handle); errno != 0 { + t.Fatalf("commit large record: %v", errno) + } + assertNativeBytes(t, nativePath, append(append([]byte(nil), base...), record...)) + }) + } +} + +func TestNativeAppendConcurrentSessionsRemainIndependent(t *testing.T) { + const sessionCount = 8 + type fixture struct { + filesystem *Filesystem + route string + path string + base []byte + handle uint64 + tail []byte + } + fixtures := make([]fixture, 0, sessionCount) + for session := 0; session < sessionCount; session++ { + filesystem, route, nativePath, base := nativeAppendTestFilesystem(t, fmt.Sprintf("parallel-%d", session)) + handle := openNativeAppendTestHandle(t, filesystem, route) + t.Cleanup(func() { _ = filesystem.Release(handle) }) + var tail []byte + for record := 0; record < 200; record++ { + tail = append(tail, fmt.Appendf(nil, "{\"session\":%d,\"record\":%d}\n", session, record)...) + } + fixtures = append(fixtures, fixture{filesystem: filesystem, route: route, path: nativePath, base: base, handle: handle, tail: tail}) + } + + var group sync.WaitGroup + errors := make(chan error, sessionCount) + for _, item := range fixtures { + item := item + group.Add(1) + go func() { + defer group.Done() + if n, errno := item.filesystem.Write(item.handle, item.tail, int64(len(item.base))); errno != 0 || n != len(item.tail) { + errors <- fmt.Errorf("write %s: n=%d errno=%v", item.route, n, errno) + return + } + if errno := item.filesystem.Fsync(item.handle); errno != 0 { + errors <- fmt.Errorf("fsync %s: %v", item.route, errno) + } + }() + } + group.Wait() + close(errors) + for err := range errors { + t.Error(err) + } + for _, item := range fixtures { + assertNativeBytes(t, item.path, append(append([]byte(nil), item.base...), item.tail...)) + } +} + +func TestRecoverNativeAppendTransactionRollsBackPartialCommit(t *testing.T) { + nativeRoot, journalRoot, nativePath, base, tail, record := nativeAppendRecoveryFixture(t, "partial") + if _, err := writeNativeAppendJournal(journalRoot, record); err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(nativePath, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteAt(tail[:len(tail)/2], int64(len(base))); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + if err := recoverNativeAppendTransactions(nativeRoot, journalRoot); err != nil { + t.Fatal(err) + } + assertNativeBytes(t, nativePath, base) + assertEmptyJournal(t, journalRoot) +} + +func TestRecoverNativeAppendTransactionKeepsVerifiedCommit(t *testing.T) { + nativeRoot, journalRoot, nativePath, base, tail, record := nativeAppendRecoveryFixture(t, "committed") + if _, err := writeNativeAppendJournal(journalRoot, record); err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(nativePath, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteAt(tail, int64(len(base))); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + if err := recoverNativeAppendTransactions(nativeRoot, journalRoot); err != nil { + t.Fatal(err) + } + assertNativeBytes(t, nativePath, append(append([]byte(nil), base...), tail...)) + assertEmptyJournal(t, journalRoot) +} + +func TestRecoverNativeAppendTransactionRejectsOutsideTarget(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + journalRoot := filepath.Join(nativeRoot, ".codexfold-native-journal") + outside := filepath.Join(root, "outside.jsonl") + if err := os.MkdirAll(nativeRoot, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(outside, []byte("{\"outside\":true}\n"), 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256([]byte("{\"tail\":true}\n")) + record := nativeAppendJournal{ + Version: nativeAppendJournalVersion, TargetPath: outside, + BaseSize: int64(len("{\"outside\":true}\n")), FinalSize: int64(len("{\"outside\":true}\n{\"tail\":true}\n")), + TailSHA256: hex.EncodeToString(digest[:]), + } + if _, err := writeNativeAppendJournal(journalRoot, record); err != nil { + t.Fatal(err) + } + if err := recoverNativeAppendTransactions(nativeRoot, journalRoot); err == nil { + t.Fatal("outside journal target was accepted") + } + assertNativeBytes(t, outside, []byte("{\"outside\":true}\n")) +} + +func nativeAppendTestFilesystem(t *testing.T, name string) (*Filesystem, string, string, []byte) { + t.Helper() + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + route := "/sessions/2026/07/16/rollout-" + name + ".jsonl" + nativePath := nativePathFromRoot(nativeRoot, route) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + return filesystem, route, nativePath, base +} + +func openNativeAppendTestHandle(t *testing.T, filesystem *Filesystem, route string) uint64 { + t.Helper() + handle, errno := filesystem.Open(route, os.O_WRONLY|os.O_APPEND) + if errno != 0 { + t.Fatalf("open native append handle: %v", errno) + } + return handle +} + +func nativeAppendRecoveryFixture(t *testing.T, name string) (string, string, string, []byte, []byte, nativeAppendJournal) { + t.Helper() + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + journalRoot := filepath.Join(nativeRoot, ".codexfold-native-journal") + nativePath := filepath.Join(nativeRoot, "sessions", "2026", "07", "16", "rollout-"+name+".jsonl") + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + tail := []byte("{\"record\":1}\n") + if err := os.WriteFile(nativePath, base, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(tail) + record := nativeAppendJournal{ + Version: nativeAppendJournalVersion, TargetPath: nativePath, + BaseSize: int64(len(base)), FinalSize: int64(len(base) + len(tail)), + TailSHA256: hex.EncodeToString(digest[:]), + } + return nativeRoot, journalRoot, nativePath, base, tail, record +} + +func assertNativeBytes(t *testing.T, path string, want []byte) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("native bytes at %s = %q err=%v, want %q", path, got, err, want) + } +} + +func assertEmptyJournal(t *testing.T, root string) { + t.Helper() + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("journal entries remained: %v", entries) + } +} diff --git a/internal/mountfs/native_fskit_fd_darwin_test.go b/internal/mountfs/native_fskit_fd_darwin_test.go new file mode 100644 index 0000000..a7a0c0a --- /dev/null +++ b/internal/mountfs/native_fskit_fd_darwin_test.go @@ -0,0 +1,929 @@ +//go:build darwin + +package mountfs + +import ( + "bytes" + "errors" + "io" + "net" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/samekind/codexfold/internal/fskitproto" + "golang.org/x/sys/unix" +) + +func TestNativeFSKitServerTransfersNativeReadFDOnlyForReadOnlyCapability(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + nativePath := filepath.Join(nativeRoot, "sessions", "2026", "07", "17", "performance.bin") + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + want := bytes.Repeat([]byte("codexfold-native-fd\n"), 4096) + if err := os.WriteFile(nativePath, want, 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + _ = client.Close() + defer stop() + + descriptorData, err := os.ReadFile(filepath.Join(root, "resource.bin")) + if err != nil { + t.Fatal(err) + } + descriptor, err := fskitproto.DecodeDescriptor(descriptorData) + if err != nil { + t.Fatal(err) + } + connection := dialNativeFDTestSocket(t, descriptor) + defer connection.Close() + + callNativeFDTestHello(t, connection, descriptor, true) + openPayload := fskitproto.NewEncoder(128) + openPayload.String("/sessions/2026/07/17/performance.bin") + openPayload.Uint32(uint32(os.O_RDONLY)) + response := callNativeFDTestFrame(t, connection, descriptor, 2, fskitproto.OpOpen, openPayload.Data()) + if response.Flags&fskitproto.FlagNativeReadFD == 0 { + t.Fatalf("read-only open flags = %#x, want native FD", response.Flags) + } + decoder := fskitproto.NewDecoder(response.Payload) + handle, err := decoder.Uint64() + if err != nil || decoder.Done() != nil { + t.Fatalf("decode native open handle=%d err=%v", handle, err) + } + + fd := receiveNativeFDTestMarker(t, connection) + file := os.NewFile(uintptr(fd), "codexfold-native-fd-test") + if file == nil { + t.Fatalf("construct received file for fd %d", fd) + } + got, readErr := io.ReadAll(file) + closeErr := file.Close() + if readErr != nil || closeErr != nil { + t.Fatalf("read received FD: read=%v close=%v", readErr, closeErr) + } + if !bytes.Equal(got, want) { + t.Fatalf("received FD bytes differ: got=%d want=%d", len(got), len(want)) + } + + callNativeFDTestHandle(t, connection, descriptor, 3, fskitproto.OpRelease, handle) + + if err := connection.Close(); err != nil { + t.Fatal(err) + } + connection = dialNativeFDTestSocket(t, descriptor) + defer connection.Close() + callNativeFDTestHello(t, connection, descriptor, true) + openPayload = fskitproto.NewEncoder(128) + openPayload.String("/sessions/2026/07/17/performance.bin") + openPayload.Uint32(uint32(os.O_RDWR)) + response = callNativeFDTestFrame(t, connection, descriptor, 2, fskitproto.OpOpen, openPayload.Data()) + if response.Flags&fskitproto.FlagNativeReadFD != 0 { + t.Fatalf("writable open unexpectedly transferred native FD: flags=%#x", response.Flags) + } +} + +func TestConfigureNativeFSKitSocketRaisesStreamBuffers(t *testing.T) { + root, err := os.MkdirTemp("/private/tmp", "codexfold-buffer-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(root) }) + address := &net.UnixAddr{Name: filepath.Join(root, "buffer.sock"), Net: "unix"} + listener, err := net.ListenUnix("unix", address) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + + accepted := make(chan *net.UnixConn, 1) + acceptErrors := make(chan error, 1) + go func() { + connection, acceptErr := listener.AcceptUnix() + if acceptErr != nil { + acceptErrors <- acceptErr + return + } + accepted <- connection + }() + client, err := net.DialUnix("unix", nil, address) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + var server *net.UnixConn + select { + case err := <-acceptErrors: + t.Fatal(err) + case server = <-accepted: + } + defer server.Close() + if err := configureNativeFSKitSocket(server); err != nil { + t.Fatal(err) + } + for _, option := range []int{syscall.SO_RCVBUF, syscall.SO_SNDBUF} { + value := unixSocketOption(t, server, option) + if value < nativeFSKitSocketBufferBytes { + t.Fatalf("socket option %d = %d, want at least %d", option, value, nativeFSKitSocketBufferBytes) + } + } +} + +func TestNativeFSKitServerTransfersSharedReadFDForLargeVirtualReads(t *testing.T) { + root := t.TempDir() + filesystem := NewCanonical() + filesystem.SetNativeRoot(filepath.Join(root, "native")) + line := []byte("{\"shared_read_fd\":true}\n") + source := bytes.Repeat(line, (6<<20)/len(line)+1) + virtualPath := "/archived_sessions/shared-read.jsonl" + if err := filesystem.AddSessionAt("shared-read", virtualPath, mountSessionFixture(t, "shared-read", source)); err != nil { + t.Fatal(err) + } + client, stop := startNativeFSKitTestServer(t, filesystem, root) + _ = client.Close() + defer stop() + descriptor := readNativeFDTestDescriptor(t, root) + connection := dialNativeFDTestSocket(t, descriptor) + defer connection.Close() + callNativeFDTestHelloCapabilities(t, connection, descriptor, fskitproto.CapabilitySharedReadFD) + + handle := openNativeFDTestPath(t, connection, descriptor, 2, virtualPath) + readLength := 5 << 20 + response := callNativeFDTestRead(t, connection, descriptor, 3, handle, 0, readLength) + if response.Flags&fskitproto.FlagSharedReadFD == 0 { + t.Fatalf("large virtual read flags = %#x, want shared FD", response.Flags) + } + count := decodeNativeFDTestReadCount(t, response) + if count != readLength { + t.Fatalf("large virtual read count = %d, want %d", count, readLength) + } + fd := receiveFDTestMarker(t, connection, fskitproto.SharedReadFDMarker) + assertMappedNativeFDTestBytes(t, fd, source[:readLength]) + if err := unix.Close(fd); err != nil { + t.Fatal(err) + } + + offset := int64(len(source) - 97) + response = callNativeFDTestRead(t, connection, descriptor, 4, handle, offset, nativeFSKitSharedReadMinimumBytes) + if response.Flags&fskitproto.FlagSharedReadFD == 0 { + t.Fatalf("EOF virtual read flags = %#x, want shared FD", response.Flags) + } + count = decodeNativeFDTestReadCount(t, response) + if count != 97 { + t.Fatalf("EOF virtual read count = %d, want 97", count) + } + fd = receiveFDTestMarker(t, connection, fskitproto.SharedReadFDMarker) + assertMappedNativeFDTestBytes(t, fd, source[len(source)-97:], nativeFSKitSharedReadMinimumBytes) + if err := unix.Close(fd); err != nil { + t.Fatal(err) + } + callNativeFDTestHandle(t, connection, descriptor, 5, fskitproto.OpRelease, handle) +} + +func TestNativeFSKitServerReusesSharedWindowAcrossVirtualReads(t *testing.T) { + root := t.TempDir() + filesystem := NewCanonical() + filesystem.SetNativeRoot(filepath.Join(root, "native")) + line := []byte("{\"shared_window\":true}\n") + source := bytes.Repeat(line, (7<<20)/len(line)+1) + virtualPath := "/archived_sessions/shared-window.jsonl" + if err := filesystem.AddSessionAt("shared-window", virtualPath, mountSessionFixture(t, "shared-window", source)); err != nil { + t.Fatal(err) + } + client, stop := startNativeFSKitTestServer(t, filesystem, root) + _ = client.Close() + defer stop() + descriptor := readNativeFDTestDescriptor(t, root) + connection := dialNativeFDTestSocket(t, descriptor) + defer connection.Close() + callNativeFDTestHelloCapabilities(t, connection, descriptor, fskitproto.CapabilitySharedWindow) + + openPayload := fskitproto.NewEncoder(128) + openPayload.String(virtualPath) + openPayload.Uint32(uint32(os.O_RDONLY)) + openResponse := callNativeFDTestFrame(t, connection, descriptor, 2, fskitproto.OpOpen, openPayload.Data()) + if openResponse.Flags&fskitproto.FlagSharedWindow == 0 { + t.Fatalf("virtual open flags = %#x, want shared window", openResponse.Flags) + } + decoder := fskitproto.NewDecoder(openResponse.Payload) + handle, err := decoder.Uint64() + if err != nil { + t.Fatal(err) + } + windowBytes, err := decoder.Uint32() + if err != nil || decoder.Done() != nil || int(windowBytes) != nativeFSKitSharedWindowBytes { + t.Fatalf("shared window bytes=%d err=%v", windowBytes, err) + } + windowFD := receiveFDTestMarker(t, connection, fskitproto.SharedWindowFDMarker) + defer unix.Close(windowFD) + windowMapping, err := unix.Mmap(windowFD, 0, nativeSharedReadMappedLength(int(windowBytes)), unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + t.Fatal(err) + } + defer unix.Munmap(windowMapping) + + assertWindowRead := func(requestID uint64, offset int64, length int, want []byte) { + t.Helper() + response := callNativeFDTestRead(t, connection, descriptor, requestID, handle, offset, length) + if response.Flags&fskitproto.FlagSharedWindow == 0 || response.Flags&fskitproto.FlagSharedReadFD != 0 { + t.Fatalf("shared window read flags = %#x", response.Flags) + } + count := decodeNativeFDTestReadCount(t, response) + if count != len(want) || !bytes.Equal(windowMapping[:count], want) { + t.Fatalf("shared window read offset=%d count=%d want=%d", offset, count, len(want)) + } + } + assertWindowRead(3, 0, 5<<20, source[:5<<20]) + assertWindowRead(4, 1<<20, 4<<20, source[1<<20:5<<20]) + assertWindowRead(5, int64(len(source)-73), 4<<20, source[len(source)-73:]) + callNativeFDTestHandle(t, connection, descriptor, 6, fskitproto.OpRelease, handle) +} + +func TestNativeFSKitServerUsesSharedFileWindowForConcurrentPread(t *testing.T) { + root := t.TempDir() + filesystem := NewCanonical() + filesystem.SetNativeRoot(filepath.Join(root, "native")) + line := []byte("{\"shared_file_window\":true}\n") + source := bytes.Repeat(line, (7<<20)/len(line)+1) + virtualPath := "/archived_sessions/shared-file-window.jsonl" + if err := filesystem.AddSessionAt("shared-file-window", virtualPath, mountSessionFixture(t, "shared-file-window", source)); err != nil { + t.Fatal(err) + } + client, stop := startNativeFSKitTestServer(t, filesystem, root) + _ = client.Close() + defer stop() + descriptor := readNativeFDTestDescriptor(t, root) + connection := dialNativeFDTestSocket(t, descriptor) + defer connection.Close() + callNativeFDTestHelloCapabilities(t, connection, descriptor, fskitproto.CapabilitySharedFileWindow) + + openPayload := fskitproto.NewEncoder(128) + openPayload.String(virtualPath) + openPayload.Uint32(uint32(os.O_RDONLY)) + openResponse := callNativeFDTestFrame(t, connection, descriptor, 2, fskitproto.OpOpen, openPayload.Data()) + if openResponse.Flags != fskitproto.FlagSharedFileWindow { + t.Fatalf("virtual open flags = %#x, want shared-file window", openResponse.Flags) + } + decoder := fskitproto.NewDecoder(openResponse.Payload) + handle, err := decoder.Uint64() + if err != nil { + t.Fatal(err) + } + windowBytes, err := decoder.Uint32() + if err != nil || decoder.Done() != nil || int(windowBytes) != nativeFSKitSharedWindowBytes { + t.Fatalf("shared-file window bytes=%d err=%v", windowBytes, err) + } + windowFD := receiveFDTestMarker(t, connection, fskitproto.SharedFileWindowFDMarker) + defer unix.Close(windowFD) + + assertWindowRead := func(requestID uint64, offset int64, length int, want []byte) { + t.Helper() + response := callNativeFDTestRead(t, connection, descriptor, requestID, handle, offset, length) + if response.Flags != fskitproto.FlagSharedFileWindow { + t.Fatalf("shared-file window read flags = %#x", response.Flags) + } + count := decodeNativeFDTestReadCount(t, response) + got := make([]byte, count) + if n, err := unix.Pread(windowFD, got, 0); err != nil || n != count { + t.Fatalf("pread shared-file window bytes=%d err=%v want=%d", n, err, count) + } + if count != len(want) || !bytes.Equal(got, want) { + t.Fatalf("shared-file read offset=%d count=%d want=%d", offset, count, len(want)) + } + } + assertWindowRead(3, 0, 5<<20, source[:5<<20]) + assertWindowRead(4, 1<<20, 4<<20, source[1<<20:5<<20]) + assertWindowRead(5, int64(len(source)-79), 4<<20, source[len(source)-79:]) + callNativeFDTestHandle(t, connection, descriptor, 6, fskitproto.OpRelease, handle) +} + +func TestNativeFSKitServerFallsBackFromPOSIXToSharedFileWindow(t *testing.T) { + root := t.TempDir() + filesystem := NewCanonical() + filesystem.SetNativeRoot(filepath.Join(root, "native")) + source := bytes.Repeat([]byte("{\"window_fallback\":true}\n"), (5<<20)/27+1) + virtualPath := "/archived_sessions/file-window-fallback.jsonl" + if err := filesystem.AddSessionAt("file-window-fallback", virtualPath, mountSessionFixture(t, "file-window-fallback", source)); err != nil { + t.Fatal(err) + } + client, stop := startNativeFSKitTestServer(t, filesystem, root) + _ = client.Close() + defer stop() + descriptor := readNativeFDTestDescriptor(t, root) + connection := dialNativeFDTestSocket(t, descriptor) + defer connection.Close() + callNativeFDTestHelloCapabilities(t, connection, descriptor, fskitproto.CapabilitySharedFileWindow|fskitproto.CapabilitySharedWindow) + + originalCreate := createNativeSharedReadObject + createNativeSharedReadObject = func() (nativeSharedReadObject, error) { + return nativeSharedReadObject{}, errors.New("injected POSIX window failure") + } + defer func() { createNativeSharedReadObject = originalCreate }() + openPayload := fskitproto.NewEncoder(128) + openPayload.String(virtualPath) + openPayload.Uint32(uint32(os.O_RDONLY)) + openResponse := callNativeFDTestFrame(t, connection, descriptor, 2, fskitproto.OpOpen, openPayload.Data()) + if openResponse.Flags != fskitproto.FlagSharedFileWindow { + t.Fatalf("POSIX window fallback flags = %#x, want shared-file window", openResponse.Flags) + } + decoder := fskitproto.NewDecoder(openResponse.Payload) + handle, err := decoder.Uint64() + if err != nil { + t.Fatal(err) + } + if _, err := decoder.Uint32(); err != nil || decoder.Done() != nil { + t.Fatalf("decode fallback window: %v", err) + } + windowFD := receiveFDTestMarker(t, connection, fskitproto.SharedFileWindowFDMarker) + defer unix.Close(windowFD) + callNativeFDTestHandle(t, connection, descriptor, 3, fskitproto.OpRelease, handle) +} + +func TestNativeFSKitServerRecyclesPrewarmedPOSIXWindowsWithinBound(t *testing.T) { + server := &nativeFSKitServer{} + const windowBytes = 2 << 20 + if err := server.prewarmSharedMemoryWindows(2, windowBytes); err != nil { + t.Fatal(err) + } + + first, firstPrewarmed, err := server.acquireSharedMemoryWindow(windowBytes) + if err != nil { + t.Fatal(err) + } + second, secondPrewarmed, err := server.acquireSharedMemoryWindow(windowBytes) + if err != nil { + t.Fatal(err) + } + if !firstPrewarmed || !secondPrewarmed { + t.Fatal("prewarmed POSIX windows were not acquired before recycle") + } + if !server.recycleSharedMemoryWindow(first) || !server.recycleSharedMemoryWindow(second) { + t.Fatal("released POSIX windows were not recycled") + } + + reused, reusedPrewarmed, err := server.acquireSharedMemoryWindow(windowBytes) + if err != nil { + t.Fatal(err) + } + if !reusedPrewarmed { + t.Fatal("recycled POSIX window was not reused") + } + if !server.recycleSharedMemoryWindow(reused) { + t.Fatal("reused POSIX window was not returned to the pool") + } + if err := server.closePrewarmedSharedMemoryWindows(); err != nil { + t.Fatal(err) + } +} + +func TestNativeSharedFileWindowIsMode0600AndUnlinked(t *testing.T) { + window, err := newNativeSharedFileWindow(2 << 20) + if err != nil { + t.Fatal(err) + } + defer window.Close() + path := window.file.Name() + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("shared-file window remained linked at %q: %v", path, err) + } + info, err := window.file.Stat() + if err != nil { + t.Fatal(err) + } + if !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 { + t.Fatalf("shared-file window mode = %v, want regular 0600", info.Mode()) + } + want := bytes.Repeat([]byte("regular-window"), 4096) + copy(window.mapping, want) + got := make([]byte, len(want)) + if n, err := unix.Pread(int(window.file.Fd()), got, 0); err != nil || n != len(got) { + t.Fatalf("pread anonymous shared-file window bytes=%d err=%v", n, err) + } + if !bytes.Equal(got, want) { + t.Fatal("anonymous shared-file window bytes changed") + } +} + +func TestNativeFSKitServerPrewarmedWindowsHaveBoundedOwnership(t *testing.T) { + server := &nativeFSKitServer{} + const windowBytes = 2 << 20 + if err := server.prewarmSharedFileWindows(2, windowBytes); err != nil { + t.Fatal(err) + } + server.sharedFileWindows.mu.Lock() + if len(server.sharedFileWindows.windows) != 2 { + server.sharedFileWindows.mu.Unlock() + t.Fatalf("prewarmed windows=%d, want=2", len(server.sharedFileWindows.windows)) + } + unusedFD := int(server.sharedFileWindows.windows[0].file.Fd()) + server.sharedFileWindows.mu.Unlock() + + acquired, prewarmed, err := server.acquireSharedFileWindow(windowBytes) + if err != nil { + t.Fatal(err) + } + if !prewarmed { + t.Fatal("prewarmed window was not consumed") + } + acquiredFD := int(acquired.file.Fd()) + if err := server.closePrewarmedSharedFileWindows(); err != nil { + t.Fatal(err) + } + if _, err := unix.FcntlInt(uintptr(unusedFD), unix.F_GETFD, 0); !errors.Is(err, unix.EBADF) { + t.Fatalf("unused prewarmed descriptor remained open: %v", err) + } + if _, err := unix.FcntlInt(uintptr(acquiredFD), unix.F_GETFD, 0); err != nil { + t.Fatalf("pool close invalidated an acquired descriptor: %v", err) + } + if err := acquired.Close(); err != nil { + t.Fatal(err) + } + + fallback, prewarmed, err := server.acquireSharedFileWindow(windowBytes) + if err != nil { + t.Fatal(err) + } + if prewarmed { + t.Fatal("empty pool reported a prewarmed window") + } + if err := fallback.Close(); err != nil { + t.Fatal(err) + } +} + +func TestNativeFSKitServerRecyclesSharedFileWindowsWithinBound(t *testing.T) { + server := &nativeFSKitServer{} + const windowBytes = 2 << 20 + if err := server.prewarmSharedFileWindows(2, windowBytes); err != nil { + t.Fatal(err) + } + + first, firstPrewarmed, err := server.acquireSharedFileWindow(windowBytes) + if err != nil { + t.Fatal(err) + } + second, secondPrewarmed, err := server.acquireSharedFileWindow(windowBytes) + if err != nil { + t.Fatal(err) + } + if !firstPrewarmed || !secondPrewarmed { + t.Fatal("prewarmed windows were not acquired before recycle") + } + if !server.recycleSharedFileWindow(first) || !server.recycleSharedFileWindow(second) { + t.Fatal("released shared-file windows were not recycled") + } + + reused, reusedPrewarmed, err := server.acquireSharedFileWindow(windowBytes) + if err != nil { + t.Fatal(err) + } + if !reusedPrewarmed { + t.Fatal("recycled shared-file window was not reused") + } + if !server.recycleSharedFileWindow(reused) { + t.Fatal("reused shared-file window was not returned to the pool") + } + + overflow, err := newNativeSharedFileWindow(windowBytes) + if err != nil { + t.Fatal(err) + } + overflowFD := int(overflow.file.Fd()) + if server.recycleSharedFileWindow(overflow) { + t.Fatal("full pool accepted an overflow shared-file window") + } + if _, err := unix.FcntlInt(uintptr(overflowFD), unix.F_GETFD, 0); !errors.Is(err, unix.EBADF) { + t.Fatalf("overflow shared-file window remained open: %v", err) + } + if err := server.closePrewarmedSharedFileWindows(); err != nil { + t.Fatal(err) + } +} + +func TestNativeFSKitServerFallsBackWhenSharedWindowPreparationFails(t *testing.T) { + root := t.TempDir() + filesystem := NewCanonical() + filesystem.SetNativeRoot(filepath.Join(root, "native")) + source := bytes.Repeat([]byte("{\"window_fallback\":true}\n"), (5<<20)/27+1) + virtualPath := "/archived_sessions/window-fallback.jsonl" + if err := filesystem.AddSessionAt("window-fallback", virtualPath, mountSessionFixture(t, "window-fallback", source)); err != nil { + t.Fatal(err) + } + client, stop := startNativeFSKitTestServer(t, filesystem, root) + _ = client.Close() + defer stop() + descriptor := readNativeFDTestDescriptor(t, root) + connection := dialNativeFDTestSocket(t, descriptor) + defer connection.Close() + callNativeFDTestHelloCapabilities(t, connection, descriptor, fskitproto.CapabilitySharedWindow) + + originalCreate := createNativeSharedReadObject + createNativeSharedReadObject = func() (nativeSharedReadObject, error) { + return nativeSharedReadObject{}, errors.New("injected shared window failure") + } + defer func() { createNativeSharedReadObject = originalCreate }() + handle := openNativeFDTestPath(t, connection, descriptor, 2, virtualPath) + response := callNativeFDTestRead(t, connection, descriptor, 3, handle, 0, nativeFSKitSharedReadMinimumBytes) + if response.Flags != 0 { + t.Fatalf("shared window fallback flags = %#x, want byte stream", response.Flags) + } + decoder := fskitproto.NewDecoder(response.Payload) + got, err := decoder.Bytes(nativeFSKitSharedReadMinimumBytes) + if err != nil || decoder.Done() != nil || !bytes.Equal(got, source[:nativeFSKitSharedReadMinimumBytes]) { + t.Fatalf("shared window fallback bytes=%d err=%v", len(got), err) + } + callNativeFDTestHandle(t, connection, descriptor, 4, fskitproto.OpRelease, handle) +} + +func TestNativeFSKitServerKeepsByteStreamWithoutSharedReadCapability(t *testing.T) { + root := t.TempDir() + filesystem := NewCanonical() + filesystem.SetNativeRoot(filepath.Join(root, "native")) + source := bytes.Repeat([]byte("{\"legacy_stream\":true}\n"), (5<<20)/24+1) + virtualPath := "/archived_sessions/legacy-stream.jsonl" + if err := filesystem.AddSessionAt("legacy-stream", virtualPath, mountSessionFixture(t, "legacy-stream", source)); err != nil { + t.Fatal(err) + } + client, stop := startNativeFSKitTestServer(t, filesystem, root) + _ = client.Close() + defer stop() + descriptor := readNativeFDTestDescriptor(t, root) + connection := dialNativeFDTestSocket(t, descriptor) + defer connection.Close() + callNativeFDTestHelloCapabilities(t, connection, descriptor, 0) + + handle := openNativeFDTestPath(t, connection, descriptor, 2, virtualPath) + response := callNativeFDTestRead(t, connection, descriptor, 3, handle, 0, nativeFSKitSharedReadMinimumBytes) + if response.Flags&fskitproto.FlagSharedReadFD != 0 { + t.Fatalf("legacy read unexpectedly transferred shared FD: flags=%#x", response.Flags) + } + decoder := fskitproto.NewDecoder(response.Payload) + got, err := decoder.Bytes(nativeFSKitSharedReadMinimumBytes) + if err != nil || decoder.Done() != nil || !bytes.Equal(got, source[:nativeFSKitSharedReadMinimumBytes]) { + t.Fatalf("legacy byte stream bytes=%d err=%v", len(got), err) + } + callNativeFDTestHandle(t, connection, descriptor, 4, fskitproto.OpRelease, handle) +} + +func TestNativeFSKitServerFallsBackWhenSharedReadFDPreparationFails(t *testing.T) { + root := t.TempDir() + filesystem := NewCanonical() + filesystem.SetNativeRoot(filepath.Join(root, "native")) + source := bytes.Repeat([]byte("{\"shared_fallback\":true}\n"), (5<<20)/27+1) + virtualPath := "/archived_sessions/shared-fallback.jsonl" + if err := filesystem.AddSessionAt("shared-fallback", virtualPath, mountSessionFixture(t, "shared-fallback", source)); err != nil { + t.Fatal(err) + } + client, stop := startNativeFSKitTestServer(t, filesystem, root) + _ = client.Close() + defer stop() + descriptor := readNativeFDTestDescriptor(t, root) + connection := dialNativeFDTestSocket(t, descriptor) + defer connection.Close() + callNativeFDTestHelloCapabilities(t, connection, descriptor, fskitproto.CapabilitySharedReadFD) + handle := openNativeFDTestPath(t, connection, descriptor, 2, virtualPath) + + originalCreate := createNativeSharedReadObject + createNativeSharedReadObject = func() (nativeSharedReadObject, error) { + return nativeSharedReadObject{}, errors.New("injected shared read object failure") + } + defer func() { createNativeSharedReadObject = originalCreate }() + response := callNativeFDTestRead(t, connection, descriptor, 3, handle, 0, nativeFSKitSharedReadMinimumBytes) + if response.Flags&fskitproto.FlagSharedReadFD != 0 { + t.Fatalf("failed shared read preparation still set FD flag: %#x", response.Flags) + } + decoder := fskitproto.NewDecoder(response.Payload) + got, err := decoder.Bytes(nativeFSKitSharedReadMinimumBytes) + if err != nil || decoder.Done() != nil || !bytes.Equal(got, source[:nativeFSKitSharedReadMinimumBytes]) { + t.Fatalf("shared read fallback bytes=%d err=%v", len(got), err) + } + callNativeFDTestHandle(t, connection, descriptor, 4, fskitproto.OpRelease, handle) +} + +func TestPrepareNativeSharedReadFDUnlinksBeforePopulation(t *testing.T) { + directory := t.TempDir() + var temporaryPath string + originalCreate := createNativeSharedReadObject + createNativeSharedReadObject = func() (nativeSharedReadObject, error) { + file, err := os.CreateTemp(directory, ".codexfold-shared-read-*") + if err == nil { + temporaryPath = file.Name() + } + return nativeSharedReadObject{file: file, unlink: func() error { return os.Remove(temporaryPath) }}, err + } + defer func() { createNativeSharedReadObject = originalCreate }() + want := bytes.Repeat([]byte("mapped-shared-read"), 1024) + file, count, err := prepareNativeSharedReadFD(len(want), func(mapping []byte) (int, error) { + if _, statErr := os.Stat(temporaryPath); !errors.Is(statErr, os.ErrNotExist) { + return 0, errors.New("shared read file remained linked during population") + } + return copy(mapping, want), nil + }) + if err != nil { + t.Fatal(err) + } + if count != len(want) { + t.Fatalf("populated bytes = %d, want %d", count, len(want)) + } + assertMappedNativeFDTestBytes(t, int(file.Fd()), want) + if err := file.Close(); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("shared read left %d temporary directory entries", len(entries)) + } +} + +func TestNativePOSIXSharedMemoryCanBeUnlinkedMappedAndTransferred(t *testing.T) { + object, err := newNativePOSIXSharedMemory() + if err != nil { + t.Fatal(err) + } + defer object.file.Close() + if err := object.unlink(); err != nil { + t.Fatal(err) + } + want := bytes.Repeat([]byte("posix-shared-memory"), 1024) + mappedLength := nativeSharedReadMappedLength(len(want)) + if err := object.file.Truncate(int64(mappedLength)); err != nil { + t.Fatal(err) + } + mapping, err := unix.Mmap(int(object.file.Fd()), 0, mappedLength, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED) + if err != nil { + t.Fatal(err) + } + copy(mapping, want) + if err := unix.Munmap(mapping); err != nil { + t.Fatal(err) + } + assertMappedNativeFDTestBytes(t, int(object.file.Fd()), want) +} + +func TestPrepareNativeSharedReadFDUsesPOSIXSharedMemory(t *testing.T) { + want := bytes.Repeat([]byte("prepared-posix-shared-memory"), 1<<18) + file, count, err := prepareNativeSharedReadFD(len(want), func(mapping []byte) (int, error) { + return copy(mapping, want), nil + }) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if count != len(want) { + t.Fatalf("prepared bytes = %d, want %d", count, len(want)) + } + assertMappedNativeFDTestBytes(t, int(file.Fd()), want) +} + +func unixSocketOption(t *testing.T, connection *net.UnixConn, option int) int { + t.Helper() + raw, err := connection.SyscallConn() + if err != nil { + t.Fatal(err) + } + var value int + var socketErr error + if err := raw.Control(func(descriptor uintptr) { + value, socketErr = syscall.GetsockoptInt(int(descriptor), syscall.SOL_SOCKET, option) + }); err != nil { + t.Fatal(err) + } + if socketErr != nil { + t.Fatal(socketErr) + } + return value +} + +func TestNativeFSKitServerFallsBackWhenReadFDPreparationFailsBeforeResponse(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + wirePath := "/sessions/2026/07/17/fallback.jsonl" + nativePath := nativePathFromRoot(nativeRoot, wirePath) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(nativePath, []byte("{\"record\":true}\n"), 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + coreHandle, errno := filesystem.Open(wirePath, os.O_RDONLY) + if errno != 0 { + t.Fatalf("open native file errno=%v", errno) + } + defer filesystem.Release(coreHandle) + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + + serverConnection, peerConnection := net.Pipe() + _ = peerConnection.Close() + defer serverConnection.Close() + connection := &nativeFSKitConnection{ + server: &nativeFSKitServer{ + filesystem: filesystem, + generation: 1, + maxPayload: fskitproto.DefaultMaxPayload, + }, + conn: serverConnection, + handles: map[uint64]*nativeFSKitHandle{ + 7: {coreHandle: coreHandle, path: wirePath, flags: os.O_RDONLY, snapshotFloor: -1}, + }, + capabilities: fskitproto.CapabilityNativeReadFD, + } + payload := fskitproto.NewEncoder(8) + payload.Uint64(7) + handled, responseWritten, err := connection.writeOpenResponseWithNativeFD(fskitproto.Frame{ + Kind: fskitproto.KindResponse, Op: fskitproto.OpOpen, RequestID: 2, Generation: 1, Payload: payload.Data(), + }) + if err != nil || handled || responseWritten { + t.Fatalf("native FD pre-send failure handled=%t response_written=%t err=%v, want byte-stream fallback", handled, responseWritten, err) + } +} + +func dialNativeFDTestSocket(t *testing.T, descriptor fskitproto.Descriptor) *net.UnixConn { + t.Helper() + address := &net.UnixAddr{Name: descriptor.SocketPath, Net: "unix"} + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + connection, err := net.DialUnix("unix", nil, address) + if err == nil { + return connection + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("dial native FD test socket: %s", descriptor.SocketPath) + return nil +} + +func callNativeFDTestHello(t *testing.T, connection *net.UnixConn, descriptor fskitproto.Descriptor, capability bool) { + t.Helper() + var capabilities uint32 + if capability { + capabilities = fskitproto.CapabilityNativeReadFD + } + callNativeFDTestHelloCapabilities(t, connection, descriptor, capabilities) +} + +func callNativeFDTestHelloCapabilities(t *testing.T, connection *net.UnixConn, descriptor fskitproto.Descriptor, capabilities uint32) { + t.Helper() + encoder := fskitproto.NewEncoder(64) + encoder.Bytes(descriptor.Token) + if capabilities != 0 { + encoder.Uint32(capabilities) + } + response := callNativeFDTestFrame(t, connection, descriptor, 1, fskitproto.OpHello, encoder.Data()) + decoder := fskitproto.NewDecoder(response.Payload) + if maxPayload, err := decoder.Uint32(); err != nil || maxPayload < 4096 { + t.Fatalf("hello max payload=%d err=%v", maxPayload, err) + } + if _, err := decoder.Uint64(); err != nil || decoder.Done() != nil { + t.Fatalf("hello response decode: %v", err) + } +} + +func readNativeFDTestDescriptor(t *testing.T, root string) fskitproto.Descriptor { + t.Helper() + data, err := os.ReadFile(filepath.Join(root, "resource.bin")) + if err != nil { + t.Fatal(err) + } + descriptor, err := fskitproto.DecodeDescriptor(data) + if err != nil { + t.Fatal(err) + } + return descriptor +} + +func openNativeFDTestPath(t *testing.T, connection *net.UnixConn, descriptor fskitproto.Descriptor, requestID uint64, path string) uint64 { + t.Helper() + payload := fskitproto.NewEncoder(128) + payload.String(path) + payload.Uint32(uint32(os.O_RDONLY)) + response := callNativeFDTestFrame(t, connection, descriptor, requestID, fskitproto.OpOpen, payload.Data()) + decoder := fskitproto.NewDecoder(response.Payload) + handle, err := decoder.Uint64() + if err != nil || decoder.Done() != nil { + t.Fatalf("decode open handle=%d err=%v", handle, err) + } + return handle +} + +func callNativeFDTestRead(t *testing.T, connection *net.UnixConn, descriptor fskitproto.Descriptor, requestID, handle uint64, offset int64, length int) fskitproto.Frame { + t.Helper() + payload := fskitproto.NewEncoder(24) + payload.Uint64(handle) + payload.Int64(offset) + payload.Uint32(uint32(length)) + return callNativeFDTestFrame(t, connection, descriptor, requestID, fskitproto.OpRead, payload.Data()) +} + +func decodeNativeFDTestReadCount(t *testing.T, response fskitproto.Frame) int { + t.Helper() + decoder := fskitproto.NewDecoder(response.Payload) + count, err := decoder.Uint32() + if err != nil || decoder.Done() != nil { + t.Fatalf("decode shared read count=%d err=%v", count, err) + } + return int(count) +} + +func callNativeFDTestFrame(t *testing.T, connection *net.UnixConn, descriptor fskitproto.Descriptor, requestID uint64, operation fskitproto.Op, payload []byte) fskitproto.Frame { + t.Helper() + if err := fskitproto.WriteFrame(connection, fskitproto.Frame{ + Kind: fskitproto.KindRequest, Op: operation, RequestID: requestID, + Generation: func() uint64 { + if operation == fskitproto.OpHello { + return 0 + } + return descriptor.Generation + }(), Payload: payload, + }, fskitproto.DefaultMaxPayload); err != nil { + t.Fatal(err) + } + response, err := fskitproto.ReadFrame(connection, fskitproto.DefaultMaxPayload) + if err != nil { + t.Fatalf("read %v response: %v", operation, err) + } + if response.Kind != fskitproto.KindResponse || response.Op != operation || response.RequestID != requestID { + t.Fatalf("unexpected %v response: %#v", operation, response) + } + if response.Status != 0 { + t.Fatalf("%v response status=%d", operation, response.Status) + } + return response +} + +func callNativeFDTestHandle(t *testing.T, connection *net.UnixConn, descriptor fskitproto.Descriptor, requestID uint64, operation fskitproto.Op, handle uint64) { + t.Helper() + payload := fskitproto.NewEncoder(8) + payload.Uint64(handle) + callNativeFDTestFrame(t, connection, descriptor, requestID, operation, payload.Data()) +} + +func receiveNativeFDTestMarker(t *testing.T, connection *net.UnixConn) int { + t.Helper() + return receiveFDTestMarker(t, connection, fskitproto.NativeReadFDMarker) +} + +func receiveFDTestMarker(t *testing.T, connection *net.UnixConn, expectedMarker byte) int { + t.Helper() + marker := make([]byte, 1) + oob := make([]byte, syscall.CmsgSpace(4)) + n, oobn, flags, _, err := connection.ReadMsgUnix(marker, oob) + if err != nil { + t.Fatalf("receive native FD marker: %v", err) + } + if n != 1 || marker[0] != expectedMarker || flags&(syscall.MSG_CTRUNC|syscall.MSG_TRUNC) != 0 { + t.Fatalf("native FD marker n=%d marker=%#x flags=%#x", n, marker, flags) + } + messages, err := syscall.ParseSocketControlMessage(oob[:oobn]) + if err != nil || len(messages) != 1 { + t.Fatalf("parse native FD control messages: count=%d err=%v", len(messages), err) + } + fds, err := syscall.ParseUnixRights(&messages[0]) + if err != nil || len(fds) != 1 || fds[0] < 0 { + t.Fatalf("parse native FD rights: fds=%v err=%v", fds, err) + } + return fds[0] +} + +func assertMappedNativeFDTestBytes(t *testing.T, fd int, want []byte, maximumBytes ...int) { + t.Helper() + var info unix.Stat_t + if err := unix.Fstat(fd, &info); err != nil { + t.Fatal(err) + } + mappedLength := nativeSharedReadMappedLength(len(want)) + maximumLength := mappedLength + if len(maximumBytes) != 0 { + maximumLength = nativeSharedReadMappedLength(maximumBytes[0]) + } + if info.Size < int64(len(want)) || info.Size > int64(maximumLength) { + t.Fatalf("shared FD size = %d, want between %d and %d", info.Size, len(want), maximumLength) + } + mapping, err := unix.Mmap(fd, 0, mappedLength, unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(mapping[:len(want)], want) { + _ = unix.Munmap(mapping) + t.Fatalf("mapped shared FD bytes differ: got=%d want=%d", len(mapping), len(want)) + } + if err := unix.Munmap(mapping); err != nil { + t.Fatal(err) + } +} diff --git a/internal/mountfs/native_fskit_metadata_darwin_test.go b/internal/mountfs/native_fskit_metadata_darwin_test.go new file mode 100644 index 0000000..e99a5c8 --- /dev/null +++ b/internal/mountfs/native_fskit_metadata_darwin_test.go @@ -0,0 +1,139 @@ +//go:build darwin + +package mountfs + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "slices" + "syscall" + "testing" + "time" + + "github.com/samekind/codexfold/internal/fskitproto" +) + +func TestNativeFSKitServerPersistsMetadataAndExtendedAttributes(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + directory := filepath.Join(nativeRoot, "sessions", "2026", "07", "17") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(nativeRoot, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(directory, "session.jsonl") + if err := os.WriteFile(target, []byte("{}\n"), 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + defer stop() + defer client.Close() + + path := "/sessions/2026/07/17/session.jsonl" + atime := time.Unix(1_720_000_000, 123_000_000) + mtime := time.Unix(1_720_000_100, 456_000_000) + setattr := fskitproto.NewEncoder(160) + setattr.String(path) + setattr.Uint32(fskitproto.SetAttrMode | fskitproto.SetAttrUID | fskitproto.SetAttrGID | fskitproto.SetAttrAccessTime | fskitproto.SetAttrModifyTime) + setattr.Uint32(0o640) + setattr.Uint32(uint32(os.Getuid())) + setattr.Uint32(uint32(os.Getgid())) + setattr.Time(atime) + setattr.Time(mtime) + if _, err := client.Call(fskitproto.OpSetattr, setattr.Data()); err != nil { + t.Fatalf("setattr: %v", err) + } + + getattr := fskitproto.NewEncoder(128) + getattr.String(path) + response, err := client.Call(fskitproto.OpGetattr, getattr.Data()) + if err != nil { + t.Fatal(err) + } + decoder := fskitproto.NewDecoder(response) + entry, err := decoder.Entry() + if err != nil || decoder.Done() != nil { + t.Fatalf("decode getattr: %v", err) + } + if entry.Mode != 0o640 || entry.UID != uint32(os.Getuid()) || entry.GID != uint32(os.Getgid()) { + t.Fatalf("metadata = mode %#o uid %d gid %d", entry.Mode, entry.UID, entry.GID) + } + if !entry.AccessTime.Equal(atime) || !entry.ModTime.Equal(mtime) { + t.Fatalf("times = atime %s mtime %s, want %s and %s", entry.AccessTime, entry.ModTime, atime, mtime) + } + + attribute := "vip.jstar.codexfold.test" + value := []byte("first") + setXattr := fskitproto.NewEncoder(160) + setXattr.String(path) + setXattr.String(attribute) + setXattr.Uint32(uint32(fskitproto.XattrAlwaysSet)) + setXattr.Bytes(value) + if _, err := client.Call(fskitproto.OpSetXattr, setXattr.Data()); err != nil { + t.Fatalf("set xattr: %v", err) + } + + getXattr := fskitproto.NewEncoder(160) + getXattr.String(path) + getXattr.String(attribute) + response, err = client.Call(fskitproto.OpGetXattr, getXattr.Data()) + if err != nil { + t.Fatalf("get xattr: %v", err) + } + decoder = fskitproto.NewDecoder(response) + got, err := decoder.Bytes(1024) + if err != nil || decoder.Done() != nil || !bytes.Equal(got, value) { + t.Fatalf("xattr value = %q err=%v", got, err) + } + + createAgain := fskitproto.NewEncoder(160) + createAgain.String(path) + createAgain.String(attribute) + createAgain.Uint32(uint32(fskitproto.XattrMustCreate)) + createAgain.Bytes([]byte("duplicate")) + if _, err := client.Call(fskitproto.OpSetXattr, createAgain.Data()); fskitproto.ErrorNumber(err) != syscall.EEXIST { + t.Fatalf("must-create error = %v, want EEXIST", err) + } + + list := fskitproto.NewEncoder(128) + list.String(path) + response, err = client.Call(fskitproto.OpListXattrs, list.Data()) + if err != nil { + t.Fatalf("list xattrs: %v", err) + } + decoder = fskitproto.NewDecoder(response) + count, err := decoder.Uint32() + if err != nil { + t.Fatal(err) + } + names := make([]string, 0, count) + for range count { + name, decodeErr := decoder.String(4096) + if decodeErr != nil { + t.Fatal(decodeErr) + } + names = append(names, name) + } + if err := decoder.Done(); err != nil || !slices.Contains(names, attribute) { + t.Fatalf("xattr names = %v err=%v", names, err) + } + + remove := fskitproto.NewEncoder(160) + remove.String(path) + remove.String(attribute) + remove.Uint32(uint32(fskitproto.XattrDelete)) + remove.Bytes(nil) + if _, err := client.Call(fskitproto.OpSetXattr, remove.Data()); err != nil { + t.Fatalf("remove xattr: %v", err) + } + if _, err := client.Call(fskitproto.OpGetXattr, getXattr.Data()); !errors.Is(err, fskitproto.StatusError{Operation: fskitproto.OpGetXattr, Errno: syscall.ENOATTR}) && fskitproto.ErrorNumber(err) != syscall.ENOATTR { + t.Fatalf("removed xattr error = %v, want ENOATTR", err) + } +} diff --git a/internal/mountfs/native_fskit_mount_darwin_test.go b/internal/mountfs/native_fskit_mount_darwin_test.go new file mode 100644 index 0000000..ba7fd7e --- /dev/null +++ b/internal/mountfs/native_fskit_mount_darwin_test.go @@ -0,0 +1,1375 @@ +//go:build darwin + +package mountfs + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/samekind/codexfold/internal/fskitproto" + "golang.org/x/sys/unix" +) + +const ( + nativeFSKitMountEnv = "CODEXFOLD_NATIVE_FSKIT_MOUNT" + nativeFSKitNativeRootEnv = "CODEXFOLD_NATIVE_FSKIT_NATIVE_ROOT" + nativeFSKitResourceEnv = "CODEXFOLD_NATIVE_FSKIT_RESOURCE" + nativeFSKitVirtualFileEnv = "CODEXFOLD_NATIVE_FSKIT_VIRTUAL_FILE" + nativeFSKitVirtualReferenceFileEnv = "CODEXFOLD_NATIVE_FSKIT_VIRTUAL_REFERENCE_FILE" +) + +func TestNativeFSKitMountedMetadataAndXattrs(t *testing.T) { + root := nativeFSKitMountedTestRoot(t) + target := filepath.Join(root, "metadata.bin") + writeMountedTestFile(t, target, []byte("{}\n")) + + if err := os.Chmod(target, 0o640); err != nil { + t.Fatalf("chmod: %v", err) + } + if err := os.Chown(target, os.Getuid(), os.Getgid()); err != nil { + t.Fatalf("chown: %v", err) + } + atime := time.Unix(1_720_000_000, 123_000_000) + mtime := time.Unix(1_720_000_100, 456_000_000) + if err := os.Chtimes(target, atime, mtime); err != nil { + t.Fatalf("chtimes: %v", err) + } + info, err := os.Stat(target) + if err != nil { + t.Fatal(err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatalf("stat payload = %T", info.Sys()) + } + if info.Mode().Perm() != 0o640 || stat.Uid != uint32(os.Getuid()) || stat.Gid != uint32(os.Getgid()) { + t.Fatalf("metadata = mode %#o uid %d gid %d", info.Mode().Perm(), stat.Uid, stat.Gid) + } + if !info.ModTime().Equal(mtime) { + t.Fatalf("mtime = %s, want %s", info.ModTime(), mtime) + } + gotAtime := time.Unix(stat.Atimespec.Sec, stat.Atimespec.Nsec) + if !gotAtime.Equal(atime) { + t.Fatalf("atime = %s, want %s", gotAtime, atime) + } + + attribute := "vip.jstar.codexfold.integration" + first := []byte("first") + if err := unix.Setxattr(target, attribute, first, unix.XATTR_CREATE); err != nil { + t.Fatalf("create xattr: %v", err) + } + if err := unix.Setxattr(target, attribute, []byte("duplicate"), unix.XATTR_CREATE); !errors.Is(err, syscall.EEXIST) { + t.Fatalf("duplicate create xattr error = %v, want EEXIST", err) + } + second := []byte("second") + if err := unix.Setxattr(target, attribute, second, unix.XATTR_REPLACE); err != nil { + t.Fatalf("replace xattr: %v", err) + } + if got := mountedTestXattr(t, target, attribute); !bytes.Equal(got, second) { + t.Fatalf("xattr value = %q, want %q", got, second) + } + if names := mountedTestXattrNames(t, target); !slices.Contains(names, attribute) { + t.Fatalf("xattr names = %v, missing %s", names, attribute) + } + finderInfo := make([]byte, 32) + copy(finderInfo, []byte("CodexFold-FSKit")) + if err := unix.Setxattr(target, "com.apple.FinderInfo", finderInfo, 0); err != nil { + t.Fatalf("set FinderInfo xattr: %v", err) + } + if got := mountedTestXattr(t, target, "com.apple.FinderInfo"); !bytes.Equal(got, finderInfo) { + t.Fatalf("FinderInfo xattr = %x, want %x", got, finderInfo) + } + if _, err := os.Stat(filepath.Join(filepath.Dir(target), "._"+filepath.Base(target))); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unexpected AppleDouble sidecar: %v", err) + } + if err := unix.Removexattr(target, attribute); err != nil { + t.Fatalf("remove xattr: %v", err) + } + if _, err := readMountedTestXattr(target, attribute); !errors.Is(err, syscall.ENOATTR) { + t.Fatalf("removed xattr error = %v, want ENOATTR", err) + } + if err := unix.Setxattr(target, attribute, []byte("missing"), unix.XATTR_REPLACE); !errors.Is(err, syscall.ENOATTR) { + t.Fatalf("replace missing xattr error = %v, want ENOATTR", err) + } +} + +func TestNativeFSKitMountedWritesAndFullSync(t *testing.T) { + root := nativeFSKitMountedTestRoot(t) + target := filepath.Join(root, "writes.bin") + writeMountedTestFile(t, target, []byte("0123456789\n")) + + file, err := os.OpenFile(target, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteAt([]byte("AB"), 2); err != nil { + file.Close() + t.Fatalf("random overwrite: %v", err) + } + if err := file.Sync(); err != nil { + file.Close() + t.Fatalf("fsync: %v", err) + } + if _, err := unix.FcntlInt(file.Fd(), unix.F_FULLFSYNC, 0); err != nil { + file.Close() + t.Fatalf("F_FULLFSYNC: %v", err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + assertMountedTestContent(t, target, []byte("01AB456789\n")) + + if err := os.Truncate(target, 6); err != nil { + t.Fatalf("truncate: %v", err) + } + appendFile, err := os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if _, err := appendFile.Write([]byte("TAIL")); err != nil { + appendFile.Close() + t.Fatalf("append: %v", err) + } + if err := appendFile.Sync(); err != nil { + appendFile.Close() + t.Fatalf("append fsync: %v", err) + } + if err := appendFile.Close(); err != nil { + t.Fatal(err) + } + assertMountedTestContent(t, target, []byte("01AB45TAIL")) + + oldEOF := filepath.Join(root, "old-eof.jsonl") + base := []byte("{\"record\":0}\n") + first := []byte("{\"record\":1}\n") + second := []byte("{\"record\":2}\n") + third := []byte("{\"record\":3}\n") + writeMountedTestFile(t, oldEOF, base) + oldEOFFile, err := os.OpenFile(oldEOF, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := oldEOFFile.WriteAt(first, int64(len(base))); err != nil { + oldEOFFile.Close() + t.Fatalf("first old-EOF write: %v", err) + } + if _, err := oldEOFFile.WriteAt(second, int64(len(base))); err != nil { + oldEOFFile.Close() + t.Fatalf("second old-EOF write: %v", err) + } + if _, err := oldEOFFile.WriteAt(third, int64(len(base))); err != nil { + oldEOFFile.Close() + t.Fatalf("third old-EOF write after cache revoke: %v", err) + } + if err := oldEOFFile.Close(); err != nil { + t.Fatal(err) + } + want := append(append(append(append([]byte(nil), base...), first...), second...), third...) + assertNativeFSKitBackingContent(t, oldEOF, want) + assertMountedTestContent(t, oldEOF, want) +} + +func TestNativeFSKitMountedOverlappingOpenLifetime(t *testing.T) { + root := nativeFSKitMountedTestRoot(t) + target := filepath.Join(root, "overlapping-open.jsonl") + writeMountedTestFile(t, target, []byte("{\"record\":0}\n")) + + first, err := os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + second, err := os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + first.Close() + t.Fatal(err) + } + if err := first.Close(); err != nil { + second.Close() + t.Fatal(err) + } + appended := []byte("{\"record\":1}\n") + if _, err := second.Write(appended); err != nil { + second.Close() + t.Fatalf("write through surviving open descriptor: %v", err) + } + if err := second.Sync(); err != nil { + second.Close() + t.Fatalf("sync through surviving open descriptor: %v", err) + } + if err := second.Close(); err != nil { + t.Fatal(err) + } + assertMountedTestContent(t, target, append([]byte("{\"record\":0}\n"), appended...)) +} + +func TestNativeFSKitMountedNamespaceOperations(t *testing.T) { + root := nativeFSKitMountedTestRoot(t) + source := filepath.Join(root, "source.jsonl") + destination := filepath.Join(root, "destination.jsonl") + writeMountedTestFile(t, source, []byte("source\n")) + writeMountedTestFile(t, destination, []byte("destination\n")) + if err := os.Rename(source, destination); err != nil { + t.Fatalf("overwrite rename: %v", err) + } + if _, err := os.Stat(source); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("renamed source still exists: %v", err) + } + assertMountedTestContent(t, destination, []byte("source\n")) + + nested := filepath.Join(root, "nested", "child") + if err := os.MkdirAll(nested, 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.Remove(nested); err != nil { + t.Fatalf("rmdir child: %v", err) + } + if err := os.Remove(filepath.Dir(nested)); err != nil { + t.Fatalf("rmdir parent: %v", err) + } + + if err := os.Symlink(destination, filepath.Join(root, "symbolic")); !isNotSupported(err) { + t.Fatalf("symlink error = %v, want ENOTSUP", err) + } + if err := os.Link(destination, filepath.Join(root, "hard")); !isNotSupported(err) { + t.Fatalf("hardlink error = %v, want ENOTSUP", err) + } +} + +func TestNativeFSKitMountedSamePathRecreate(t *testing.T) { + root := nativeFSKitMountedTestRoot(t) + target := filepath.Join(root, "same-path.jsonl") + for round := 1; round <= 20; round++ { + first := []byte(fmt.Sprintf("{\"round\":%d,\"phase\":\"first\"}\n", round)) + second := []byte(fmt.Sprintf("{\"round\":%d,\"phase\":\"second\"}\n", round)) + writeMountedTestFile(t, target, first) + assertMountedTestContent(t, target, first) + if err := os.Remove(target); err != nil { + t.Fatalf("round %d remove first generation: %v", round, err) + } + if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("round %d removed path stat = %v, want ENOENT", round, err) + } + writeMountedTestFile(t, target, second) + assertMountedTestContent(t, target, second) + if err := os.Remove(target); err != nil { + t.Fatalf("round %d remove second generation: %v", round, err) + } + } +} + +func TestNativeFSKitMountedArchiveRoundTripAndMmap(t *testing.T) { + mountPoint := nativeFSKitMountPoint(t) + relativeDirectory := filepath.Join("2099", "12", "31") + activeDirectory := filepath.Join(mountPoint, "sessions", relativeDirectory) + archiveDirectory := filepath.Join(mountPoint, "archived_sessions", relativeDirectory) + if err := os.MkdirAll(activeDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(archiveDirectory, 0o700); err != nil { + t.Fatal(err) + } + name := fmt.Sprintf("archive-roundtrip-%d.jsonl", time.Now().UnixNano()) + active := filepath.Join(activeDirectory, name) + archived := filepath.Join(archiveDirectory, name) + t.Cleanup(func() { + _ = os.Remove(active) + _ = os.Remove(archived) + }) + content := []byte("{\"archive\":true}\n") + writeMountedTestFile(t, active, content) + + file, err := os.Open(active) + if err != nil { + t.Fatal(err) + } + mapped, err := unix.Mmap(int(file.Fd()), 0, len(content), unix.PROT_READ, unix.MAP_PRIVATE) + if err != nil { + file.Close() + t.Fatalf("mmap: %v", err) + } + if !bytes.Equal(mapped, content) { + unix.Munmap(mapped) + file.Close() + t.Fatalf("mmap bytes = %q, want %q", mapped, content) + } + if err := unix.Munmap(mapped); err != nil { + file.Close() + t.Fatalf("munmap: %v", err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + + if err := os.Rename(active, archived); err != nil { + t.Fatalf("archive rename: %v", err) + } + assertMountedTestContent(t, archived, content) + if err := os.Rename(archived, active); err != nil { + t.Fatalf("unarchive rename: %v", err) + } + assertMountedTestContent(t, active, content) +} + +func TestNativeFSKitMountedOpenUnlink(t *testing.T) { + root := nativeFSKitMountedTestRoot(t) + target := filepath.Join(root, "open-unlink.jsonl") + writeMountedTestFile(t, target, []byte("before")) + + file, err := os.OpenFile(target, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(target); err != nil { + file.Close() + t.Fatalf("unlink open file: %v", err) + } + if _, err := file.WriteAt([]byte("after"), 0); err != nil { + file.Close() + t.Fatalf("write unlinked file: %v", err) + } + buffer := make([]byte, 6) + if _, err := file.ReadAt(buffer, 0); err != nil { + file.Close() + t.Fatalf("read unlinked file: %v", err) + } + if !bytes.Equal(buffer, []byte("aftere")) { + file.Close() + t.Fatalf("unlinked content = %q", buffer) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unlinked path exists after close: %v", err) + } +} + +func TestNativeFSKitMountedExternalNamespaceRefresh(t *testing.T) { + mountPoint := nativeFSKitMountPoint(t) + nativeRoot := os.Getenv(nativeFSKitNativeRootEnv) + if nativeRoot == "" { + t.Skipf("set %s to run external namespace refresh", nativeFSKitNativeRootEnv) + } + if !filepath.IsAbs(nativeRoot) { + t.Fatalf("%s must be absolute", nativeFSKitNativeRootEnv) + } + + relative := filepath.Join("sessions", "2099", "12", "31", fmt.Sprintf("external-%d.bin", time.Now().UnixNano())) + nativePath := filepath.Join(nativeRoot, relative) + mountedPath := filepath.Join(mountPoint, relative) + parentRoute := "/" + filepath.ToSlash(filepath.Dir(relative)) + version, observeVersion := nativeFSKitMountedNamespaceVersion(t) + if err := os.MkdirAll(filepath.Dir(nativePath), 0o700); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Remove(nativePath) }) + // Prime the mounted kernel's negative name cache before the native-side + // creator appears. This is the path a real Codex restart or file watcher + // hits after observing a session path before its rollout exists. + if _, err := os.Stat(mountedPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("uncreated mounted path stat = %v, want not exist", err) + } + if err := os.WriteFile(nativePath, []byte("external-one\n"), 0o600); err != nil { + t.Fatal(err) + } + if observeVersion { + version = waitForNativeFSKitNamespaceAdvance(t, version, 3*time.Second) + logNativeFSKitMountedEntry(t, "after create", parentRoute) + } + waitForMountedContent(t, mountedPath, []byte("external-one\n"), 3*time.Second) + + if err := os.Remove(nativePath); err != nil { + t.Fatal(err) + } + if observeVersion { + version = waitForNativeFSKitNamespaceAdvance(t, version, 3*time.Second) + logNativeFSKitMountedEntry(t, "after remove", parentRoute) + } + waitForMountedAbsence(t, mountedPath, 3*time.Second) + if err := os.WriteFile(nativePath, []byte("external-two\n"), 0o600); err != nil { + t.Fatal(err) + } + if observeVersion { + version = waitForNativeFSKitNamespaceAdvance(t, version, 3*time.Second) + logNativeFSKitMountedEntry(t, "after recreate", parentRoute) + t.Logf("native namespace reached version %d after external recreation", version) + } + waitForMountedContent(t, mountedPath, []byte("external-two\n"), 3*time.Second) +} + +func TestNativeFSKitMountedExternalDirectoryNamespaceRefresh(t *testing.T) { + mountPoint := nativeFSKitMountPoint(t) + nativeRoot := os.Getenv(nativeFSKitNativeRootEnv) + if nativeRoot == "" { + t.Skipf("set %s to run external directory refresh", nativeFSKitNativeRootEnv) + } + relative := filepath.Join("sessions", "2099", "12", "31", fmt.Sprintf("external-directory-%d", time.Now().UnixNano())) + nativeDirectory := filepath.Join(nativeRoot, relative) + mountedDirectory := filepath.Join(mountPoint, relative) + mountedChild := filepath.Join(mountedDirectory, "child.jsonl") + if _, err := os.Stat(mountedDirectory); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("uncreated mounted directory stat = %v, want not exist", err) + } + if _, err := os.Stat(mountedChild); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("uncreated mounted child stat = %v, want not exist", err) + } + + version, observeVersion := nativeFSKitMountedNamespaceVersion(t) + if err := os.MkdirAll(nativeDirectory, 0o750); err != nil { + t.Fatal(err) + } + nativeChild := filepath.Join(nativeDirectory, "child.jsonl") + want := []byte("external-directory-content\n") + if err := os.WriteFile(nativeChild, want, 0o640); err != nil { + t.Fatal(err) + } + before, err := os.Stat(nativeChild) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(nativeDirectory) }) + if observeVersion { + version = waitForNativeFSKitNamespaceAdvance(t, version, 3*time.Second) + } + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if info, statErr := os.Stat(mountedDirectory); statErr == nil && info.IsDir() { + break + } + time.Sleep(50 * time.Millisecond) + } + if info, err := os.Stat(mountedDirectory); err != nil || !info.IsDir() { + t.Fatalf("mounted external directory = %v, want a directory", err) + } + waitForMountedContent(t, mountedChild, want, 3*time.Second) + mountedInfo, err := os.Stat(mountedChild) + if err != nil { + t.Fatal(err) + } + if mountedInfo.Mode().Perm() != before.Mode().Perm() || mountedInfo.Size() != before.Size() { + t.Fatalf("mounted child metadata mode=%#o size=%d, want mode=%#o size=%d", mountedInfo.Mode().Perm(), mountedInfo.Size(), before.Mode().Perm(), before.Size()) + } + + if err := os.RemoveAll(nativeDirectory); err != nil { + t.Fatal(err) + } + if observeVersion { + version = waitForNativeFSKitNamespaceAdvance(t, version, 3*time.Second) + } + waitForMountedAbsence(t, mountedChild, 3*time.Second) + waitForMountedAbsence(t, mountedDirectory, 3*time.Second) +} + +func TestNativeFSKitMountedPerformance(t *testing.T) { + mountPoint := nativeFSKitMountPoint(t) + nativeRoot := os.Getenv(nativeFSKitNativeRootEnv) + if nativeRoot == "" { + t.Skipf("set %s to run native FSKit performance", nativeFSKitNativeRootEnv) + } + root := nativeFSKitMountedTestRoot(t) + relativeRoot, err := filepath.Rel(mountPoint, root) + if err != nil { + t.Fatal(err) + } + mountedPath := filepath.Join(root, "performance.bin") + nativePath := filepath.Join(nativeRoot, relativeRoot, "performance.bin") + const sourceBytes = int64(256 << 20) + versionBefore, observeVersion := nativeFSKitMountedNamespaceVersion(t) + if err := writePerformanceFixture(nativePath, sourceBytes); err != nil { + t.Fatal(err) + } + nativeInfo, err := os.Stat(nativePath) + if err != nil { + t.Fatal(err) + } + versionAfterWrite, _ := nativeFSKitMountedNamespaceVersion(t) + mountedInfo, mountedErr := os.Stat(mountedPath) + t.Logf( + "native-fskit performance fixture native_size=%d mounted_size=%d mounted_err=%v namespace_before=%d namespace_after_write=%d observe_version=%t", + nativeInfo.Size(), fileInfoSize(mountedInfo), mountedErr, versionBefore, versionAfterWrite, observeVersion, + ) + logNativeFSKitMountedEntry(t, "performance-after-write", mountedPath) + time.Sleep(time.Second) + versionAfterSettle, _ := nativeFSKitMountedNamespaceVersion(t) + mountedInfo, mountedErr = os.Stat(mountedPath) + t.Logf( + "native-fskit performance settle mounted_size=%d mounted_err=%v namespace_after_settle=%d", + fileInfoSize(mountedInfo), mountedErr, versionAfterSettle, + ) + logNativeFSKitMountedEntry(t, "performance-after-settle", mountedPath) + waitForMountedSize(t, mountedPath, sourceBytes, 5*time.Second) + + nativeCold, nativeBypass, err := sequentialReadMetric(nativePath, true) + if err != nil { + t.Fatal(err) + } + mountedCold, mountedBypass, err := sequentialReadMetric(mountedPath, true) + if err != nil { + t.Fatal(err) + } + nativeWarm, err := medianSequentialThroughput(nativePath, 3) + if err != nil { + t.Fatal(err) + } + mountedWarm, err := medianSequentialThroughput(mountedPath, 3) + if err != nil { + t.Fatal(err) + } + nativeHash, err := streamingSHA256(nativePath) + if err != nil { + t.Fatal(err) + } + mountedHash, err := streamingSHA256(mountedPath) + if err != nil { + t.Fatal(err) + } + if nativeHash != mountedHash { + t.Fatal("mounted performance file differs from the native source") + } + t.Logf( + "native-fskit performance pre-gate bytes=%d cold_native=%.2fMiB/s cold_mounted=%.2fMiB/s cold_ratio=%.3f warm_native=%.2fMiB/s warm_mounted=%.2fMiB/s warm_ratio=%.3f native_nocache=%t mounted_nocache=%t", + sourceBytes, + nativeCold/(1<<20), mountedCold/(1<<20), mountedCold/nativeCold, + nativeWarm/(1<<20), mountedWarm/(1<<20), mountedWarm/nativeWarm, + nativeBypass, mountedBypass, + ) + const minimumThroughput = float64(500 << 20) + if mountedCold < minimumThroughput || mountedWarm < minimumThroughput { + t.Fatalf("mounted throughput below 500 MiB/s: cold=%.2f MiB/s warm=%.2f MiB/s", mountedCold/(1<<20), mountedWarm/(1<<20)) + } + if mountedCold/nativeCold < 0.10 || mountedWarm/nativeWarm < 0.05 { + t.Fatalf("mounted/native throughput ratio too low: cold=%.3f warm=%.3f", mountedCold/nativeCold, mountedWarm/nativeWarm) + } + + latencyPath := filepath.Join(root, "append-fsync.jsonl") + writeMountedTestFile(t, latencyPath, []byte("{}\n")) + file, err := os.OpenFile(latencyPath, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + latencies := make([]time.Duration, 0, 100) + for index := 0; index < 100; index++ { + started := time.Now() + if _, err := fmt.Fprintf(file, "{\"append_fsync\":%d}\n", index); err != nil { + file.Close() + t.Fatal(err) + } + if err := file.Sync(); err != nil { + file.Close() + t.Fatal(err) + } + latencies = append(latencies, time.Since(started)) + } + if _, err := unix.FcntlInt(file.Fd(), unix.F_FULLFSYNC, 0); err != nil { + file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + p50 := durationPercentile(latencies, 0.50) + p95 := durationPercentile(latencies, 0.95) + p99 := durationPercentile(latencies, 0.99) + if p95 > 100*time.Millisecond { + t.Fatalf("append plus fsync p95 exceeded 100 ms: %s", p95) + } + t.Logf( + "native-fskit performance bytes=%d cold_native=%.2fMiB/s cold_mounted=%.2fMiB/s cold_ratio=%.3f warm_native=%.2fMiB/s warm_mounted=%.2fMiB/s warm_ratio=%.3f native_nocache=%t mounted_nocache=%t append_fsync_p50=%s append_fsync_p95=%s append_fsync_p99=%s", + sourceBytes, + nativeCold/(1<<20), mountedCold/(1<<20), mountedCold/nativeCold, + nativeWarm/(1<<20), mountedWarm/(1<<20), mountedWarm/nativeWarm, + nativeBypass, mountedBypass, p50, p95, p99, + ) +} + +func TestNativeFSKitMountedReadAheadCoherency(t *testing.T) { + mountPoint := nativeFSKitMountPoint(t) + nativeRoot := os.Getenv(nativeFSKitNativeRootEnv) + if nativeRoot == "" { + t.Skipf("set %s to run native FSKit read-ahead coherency", nativeFSKitNativeRootEnv) + } + root := nativeFSKitMountedTestRoot(t) + relativeRoot, err := filepath.Rel(mountPoint, root) + if err != nil { + t.Fatal(err) + } + mountedPath := filepath.Join(root, "read-ahead.jsonl") + nativePath := filepath.Join(nativeRoot, relativeRoot, "read-ahead.jsonl") + const blockSize = 1 << 20 + const lineSize = 4096 + prefix := []byte("{\"payload\":\"") + suffix := []byte("\"}\n") + line := make([]byte, lineSize) + copy(line, prefix) + for index := len(prefix); index < len(line)-len(suffix); index++ { + line[index] = byte('a' + index%26) + } + copy(line[len(line)-len(suffix):], suffix) + content := bytes.Repeat(line, 4*blockSize/lineSize) + if !completeJSONL(content) { + t.Fatal("read-ahead fixture is not valid JSONL") + } + writeMountedTestFile(t, mountedPath, content) + + reader, err := os.Open(mountedPath) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + for _, testCase := range []struct { + offset int64 + length int + }{ + {offset: 0, length: 8192}, + {offset: blockSize - 4096, length: 8192}, + {offset: blockSize + 2048, length: 16384}, + {offset: 2*blockSize - 7777, length: 12000}, + {offset: 333333, length: 7777}, + } { + assertOpenFileRange(t, reader, content, testCase.offset, testCase.length) + } + + const concurrentReaders = 8 + const concurrentRounds = 64 + const concurrentReadBytes = 32 << 10 + concurrentErrors := make(chan error, concurrentReaders) + var concurrent sync.WaitGroup + for worker := 0; worker < concurrentReaders; worker++ { + concurrent.Add(1) + go func(worker int) { + defer concurrent.Done() + for round := 0; round < concurrentRounds; round++ { + offset := int64((worker*7919 + round*65537) % (len(content) - concurrentReadBytes)) + buffer := make([]byte, concurrentReadBytes) + n, readErr := reader.ReadAt(buffer, offset) + if readErr != nil || n != len(buffer) || !bytes.Equal(buffer, content[offset:offset+int64(len(buffer))]) { + concurrentErrors <- fmt.Errorf("worker=%d round=%d offset=%d bytes=%d error=%v", worker, round, offset, n, readErr) + return + } + } + }(worker) + } + concurrent.Wait() + close(concurrentErrors) + for concurrentErr := range concurrentErrors { + t.Fatal(concurrentErr) + } + + eofOffset := int64(len(content) - 100) + eofBuffer := make([]byte, 4096) + n, err := reader.ReadAt(eofBuffer, eofOffset) + if !errors.Is(err, io.EOF) || n != 100 || !bytes.Equal(eofBuffer[:n], content[eofOffset:]) { + t.Fatalf("EOF read n=%d err=%v", n, err) + } + + writeOffset := int64(128<<10 + len(prefix) + 64) + replacement := bytes.Repeat([]byte{'Z'}, 512) + writer, err := os.OpenFile(mountedPath, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := writer.WriteAt(replacement, writeOffset); err != nil { + writer.Close() + t.Fatalf("mounted overwrite: %v", err) + } + if err := writer.Sync(); err != nil { + writer.Close() + t.Fatalf("mounted overwrite sync: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + copy(content[writeOffset:], replacement) + if !completeJSONL(content) { + t.Fatal("mounted overwrite made the fixture invalid JSONL") + } + assertOpenFileRange(t, reader, content, writeOffset, len(replacement)) + + truncateSize := int64(blockSize + lineSize) + if err := os.Truncate(mountedPath, truncateSize); err != nil { + t.Fatalf("mounted truncate: %v", err) + } + truncated := content[:truncateSize] + if !completeJSONL(truncated) { + t.Fatal("truncated fixture is not valid JSONL") + } + truncateBuffer := make([]byte, 4096) + n, err = reader.ReadAt(truncateBuffer, truncateSize-100) + if !errors.Is(err, io.EOF) || n != 100 || !bytes.Equal(truncateBuffer[:n], truncated[truncateSize-100:]) { + t.Fatalf("post-truncate read n=%d err=%v", n, err) + } + + externalOffset := int64(64<<10 + len(prefix) + 32) + externalReplacement := bytes.Repeat([]byte{'X'}, 256) + native, err := os.OpenFile(nativePath, os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := native.WriteAt(externalReplacement, externalOffset); err != nil { + native.Close() + t.Fatalf("native overwrite: %v", err) + } + if err := native.Sync(); err != nil { + native.Close() + t.Fatalf("native overwrite sync: %v", err) + } + if err := native.Close(); err != nil { + t.Fatal(err) + } + copy(truncated[externalOffset:], externalReplacement) + if !completeJSONL(truncated) { + t.Fatal("external overwrite made the fixture invalid JSONL") + } + waitForOpenFileRange(t, reader, truncated, externalOffset, len(externalReplacement), 3*time.Second) + + renamedPath := filepath.Join(root, "read-ahead-renamed.jsonl") + if err := os.Rename(mountedPath, renamedPath); err != nil { + t.Fatalf("rename cached file: %v", err) + } + assertOpenFileRange(t, reader, truncated, 0, 8192) + assertMountedTestContent(t, renamedPath, truncated) +} + +func TestNativeFSKitMountedVirtualConcurrentReadAhead(t *testing.T) { + virtualPath := os.Getenv(nativeFSKitVirtualFileEnv) + referencePath := os.Getenv(nativeFSKitVirtualReferenceFileEnv) + if virtualPath == "" || referencePath == "" { + t.Skipf("set %s and %s to run packed virtual concurrent reads", nativeFSKitVirtualFileEnv, nativeFSKitVirtualReferenceFileEnv) + } + if !filepath.IsAbs(virtualPath) || !filepath.IsAbs(referencePath) { + t.Fatal("packed virtual and reference paths must be absolute") + } + virtual, err := os.Open(virtualPath) + if err != nil { + t.Fatal(err) + } + defer virtual.Close() + reference, err := os.Open(referencePath) + if err != nil { + t.Fatal(err) + } + defer reference.Close() + referenceInfo, err := reference.Stat() + if err != nil { + t.Fatal(err) + } + virtualInfo, err := virtual.Stat() + if err != nil { + t.Fatal(err) + } + const readers = 8 + const rounds = 8 + const readBytes = 32 << 10 + if referenceInfo.Size() < readBytes || virtualInfo.Size() < referenceInfo.Size() { + t.Fatalf("virtual bytes=%d reference bytes=%d", virtualInfo.Size(), referenceInfo.Size()) + } + + errorsByReader := make(chan error, readers) + var concurrent sync.WaitGroup + for worker := 0; worker < readers; worker++ { + concurrent.Add(1) + go func(worker int) { + defer concurrent.Done() + for round := 0; round < rounds; round++ { + limit := referenceInfo.Size() - readBytes + offset := int64(worker*104729+round*15485863) % (limit + 1) + got := make([]byte, readBytes) + want := make([]byte, readBytes) + gotN, gotErr := virtual.ReadAt(got, offset) + wantN, wantErr := reference.ReadAt(want, offset) + if gotErr != nil || wantErr != nil || gotN != readBytes || wantN != readBytes || !bytes.Equal(got, want) { + errorsByReader <- fmt.Errorf("worker=%d round=%d offset=%d virtual=%d/%v reference=%d/%v", worker, round, offset, gotN, gotErr, wantN, wantErr) + return + } + } + }(worker) + } + concurrent.Wait() + close(errorsByReader) + for readErr := range errorsByReader { + t.Fatal(readErr) + } +} + +func TestNativeFSKitMountedManagedPerformance(t *testing.T) { + virtualPath := os.Getenv(nativeFSKitVirtualFileEnv) + referencePath := os.Getenv(nativeFSKitVirtualReferenceFileEnv) + if virtualPath == "" || referencePath == "" { + t.Skipf("set %s and %s to run packed virtual performance", nativeFSKitVirtualFileEnv, nativeFSKitVirtualReferenceFileEnv) + } + if !filepath.IsAbs(virtualPath) || !filepath.IsAbs(referencePath) { + t.Fatal("packed virtual and reference paths must be absolute") + } + + referenceInfo, err := os.Stat(referencePath) + if err != nil { + t.Fatal(err) + } + virtualInfo, err := os.Stat(virtualPath) + if err != nil { + t.Fatal(err) + } + if virtualInfo.Size() != referenceInfo.Size() { + t.Fatalf("virtual bytes=%d reference bytes=%d", virtualInfo.Size(), referenceInfo.Size()) + } + coldReferencePath, copyBypass, err := uncachedReferenceCopy(referencePath) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.Remove(coldReferencePath); err != nil && !errors.Is(err, os.ErrNotExist) { + t.Errorf("remove uncached reference copy: %v", err) + } + }) + if !copyBypass { + t.Fatal("F_NOCACHE was not applied while creating the cold reference copy") + } + + nativeCold, nativeBypass, err := sequentialReadMetric(coldReferencePath, true) + if err != nil { + t.Fatal(err) + } + virtualCold, virtualBypass, err := sequentialReadMetric(virtualPath, true) + if err != nil { + t.Fatal(err) + } + nativeWarm, err := medianSequentialThroughput(coldReferencePath, 3) + if err != nil { + t.Fatal(err) + } + virtualWarm, err := medianSequentialThroughput(virtualPath, 3) + if err != nil { + t.Fatal(err) + } + nativeHash, err := streamingSHA256(coldReferencePath) + if err != nil { + t.Fatal(err) + } + virtualHash, err := streamingSHA256(virtualPath) + if err != nil { + t.Fatal(err) + } + if nativeHash != virtualHash { + t.Fatal("mounted managed file differs from its retained native snapshot") + } + + coldRatio := virtualCold / nativeCold + warmRatio := virtualWarm / nativeWarm + t.Logf( + "native-fskit managed performance bytes=%d cold_native=%.2fMiB/s cold_virtual=%.2fMiB/s cold_ratio=%.3f warm_native=%.2fMiB/s warm_virtual=%.2fMiB/s warm_ratio=%.3f native_nocache=%t virtual_nocache=%t", + referenceInfo.Size(), + nativeCold/(1<<20), virtualCold/(1<<20), coldRatio, + nativeWarm/(1<<20), virtualWarm/(1<<20), warmRatio, + nativeBypass, virtualBypass, + ) + if !nativeBypass || !virtualBypass { + t.Fatal("F_NOCACHE was not applied to both cold-read paths") + } + if coldRatio < 0.70 { + t.Fatalf("managed cold throughput ratio %.3f is below 0.70", coldRatio) + } + if warmRatio < 0.80 { + t.Fatalf("managed warm throughput ratio %.3f is below 0.80", warmRatio) + } +} + +func TestNativeFSKitMountedManagedCacheSurvivesUnrelatedNamespaceChange(t *testing.T) { + virtualPath := os.Getenv(nativeFSKitVirtualFileEnv) + referencePath := os.Getenv(nativeFSKitVirtualReferenceFileEnv) + if virtualPath == "" || referencePath == "" { + t.Skipf("set %s and %s to run managed cache coherency", nativeFSKitVirtualFileEnv, nativeFSKitVirtualReferenceFileEnv) + } + root := nativeFSKitMountedTestRoot(t) + // Let setup namespace changes drain before establishing the cache baseline. + time.Sleep(750 * time.Millisecond) + for index := 0; index < 2; index++ { + if _, _, err := sequentialReadMetric(virtualPath, false); err != nil { + t.Fatal(err) + } + } + writeMountedTestFile(t, filepath.Join(root, "unrelated.bin"), []byte("unrelated namespace change\n")) + time.Sleep(750 * time.Millisecond) + + nativeWarm, err := medianSequentialThroughput(referencePath, 3) + if err != nil { + t.Fatal(err) + } + virtualWarm, _, err := sequentialReadMetric(virtualPath, false) + if err != nil { + t.Fatal(err) + } + ratio := virtualWarm / nativeWarm + t.Logf( + "native-fskit managed cache after unrelated namespace change native=%.2fMiB/s virtual=%.2fMiB/s ratio=%.3f", + nativeWarm/(1<<20), virtualWarm/(1<<20), ratio, + ) + if ratio < 0.80 { + t.Fatalf("unrelated namespace change reduced managed warm throughput ratio to %.3f", ratio) + } +} + +func uncachedReferenceCopy(sourcePath string) (string, bool, error) { + source, err := os.Open(sourcePath) + if err != nil { + return "", false, fmt.Errorf("open cold reference source: %w", err) + } + defer source.Close() + sourceBypass := false + if _, err := unix.FcntlInt(source.Fd(), unix.F_NOCACHE, 1); err == nil { + sourceBypass = true + } + + destination, err := os.CreateTemp(filepath.Dir(sourcePath), ".codexfold-cold-reference-*.jsonl") + if err != nil { + return "", false, fmt.Errorf("create cold reference copy: %w", err) + } + destinationPath := destination.Name() + keep := false + defer func() { + _ = destination.Close() + if !keep { + _ = os.Remove(destinationPath) + } + }() + destinationBypass := false + if _, err := unix.FcntlInt(destination.Fd(), unix.F_NOCACHE, 1); err == nil { + destinationBypass = true + } + + buffer := make([]byte, 4<<20) + for { + count, readErr := source.Read(buffer) + if count > 0 { + written := 0 + for written < count { + amount, writeErr := destination.Write(buffer[written:count]) + if writeErr != nil { + return "", sourceBypass && destinationBypass, fmt.Errorf("write cold reference copy: %w", writeErr) + } + if amount == 0 { + return "", sourceBypass && destinationBypass, io.ErrShortWrite + } + written += amount + } + } + if errors.Is(readErr, io.EOF) { + break + } + if readErr != nil { + return "", sourceBypass && destinationBypass, fmt.Errorf("read cold reference source: %w", readErr) + } + if count == 0 { + return "", sourceBypass && destinationBypass, io.ErrNoProgress + } + } + if err := destination.Sync(); err != nil { + return "", sourceBypass && destinationBypass, fmt.Errorf("sync cold reference copy: %w", err) + } + if err := destination.Close(); err != nil { + return "", sourceBypass && destinationBypass, fmt.Errorf("close cold reference copy: %w", err) + } + keep = true + return destinationPath, sourceBypass && destinationBypass, nil +} + +func writePerformanceFixture(path string, size int64) error { + file, err := os.Create(path) + if err != nil { + return err + } + buffer := bytes.Repeat([]byte("{\"codexfold_performance\":true}\n"), 1<<15) + var written int64 + for written < size { + chunk := buffer + if remaining := size - written; int64(len(chunk)) > remaining { + chunk = chunk[:remaining] + } + n, writeErr := file.Write(chunk) + written += int64(n) + if writeErr != nil { + _ = file.Close() + return writeErr + } + } + return errors.Join(file.Sync(), file.Close()) +} + +func sequentialReadMetric(path string, bypassCache bool) (float64, bool, error) { + file, err := os.Open(path) + if err != nil { + return 0, false, err + } + defer file.Close() + bypassApplied := false + if bypassCache { + if _, err := unix.FcntlInt(file.Fd(), unix.F_NOCACHE, 1); err == nil { + bypassApplied = true + } + } + // io.CopyBuffer would let io.Discard.ReadFrom ignore this buffer and turn + // the bulk metric into thousands of tiny FSKit calls. + buffer := make([]byte, 4<<20) + started := time.Now() + var read int64 + for { + count, readErr := file.Read(buffer) + read += int64(count) + if errors.Is(readErr, io.EOF) { + break + } + if readErr != nil { + return 0, bypassApplied, readErr + } + if count == 0 { + return 0, bypassApplied, io.ErrNoProgress + } + } + duration := time.Since(started) + if duration <= 0 { + return 0, bypassApplied, errors.New("sequential read duration is unavailable") + } + return float64(read) / duration.Seconds(), bypassApplied, nil +} + +func medianSequentialThroughput(path string, runs int) (float64, error) { + values := make([]float64, 0, runs) + for index := 0; index < runs; index++ { + value, _, err := sequentialReadMetric(path, false) + if err != nil { + return 0, err + } + values = append(values, value) + } + sort.Float64s(values) + return values[len(values)/2], nil +} + +func streamingSHA256(path string) ([sha256.Size]byte, error) { + file, err := os.Open(path) + if err != nil { + return [sha256.Size]byte{}, err + } + defer file.Close() + digest := sha256.New() + if _, err := io.CopyBuffer(digest, file, make([]byte, 4<<20)); err != nil { + return [sha256.Size]byte{}, err + } + var result [sha256.Size]byte + copy(result[:], digest.Sum(nil)) + return result, nil +} + +func durationPercentile(values []time.Duration, percentile float64) time.Duration { + ordered := append([]time.Duration(nil), values...) + slices.Sort(ordered) + index := int(float64(len(ordered)-1) * percentile) + return ordered[index] +} + +func waitForMountedSize(t *testing.T, path string, want int64, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var size int64 + var lastErr error + for time.Now().Before(deadline) { + info, err := os.Stat(path) + if err == nil { + size = info.Size() + if size == want { + return + } + } + lastErr = err + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("mounted size %s = %d err=%v, want %d", path, size, lastErr, want) +} + +func nativeFSKitMountedTestRoot(t *testing.T) string { + t.Helper() + mountPoint := nativeFSKitMountPoint(t) + base := filepath.Join(mountPoint, "sessions", "2099", "12", "31") + if err := os.MkdirAll(base, 0o700); err != nil { + t.Fatalf("create mounted test base: %v", err) + } + root, err := os.MkdirTemp(base, "native-fskit-integration-") + if err != nil { + t.Fatalf("create mounted test root: %v", err) + } + relativeRoot, err := filepath.Rel(mountPoint, root) + if err != nil { + t.Fatalf("resolve mounted test root: %v", err) + } + nativeRoot := os.Getenv(nativeFSKitNativeRootEnv) + nativePath := "" + if nativeRoot != "" { + if !filepath.IsAbs(nativeRoot) { + t.Fatalf("%s must be absolute", nativeFSKitNativeRootEnv) + } + nativePath = filepath.Join(nativeRoot, relativeRoot) + } + t.Cleanup(func() { + var cleanupErr error + if err := os.RemoveAll(root); err != nil && !errors.Is(err, os.ErrNotExist) { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("mounted path: %w", err)) + } + if nativePath != "" { + if err := os.RemoveAll(nativePath); err != nil && !errors.Is(err, os.ErrNotExist) { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("native path: %w", err)) + } + } + if cleanupErr != nil { + t.Errorf("cleanup mounted test root: %v", cleanupErr) + } + }) + return root +} + +func assertOpenFileRange(t *testing.T, file *os.File, want []byte, offset int64, length int) { + t.Helper() + buffer := make([]byte, length) + n, err := file.ReadAt(buffer, offset) + if err != nil { + t.Fatalf("read offset=%d length=%d: %v", offset, length, err) + } + if n != length || !bytes.Equal(buffer, want[offset:offset+int64(length)]) { + t.Fatalf("range offset=%d length=%d differs", offset, length) + } +} + +func waitForOpenFileRange(t *testing.T, file *os.File, want []byte, offset int64, length int, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var last []byte + var lastErr error + for time.Now().Before(deadline) { + last = make([]byte, length) + var n int + n, lastErr = file.ReadAt(last, offset) + if lastErr == nil && n == length && bytes.Equal(last, want[offset:offset+int64(length)]) { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("open file range offset=%d length=%d remained stale: err=%v bytes=%x", offset, length, lastErr, last) +} + +func nativeFSKitMountPoint(t *testing.T) string { + t.Helper() + mountPoint := os.Getenv(nativeFSKitMountEnv) + if mountPoint == "" { + t.Skipf("set %s to run native FSKit mount tests", nativeFSKitMountEnv) + } + if !filepath.IsAbs(mountPoint) { + t.Fatalf("%s must be absolute", nativeFSKitMountEnv) + } + return mountPoint +} + +func writeMountedTestFile(t *testing.T, path string, content []byte) { + t.Helper() + if err := os.WriteFile(path, content, 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func assertMountedTestContent(t *testing.T, path string, want []byte) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if !bytes.Equal(got, want) { + t.Fatalf("content %s = %q, want %q", path, got, want) + } +} + +func assertNativeFSKitBackingContent(t *testing.T, mountedPath string, want []byte) { + t.Helper() + mountPoint := nativeFSKitMountPoint(t) + nativeRoot := os.Getenv(nativeFSKitNativeRootEnv) + if nativeRoot == "" { + t.Skipf("set %s to verify native backing content", nativeFSKitNativeRootEnv) + } + relative, err := filepath.Rel(mountPoint, mountedPath) + if err != nil { + t.Fatalf("resolve native backing path: %v", err) + } + path := filepath.Join(nativeRoot, relative) + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read native backing %s: %v", path, err) + } + if !bytes.Equal(got, want) { + t.Fatalf("native backing %s = %q, want %q", path, got, want) + } +} + +func mountedTestXattr(t *testing.T, path string, attribute string) []byte { + t.Helper() + value, err := readMountedTestXattr(path, attribute) + if err != nil { + t.Fatalf("get xattr: %v", err) + } + return value +} + +func readMountedTestXattr(path string, attribute string) ([]byte, error) { + size, err := unix.Getxattr(path, attribute, nil) + if err != nil { + return nil, err + } + value := make([]byte, size) + n, err := unix.Getxattr(path, attribute, value) + return value[:n], err +} + +func mountedTestXattrNames(t *testing.T, path string) []string { + t.Helper() + size, err := unix.Listxattr(path, nil) + if err != nil { + t.Fatalf("size xattrs: %v", err) + } + buffer := make([]byte, size) + n, err := unix.Listxattr(path, buffer) + if err != nil { + t.Fatalf("list xattrs: %v", err) + } + buffer = buffer[:n] + var names []string + for len(buffer) > 0 { + index := bytes.IndexByte(buffer, 0) + if index < 0 { + t.Fatalf("malformed xattr name list %q", buffer) + } + names = append(names, string(buffer[:index])) + buffer = buffer[index+1:] + } + return names +} + +func isNotSupported(err error) bool { + return errors.Is(err, syscall.ENOTSUP) || errors.Is(err, syscall.EOPNOTSUPP) +} + +func waitForMountedContent(t *testing.T, path string, want []byte, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var last []byte + var lastErr error + for time.Now().Before(deadline) { + last, lastErr = os.ReadFile(path) + if lastErr == nil && bytes.Equal(last, want) { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("mounted content %s = %q err=%v, want %q", path, last, lastErr, want) +} + +func waitForMountedAbsence(t *testing.T, path string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + _, lastErr = os.Stat(path) + if errors.Is(lastErr, os.ErrNotExist) { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("mounted path %s remained visible: %v", path, lastErr) +} + +func nativeFSKitMountedNamespaceVersion(t *testing.T) (uint64, bool) { + t.Helper() + resource := os.Getenv(nativeFSKitResourceEnv) + if resource == "" { + return 0, false + } + client, err := fskitproto.DialResource(resource, time.Second) + if err != nil { + t.Fatalf("dial native FSKit resource: %v", err) + } + defer client.Close() + payload, err := client.Call(fskitproto.OpNamespaceVersion, nil) + if err != nil { + t.Fatalf("read native FSKit namespace version: %v", err) + } + decoder := fskitproto.NewDecoder(payload) + version, err := decoder.Uint64() + if err != nil || decoder.Done() != nil { + t.Fatalf("decode native FSKit namespace version: %v", err) + } + return version, true +} + +func logNativeFSKitMountedEntry(t *testing.T, stage string, path string) { + t.Helper() + resource := os.Getenv(nativeFSKitResourceEnv) + if resource == "" { + return + } + if filepath.IsAbs(path) && !canonicalNamespacePath(path) { + relative, err := filepath.Rel(nativeFSKitMountPoint(t), path) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + t.Fatalf("resolve native FSKit entry route %s: %v", path, err) + } + path = "/" + filepath.ToSlash(relative) + } + client, err := fskitproto.DialResource(resource, time.Second) + if err != nil { + t.Fatalf("dial native FSKit resource: %v", err) + } + defer client.Close() + request := fskitproto.NewEncoder(len(path) + 8) + request.String(path) + payload, err := client.Call(fskitproto.OpGetattr, request.Data()) + if err != nil { + t.Fatalf("get native FSKit entry %s: %v", path, err) + } + decoder := fskitproto.NewDecoder(payload) + entry, err := decoder.Entry() + if err != nil || decoder.Done() != nil { + t.Fatalf("decode native FSKit entry %s: %v", path, err) + } + t.Logf( + "%s path=%s node=%d size=%d namespace=%d mtime=%d.%09d ctime=%d.%09d", + stage, path, entry.NodeID, entry.Size, entry.NamespaceID, + entry.ModTime.Unix(), entry.ModTime.Nanosecond(), + entry.ChangeTime.Unix(), entry.ChangeTime.Nanosecond(), + ) +} + +func fileInfoSize(info os.FileInfo) int64 { + if info == nil { + return -1 + } + return info.Size() +} + +func waitForNativeFSKitNamespaceAdvance(t *testing.T, previous uint64, timeout time.Duration) uint64 { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + version, _ := nativeFSKitMountedNamespaceVersion(t) + if version > previous { + return version + } + time.Sleep(25 * time.Millisecond) + } + t.Fatalf("native FSKit namespace version did not advance beyond %d", previous) + return previous +} diff --git a/internal/mountfs/native_fskit_server.go b/internal/mountfs/native_fskit_server.go new file mode 100644 index 0000000..1c4f187 --- /dev/null +++ b/internal/mountfs/native_fskit_server.go @@ -0,0 +1,1477 @@ +package mountfs + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/subtle" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "os" + "path" + "path/filepath" + "sort" + "strings" + "sync" + "syscall" + "time" + + "github.com/samekind/codexfold/internal/buildid" + "github.com/samekind/codexfold/internal/fskitproto" + "github.com/samekind/codexfold/internal/mountid" +) + +type NativeFSKitServerOptions struct { + SocketPath string + ResourcePath string + Token []byte + Generation uint64 + MaxPayload uint32 + BuildSHA256 string + Recorder func(string) + PrewarmSharedMemoryWindows int + PrewarmSharedFileWindows int +} + +const ( + nativeFSKitBufferedReadChunkBytes = 4 << 20 + nativeFSKitSocketBufferBytes = 4 << 20 + nativeFSKitSharedReadMinimumBytes = 4 << 20 + nativeFSKitSharedWindowBytes = 16 << 20 + nativeFSKitMaximumPrewarmWindows = 4 +) + +type nativeFSKitServer struct { + filesystem *Filesystem + token []byte + generation uint64 + maxPayload uint32 + recorder func(string) + health []byte + startedAt time.Time + nodes nativeFSKitNodes + sharedMemoryWindows nativeSharedFileWindowPool + sharedFileWindows nativeSharedFileWindowPool +} + +type nativeSharedFileWindowPool struct { + mu sync.Mutex + windows []*nativeSharedReadWindow + limit int + windowBytes int +} + +type nativeFSKitNodes struct { + mu sync.Mutex + version uint64 + next uint64 + byPath map[string]nativeFSKitNode +} + +type nativeFSKitNode struct { + id uint64 + objectID string +} + +type nativeFSKitConnection struct { + server *nativeFSKitServer + conn net.Conn + handles map[uint64]*nativeFSKitHandle + nextHandle uint64 + capabilities uint32 +} + +type nativeFSKitHandle struct { + coreHandle uint64 + path string + flags int + snapshotWrite bool + snapshotFloor int64 + health bool + sharedWindow *nativeSharedReadWindow + sharedWindowFlag uint32 +} + +func ServeNativeFSKit(ctx context.Context, filesystem *Filesystem, options NativeFSKitServerOptions) error { + if filesystem == nil { + return errors.New("FSKit server filesystem is required") + } + if options.SocketPath == "" || !filepath.IsAbs(options.SocketPath) { + return errors.New("FSKit server socket path must be absolute") + } + if len(options.SocketPath) >= 104 { + return errors.New("FSKit server socket path exceeds the macOS Unix socket limit") + } + if options.ResourcePath == "" || !filepath.IsAbs(options.ResourcePath) { + return errors.New("FSKit resource path must be absolute") + } + if fskitproto.UsesDirectoryResource(options.ResourcePath) { + relativeSocket, err := filepath.Rel(filepath.Clean(options.ResourcePath), filepath.Clean(options.SocketPath)) + if err != nil || relativeSocket == "." || relativeSocket == ".." || strings.HasPrefix(relativeSocket, ".."+string(filepath.Separator)) { + return errors.New("directory FSKit resource requires its Unix socket inside the resource directory") + } + } + if options.MaxPayload == 0 { + options.MaxPayload = fskitproto.DefaultMaxPayload + } + if options.PrewarmSharedMemoryWindows < 0 || options.PrewarmSharedMemoryWindows > nativeFSKitMaximumPrewarmWindows { + return fmt.Errorf("FSKit shared-memory window prewarm count must be between 0 and %d", nativeFSKitMaximumPrewarmWindows) + } + if options.PrewarmSharedFileWindows < 0 || options.PrewarmSharedFileWindows > nativeFSKitMaximumPrewarmWindows { + return fmt.Errorf("FSKit shared-file window prewarm count must be between 0 and %d", nativeFSKitMaximumPrewarmWindows) + } + if len(options.Token) == 0 { + options.Token = make([]byte, 32) + if _, err := rand.Read(options.Token); err != nil { + return fmt.Errorf("generate FSKit authentication token: %w", err) + } + } + if len(options.Token) < 16 || len(options.Token) > 256 { + return errors.New("FSKit authentication token must contain 16 to 256 bytes") + } + if options.Generation == 0 { + var generation [8]byte + if _, err := rand.Read(generation[:]); err != nil { + return fmt.Errorf("generate FSKit mount generation: %w", err) + } + for _, value := range generation { + options.Generation = options.Generation<<8 | uint64(value) + } + if options.Generation == 0 { + options.Generation = 1 + } + } + if options.BuildSHA256 == "" { + var err error + options.BuildSHA256, err = buildid.CurrentSHA256() + if err != nil { + return fmt.Errorf("hash native FSKit daemon executable: %w", err) + } + } + health, err := mountid.New(options.BuildSHA256) + if err != nil { + return fmt.Errorf("generate native FSKit mount identity: %w", err) + } + if err := os.MkdirAll(filepath.Dir(options.SocketPath), 0o700); err != nil { + return fmt.Errorf("create FSKit socket directory: %w", err) + } + if err := removeStaleUnixSocket(options.SocketPath); err != nil { + return err + } + listener, err := net.Listen("unix", options.SocketPath) + if err != nil { + return fmt.Errorf("listen on FSKit socket: %w", err) + } + defer listener.Close() + defer os.Remove(options.SocketPath) + if err := os.Chmod(options.SocketPath, 0o600); err != nil { + return fmt.Errorf("restrict FSKit socket: %w", err) + } + server := &nativeFSKitServer{ + filesystem: filesystem, + token: append([]byte(nil), options.Token...), + generation: options.Generation, + maxPayload: options.MaxPayload, + recorder: options.Recorder, + health: []byte(health), + startedAt: time.Now(), + nodes: nativeFSKitNodes{ + version: filesystem.NamespaceVersion(), + next: 4, + byPath: map[string]nativeFSKitNode{"/": {id: 2}}, + }, + } + windowBytes := min(nativeFSKitSharedWindowBytes, int(options.MaxPayload)-4) + if err := server.prewarmSharedMemoryWindows(options.PrewarmSharedMemoryWindows, windowBytes); err != nil { + server.record(fmt.Sprintf("io=shared_window_prewarm_error error=%q", err.Error())) + } + if err := server.prewarmSharedFileWindows(options.PrewarmSharedFileWindows, windowBytes); err != nil { + server.record(fmt.Sprintf("io=shared_file_window_prewarm_error error=%q", err.Error())) + } + defer func() { + if err := errors.Join(server.closePrewarmedSharedMemoryWindows(), server.closePrewarmedSharedFileWindows()); err != nil { + server.record(fmt.Sprintf("io=shared_file_window_pool_close_error error=%q", err.Error())) + } + }() + descriptor, err := fskitproto.EncodeDescriptor(fskitproto.Descriptor{ + Generation: options.Generation, + SocketPath: options.SocketPath, + Token: options.Token, + }) + if err != nil { + return err + } + if err := writeNativeFSKitResource(options.ResourcePath, descriptor); err != nil { + return err + } + go func() { + <-ctx.Done() + _ = listener.Close() + }() + for { + connection, err := listener.Accept() + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return ctx.Err() + } + return fmt.Errorf("accept FSKit connection: %w", err) + } + if err := configureNativeFSKitSocket(connection); err != nil { + server.record(fmt.Sprintf("connection_buffer_error error=%q", err.Error())) + } + go (&nativeFSKitConnection{server: server, conn: connection, handles: make(map[uint64]*nativeFSKitHandle), nextHandle: 1}).serve() + } +} + +func configureNativeFSKitSocket(connection net.Conn) error { + unixConnection, ok := connection.(*net.UnixConn) + if !ok { + return nil + } + return errors.Join( + unixConnection.SetReadBuffer(nativeFSKitSocketBufferBytes), + unixConnection.SetWriteBuffer(nativeFSKitSocketBufferBytes), + ) +} + +func removeStaleUnixSocket(socketPath string) error { + info, err := os.Lstat(socketPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("inspect FSKit socket path: %w", err) + } + if info.Mode()&os.ModeSocket == 0 { + return errors.New("FSKit socket path exists and is not a Unix socket") + } + connection, dialErr := net.DialTimeout("unix", socketPath, 100*time.Millisecond) + if dialErr == nil { + _ = connection.Close() + return errors.New("FSKit socket is already active") + } + if err := os.Remove(socketPath); err != nil { + return fmt.Errorf("remove stale FSKit socket: %w", err) + } + return nil +} + +func writeNativeFSKitResource(resourcePath string, data []byte) error { + descriptorPath, err := fskitproto.ResourceDescriptorPath(resourcePath) + if err != nil { + return err + } + if fskitproto.UsesDirectoryResource(resourcePath) { + if err := os.MkdirAll(resourcePath, 0o700); err != nil { + return fmt.Errorf("create FSKit resource directory: %w", err) + } + if err := os.Chmod(resourcePath, 0o700); err != nil { + return fmt.Errorf("restrict FSKit resource directory: %w", err) + } + } + if err := os.MkdirAll(filepath.Dir(descriptorPath), 0o700); err != nil { + return fmt.Errorf("create FSKit resource directory: %w", err) + } + temporary, err := os.CreateTemp(filepath.Dir(descriptorPath), ".codexfold-fskit-resource-*") + if err != nil { + return fmt.Errorf("create FSKit resource: %w", err) + } + temporaryPath := temporary.Name() + committed := false + defer func() { + _ = temporary.Close() + if !committed { + _ = os.Remove(temporaryPath) + } + }() + if err := temporary.Chmod(0o600); err != nil { + return fmt.Errorf("restrict FSKit resource: %w", err) + } + if _, err := temporary.Write(data); err != nil { + return fmt.Errorf("write FSKit resource: %w", err) + } + if err := temporary.Sync(); err != nil { + return fmt.Errorf("sync FSKit resource: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close FSKit resource: %w", err) + } + if err := os.Rename(temporaryPath, descriptorPath); err != nil { + return fmt.Errorf("publish FSKit resource: %w", err) + } + committed = true + return nil +} + +func (c *nativeFSKitConnection) serve() { + defer c.conn.Close() + defer c.releaseHandles() + authenticated := false + for { + request, err := fskitproto.ReadFrame(c.conn, c.server.maxPayload) + if err != nil { + if !errors.Is(err, io.EOF) && !errors.Is(err, net.ErrClosed) { + c.server.record(fmt.Sprintf("connection_error error=%q", err.Error())) + } + return + } + response := fskitproto.Frame{ + Kind: fskitproto.KindResponse, + Op: request.Op, + RequestID: request.RequestID, + Generation: c.server.generation, + } + if request.Kind != fskitproto.KindRequest { + response.Status = int32(syscall.EPROTO) + } else if !authenticated { + if request.Op != fskitproto.OpHello { + response.Status = int32(syscall.EACCES) + } else { + response.Payload, response.Status = c.hello(request.Payload) + authenticated = response.Status == 0 + } + } else if request.Generation != c.server.generation { + response.Status = int32(syscall.ESTALE) + } else { + c.recordReadRequest(request.Payload, request.Op) + streamed, responseWritten, streamStatus, streamErr := c.streamRead(request) + if streamErr != nil { + if responseWritten { + return + } + response.Status = int32(errnoFor(streamErr)) + } else if streamed && responseWritten { + c.server.record(fmt.Sprintf("operation=%s request=%d status=%d payload=%d", nativeFSKitOperationName(request.Op), request.RequestID, 0, len(request.Payload))) + continue + } else if streamed { + response.Status = streamStatus + } else { + response.Payload, response.Status = c.dispatch(request.Op, request.Payload) + } + } + if response.Status == 0 && request.Op == fskitproto.OpOpen && authenticated { + handled, responseWritten, transferErr := c.writeOpenResponseWithOptimizedFD(response) + if transferErr != nil { + if responseWritten { + return + } + response.Status = int32(errnoFor(transferErr)) + response.Payload = nil + } else if handled && responseWritten { + c.server.record(fmt.Sprintf("operation=%s request=%d status=%d payload=%d", nativeFSKitOperationName(request.Op), request.RequestID, 0, len(request.Payload))) + continue + } + } + c.server.record(fmt.Sprintf("operation=%s request=%d status=%d payload=%d", nativeFSKitOperationName(request.Op), request.RequestID, response.Status, len(request.Payload))) + if err := fskitproto.WriteFrame(c.conn, response, c.server.maxPayload); err != nil { + return + } + if !authenticated || response.Status == int32(syscall.ESTALE) { + return + } + } +} + +// streamRead writes successful read responses without copying the payload +// through the generic frame encoder. Stable native files use sendfile; virtual +// files and pending native appends use bounded chunks. +func (c *nativeFSKitConnection) streamRead(request fskitproto.Frame) (handled bool, responseWritten bool, status int32, err error) { + if request.Op != fskitproto.OpRead { + return false, false, 0, nil + } + decoder := fskitproto.NewDecoder(request.Payload) + handle, offset, length, errno := decodeRead(decoder, c.server.maxPayload) + if errno != 0 { + return false, false, 0, nil + } + handleState, exists := c.handles[handle] + if !exists || handleState.health { + return false, false, 0, nil + } + response := fskitproto.Frame{ + Kind: fskitproto.KindResponse, + Op: request.Op, + RequestID: request.RequestID, + Generation: c.server.generation, + } + started := false + startedAt := time.Now() + handled = false + n := 0 + var streamErr error + if nativeFileStreamingAvailable() { + handled, n, streamErr = c.server.filesystem.StreamNativeRead(handleState.coreHandle, offset, length, func(file *os.File, fileOffset int64, count int) (int, error) { + started = true + if err := fskitproto.WriteFrameHeader(c.conn, response, 4+count, c.server.maxPayload); err != nil { + return 0, err + } + var lengthPrefix [4]byte + binary.LittleEndian.PutUint32(lengthPrefix[:], uint32(count)) + if err := fskitproto.WriteFramePayload(c.conn, lengthPrefix[:]); err != nil { + return 0, err + } + return sendNativeFile(c.conn, file, fileOffset, count) + }) + } + if !handled { + if handleState.sharedWindow != nil && handleState.sharedWindowFlag != 0 && length <= handleState.sharedWindow.capacity { + windowHandled, windowWritten, windowBytes, windowErr := c.streamSharedReadWindow( + response, + handleState, + offset, + length, + ) + if windowHandled { + transport := "shared_window" + if handleState.sharedWindowFlag == fskitproto.FlagSharedFileWindow { + transport = "shared_file_window" + } + c.server.record(fmt.Sprintf("io=read_result handle=%d offset=%d bytes=%d duration_ns=%d transport=%s", handle, offset, windowBytes, time.Since(startedAt).Nanoseconds(), transport)) + return true, windowWritten, 0, windowErr + } + } + if c.capabilities&fskitproto.CapabilitySharedReadFD != 0 && + nativeSharedReadFDAvailable() && length >= nativeFSKitSharedReadMinimumBytes { + sharedHandled, sharedWritten, sharedBytes, sharedErr := c.streamSharedReadFD( + response, + handleState.coreHandle, + offset, + length, + ) + if sharedHandled { + c.server.record(fmt.Sprintf("io=read_result handle=%d offset=%d bytes=%d duration_ns=%d transport=shared_fd", handle, offset, sharedBytes, time.Since(startedAt).Nanoseconds())) + return true, sharedWritten, 0, sharedErr + } + } + startedAt = time.Now() + n, streamErr = c.server.filesystem.StreamBufferedRead(handleState.coreHandle, offset, length, nativeFSKitBufferedReadChunkBytes, func(total int, chunk []byte) error { + if !started { + if err := fskitproto.WriteFrameHeader(c.conn, response, 4+total, c.server.maxPayload); err != nil { + return err + } + var lengthPrefix [4]byte + binary.LittleEndian.PutUint32(lengthPrefix[:], uint32(total)) + if err := fskitproto.WriteFramePayload(c.conn, lengthPrefix[:]); err != nil { + return err + } + started = true + } + return fskitproto.WriteFramePayload(c.conn, chunk) + }) + c.server.record(fmt.Sprintf("io=read_result handle=%d offset=%d bytes=%d duration_ns=%d", handle, offset, n, time.Since(startedAt).Nanoseconds())) + if streamErr != nil { + return true, started, 0, streamErr + } + return true, started, 0, nil + } + c.server.record(fmt.Sprintf("io=read_result handle=%d offset=%d bytes=%d duration_ns=%d", handle, offset, n, time.Since(startedAt).Nanoseconds())) + if streamErr != nil { + return true, started, 0, streamErr + } + return true, started, 0, nil +} + +func (c *nativeFSKitConnection) streamSharedReadWindow(response fskitproto.Frame, handle *nativeFSKitHandle, offset int64, length int) (handled bool, responseWritten bool, n int, err error) { + if length < 0 || length > handle.sharedWindow.capacity { + return true, false, 0, errors.New("shared window read exceeded negotiated capacity") + } + n, readErrno := c.server.filesystem.Read(handle.coreHandle, handle.sharedWindow.mapping[:length], offset) + if readErrno != 0 { + return true, false, n, readErrno + } + if n < 0 || n > length { + return true, false, n, errors.New("shared window returned an invalid byte count") + } + encoder := fskitproto.NewEncoder(4) + encoder.Uint32(uint32(n)) + if handle.sharedWindowFlag != fskitproto.FlagSharedWindow && handle.sharedWindowFlag != fskitproto.FlagSharedFileWindow { + return true, false, n, errors.New("shared window has an invalid transport flag") + } + response.Flags |= handle.sharedWindowFlag + response.Payload = encoder.Data() + if err := fskitproto.WriteFrame(c.conn, response, c.server.maxPayload); err != nil { + return true, true, n, err + } + return true, true, n, nil +} + +func (c *nativeFSKitConnection) streamSharedReadFD(response fskitproto.Frame, coreHandle uint64, offset int64, length int) (handled bool, responseWritten bool, n int, err error) { + file, n, prepareErr := prepareNativeSharedReadFD(length, func(mapping []byte) (int, error) { + read, readErrno := c.server.filesystem.Read(coreHandle, mapping, offset) + if readErrno != 0 { + return read, readErrno + } + if read < 0 || read > len(mapping) { + return read, errors.New("shared read returned an invalid byte count") + } + return read, nil + }) + if prepareErr != nil { + c.server.record(fmt.Sprintf("io=shared_read_fallback offset=%d bytes=%d error=%q", offset, length, prepareErr.Error())) + return false, false, 0, nil + } + defer file.Close() + encoder := fskitproto.NewEncoder(4) + encoder.Uint32(uint32(n)) + response.Payload = encoder.Data() + if n == 0 { + if err := fskitproto.WriteFrame(c.conn, response, c.server.maxPayload); err != nil { + return true, true, 0, err + } + return true, true, 0, nil + } + response.Flags |= fskitproto.FlagSharedReadFD + if err := fskitproto.WriteFrame(c.conn, response, c.server.maxPayload); err != nil { + return true, true, n, err + } + if err := sendSharedReadFD(c.conn, file); err != nil { + return true, true, n, err + } + return true, true, n, nil +} + +func (c *nativeFSKitConnection) recordReadRequest(payload []byte, operation fskitproto.Op) { + if operation != fskitproto.OpRead { + return + } + decoder := fskitproto.NewDecoder(payload) + handle, offset, length, errno := decodeRead(decoder, c.server.maxPayload) + if errno == 0 { + c.server.record(fmt.Sprintf("io=read handle=%d offset=%d bytes=%d", handle, offset, length)) + } +} + +func (c *nativeFSKitConnection) hello(payload []byte) ([]byte, int32) { + decoder := fskitproto.NewDecoder(payload) + token, err := decoder.Bytes(256) + if err != nil || len(token) != len(c.server.token) || subtle.ConstantTimeCompare(token, c.server.token) != 1 { + return nil, int32(syscall.EACCES) + } + var capabilities uint32 + if decoder.Remaining() != 0 { + capabilities, err = decoder.Uint32() + if err != nil { + return nil, int32(syscall.EINVAL) + } + } + if decoder.Done() != nil { + return nil, int32(syscall.EINVAL) + } + c.capabilities = capabilities + encoder := fskitproto.NewEncoder(16) + encoder.Uint32(c.server.maxPayload) + encoder.Uint64(c.server.filesystem.NamespaceVersion()) + if capabilities&fskitproto.CapabilityContentGeneration != 0 { + encoder.Uint32(capabilities & fskitproto.CapabilityContentGeneration) + } + return encoder.Data(), 0 +} + +func (c *nativeFSKitConnection) writeOpenResponseWithNativeFD(response fskitproto.Frame) (handled bool, responseWritten bool, err error) { + if c.capabilities&fskitproto.CapabilityNativeReadFD == 0 { + return false, false, nil + } + decoder := fskitproto.NewDecoder(response.Payload) + wireHandle, decodeErr := decoder.Uint64() + if decodeErr != nil || decoder.Done() != nil { + return false, false, nil + } + handleState, exists := c.handles[wireHandle] + if !exists || handleState.health || handleState.flags&(os.O_WRONLY|os.O_RDWR) != 0 { + return false, false, nil + } + started := false + handled, transferErr := c.server.filesystem.WithNativeReadFD(handleState.coreHandle, func(file *os.File) error { + started = true + response.Flags |= fskitproto.FlagNativeReadFD + if err := fskitproto.WriteFrame(c.conn, response, c.server.maxPayload); err != nil { + return err + } + return sendNativeReadFD(c.conn, file) + }) + if !handled { + return false, false, nil + } + if transferErr != nil && !started { + return false, false, nil + } + return true, started, transferErr +} + +func (c *nativeFSKitConnection) writeOpenResponseWithOptimizedFD(response fskitproto.Frame) (handled bool, responseWritten bool, err error) { + handled, responseWritten, err = c.writeOpenResponseWithNativeFD(response) + if handled || err != nil { + return handled, responseWritten, err + } + if c.capabilities&(fskitproto.CapabilitySharedWindow|fskitproto.CapabilitySharedFileWindow) == 0 || !nativeSharedReadFDAvailable() { + return false, false, nil + } + decoder := fskitproto.NewDecoder(response.Payload) + wireHandle, decodeErr := decoder.Uint64() + if decodeErr != nil || decoder.Done() != nil { + return false, false, nil + } + handleState, exists := c.handles[wireHandle] + if !exists || handleState.health || handleState.flags&(os.O_WRONLY|os.O_RDWR) != 0 { + return false, false, nil + } + windowBytes := min(nativeFSKitSharedWindowBytes, int(c.server.maxPayload)-4) + var window *nativeSharedReadWindow + var windowFlag uint32 + if c.capabilities&fskitproto.CapabilitySharedWindow != 0 { + sharedWindow, prewarmed, createErr := c.server.acquireSharedMemoryWindow(windowBytes) + if createErr != nil { + c.server.record(fmt.Sprintf("io=shared_window_fallback error=%q", createErr.Error())) + } else { + window = sharedWindow + windowFlag = fskitproto.FlagSharedWindow + if prewarmed { + c.server.record("io=shared_window_pool_hit") + } + } + } + if window == nil && c.capabilities&fskitproto.CapabilitySharedFileWindow != 0 && nativeSharedFileWindowAvailable() { + fileWindow, prewarmed, createErr := c.server.acquireSharedFileWindow(windowBytes) + if createErr != nil { + c.server.record(fmt.Sprintf("io=shared_file_window_fallback error=%q", createErr.Error())) + } else { + window = fileWindow + windowFlag = fskitproto.FlagSharedFileWindow + if prewarmed { + c.server.record("io=shared_file_window_pool_hit") + } + } + } + if window == nil { + return false, false, nil + } + handleState.sharedWindow = window + handleState.sharedWindowFlag = windowFlag + encoder := fskitproto.NewEncoder(12) + encoder.Uint64(wireHandle) + encoder.Uint32(uint32(window.capacity)) + response.Payload = encoder.Data() + response.Flags |= windowFlag + if err := fskitproto.WriteFrame(c.conn, response, c.server.maxPayload); err != nil { + return true, true, err + } + var transferErr error + if windowFlag == fskitproto.FlagSharedFileWindow { + transferErr = sendSharedFileWindowFD(c.conn, window.file) + } else { + transferErr = sendSharedWindowFD(c.conn, window.file) + } + if transferErr != nil { + return true, true, transferErr + } + return true, true, nil +} + +func (s *nativeFSKitServer) prewarmSharedFileWindows(count int, length int) error { + if count == 0 || !nativeSharedFileWindowAvailable() { + return nil + } + return s.sharedFileWindows.prewarm(count, length, newNativeSharedFileWindow, "shared-file") +} + +func (s *nativeFSKitServer) acquireSharedFileWindow(length int) (*nativeSharedReadWindow, bool, error) { + return s.sharedFileWindows.acquire(length, newNativeSharedFileWindow) +} + +func (s *nativeFSKitServer) recycleSharedFileWindow(window *nativeSharedReadWindow) bool { + return s.sharedFileWindows.recycle(window) +} + +func (s *nativeFSKitServer) closePrewarmedSharedFileWindows() error { + return s.sharedFileWindows.close() +} + +func (s *nativeFSKitServer) prewarmSharedMemoryWindows(count int, length int) error { + if count == 0 || !nativeSharedReadFDAvailable() { + return nil + } + return s.sharedMemoryWindows.prewarm(count, length, newNativeSharedReadWindow, "shared-memory") +} + +func (s *nativeFSKitServer) acquireSharedMemoryWindow(length int) (*nativeSharedReadWindow, bool, error) { + return s.sharedMemoryWindows.acquire(length, newNativeSharedReadWindow) +} + +func (s *nativeFSKitServer) recycleSharedMemoryWindow(window *nativeSharedReadWindow) bool { + return s.sharedMemoryWindows.recycle(window) +} + +func (s *nativeFSKitServer) closePrewarmedSharedMemoryWindows() error { + return s.sharedMemoryWindows.close() +} + +type nativeSharedWindowFactory func(int) (*nativeSharedReadWindow, error) + +func (p *nativeSharedFileWindowPool) prewarm(count int, length int, create nativeSharedWindowFactory, kind string) error { + if length <= 0 || create == nil { + return fmt.Errorf("invalid %s window prewarm request", kind) + } + windows := make([]*nativeSharedReadWindow, 0, count) + for index := 0; index < count; index++ { + window, err := create(length) + if err != nil { + var closeErr error + for _, created := range windows { + closeErr = errors.Join(closeErr, created.Close()) + } + return errors.Join(fmt.Errorf("prewarm %s window %d: %w", kind, index, err), closeErr) + } + clear(window.mapping) + windows = append(windows, window) + } + p.mu.Lock() + p.limit = count + p.windowBytes = length + p.windows = append(p.windows, windows...) + p.mu.Unlock() + return nil +} + +func (p *nativeSharedFileWindowPool) acquire(length int, create nativeSharedWindowFactory) (*nativeSharedReadWindow, bool, error) { + p.mu.Lock() + for index := len(p.windows) - 1; index >= 0; index-- { + window := p.windows[index] + if window.capacity != length { + continue + } + p.windows = append(p.windows[:index], p.windows[index+1:]...) + p.mu.Unlock() + return window, true, nil + } + p.mu.Unlock() + window, err := create(length) + return window, false, err +} + +func (p *nativeSharedFileWindowPool) recycle(window *nativeSharedReadWindow) bool { + if window == nil { + return false + } + p.mu.Lock() + recycle := p.limit > 0 && + len(p.windows) < p.limit && + window.capacity == p.windowBytes && + window.file != nil && + window.mapping != nil + if recycle { + p.windows = append(p.windows, window) + } + p.mu.Unlock() + if !recycle { + _ = window.Close() + } + return recycle +} + +func (p *nativeSharedFileWindowPool) close() error { + p.mu.Lock() + windows := p.windows + p.windows = nil + p.limit = 0 + p.windowBytes = 0 + p.mu.Unlock() + var result error + for _, window := range windows { + result = errors.Join(result, window.Close()) + } + return result +} + +func (c *nativeFSKitConnection) dispatch(operation fskitproto.Op, payload []byte) ([]byte, int32) { + c.server.nodes.syncVersion(c.server.filesystem.NamespaceVersion()) + decoder := fskitproto.NewDecoder(payload) + switch operation { + case fskitproto.OpPing: + if err := decoder.Done(); err != nil { + return nil, int32(syscall.EINVAL) + } + return nil, 0 + case fskitproto.OpGetattr: + name, errno := decodePath(decoder) + if errno != 0 { + return nil, int32(errno) + } + entry, errno := c.server.entry(name) + if errno != 0 { + return nil, int32(errno) + } + encoder := fskitproto.NewEncoder(160) + encoder.EntryForCapabilities(entry, c.capabilities) + return encoder.Data(), 0 + case fskitproto.OpReadDir: + name, errno := decodePath(decoder) + if errno != 0 { + return nil, int32(errno) + } + entries, errno := c.server.readDir(name) + if errno != 0 { + return nil, int32(errno) + } + encoder := fskitproto.NewEncoder(16 + len(entries)*160) + encoder.Uint32(uint32(len(entries))) + for _, entry := range entries { + encoder.EntryForCapabilities(entry, c.capabilities) + } + return encoder.Data(), 0 + case fskitproto.OpOpen: + name, flags, errno := decodeOpen(decoder) + if errno != 0 { + return nil, int32(errno) + } + openFlags := int(flags &^ fskitproto.OpenFlagSnapshot) + if cleanPath(name) == "/"+mountid.Path { + if openFlags&(os.O_WRONLY|os.O_RDWR) != 0 { + return nil, int32(syscall.EPERM) + } + handle := c.addHealthHandle(name, openFlags) + encoder := fskitproto.NewEncoder(8) + encoder.Uint64(handle) + return encoder.Data(), 0 + } + coreHandle, errno := c.server.filesystem.Open(name, openFlags) + if errno != 0 { + return nil, int32(errno) + } + handle := c.addHandle(coreHandle, name, openFlags, flags&fskitproto.OpenFlagSnapshot != 0) + encoder := fskitproto.NewEncoder(8) + encoder.Uint64(handle) + return encoder.Data(), 0 + case fskitproto.OpCreate: + name, flags, errno := decodeOpen(decoder) + if errno != 0 { + return nil, int32(errno) + } + _, existing := c.server.filesystem.Getattr(name) + if existing == 0 { + return nil, int32(syscall.EEXIST) + } + if c.server.filesystem.nativeNamespaceRefreshActive(name) { + return nil, int32(existing) + } + openFlags := int(flags&^fskitproto.OpenFlagSnapshot) | os.O_CREATE | os.O_EXCL + coreHandle, errno := c.server.filesystem.Open(name, openFlags) + if errno != 0 { + return nil, int32(errno) + } + handle := c.addHandle(coreHandle, name, openFlags, flags&fskitproto.OpenFlagSnapshot != 0) + c.server.filesystem.bumpNamespaceVersion() + c.server.nodes.acceptVersion(c.server.filesystem.NamespaceVersion()) + entry, errno := c.server.entry(name) + if errno != 0 { + _ = c.server.filesystem.Release(coreHandle) + delete(c.handles, handle) + return nil, int32(errno) + } + encoder := fskitproto.NewEncoder(176) + encoder.Uint64(handle) + encoder.EntryForCapabilities(entry, c.capabilities) + return encoder.Data(), 0 + case fskitproto.OpRead: + handle, offset, length, errno := decodeRead(decoder, c.server.maxPayload) + handleState, exists := c.handles[handle] + if errno != 0 || !exists { + if errno == 0 { + errno = syscall.EBADF + } + return nil, int32(errno) + } + if handleState.health { + if offset >= int64(len(c.server.health)) { + encoder := fskitproto.NewEncoder(4) + encoder.Bytes(nil) + return encoder.Data(), 0 + } + end := min(int64(len(c.server.health)), offset+int64(length)) + encoder := fskitproto.NewEncoder(4 + int(end-offset)) + encoder.Bytes(c.server.health[offset:end]) + return encoder.Data(), 0 + } + buffer := make([]byte, length) + started := time.Now() + n, errno := c.server.filesystem.Read(handleState.coreHandle, buffer, offset) + c.server.record(fmt.Sprintf("io=read_result handle=%d offset=%d bytes=%d duration_ns=%d", handle, offset, n, time.Since(started).Nanoseconds())) + if errno != 0 { + return nil, int32(errno) + } + encoder := fskitproto.NewEncoder(4 + n) + encoder.Bytes(buffer[:n]) + return encoder.Data(), 0 + case fskitproto.OpWrite: + handle, offset, data, errno := decodeWrite(decoder, c.server.maxPayload) + c.server.record(fmt.Sprintf("io=write handle=%d offset=%d bytes=%d", handle, offset, len(data))) + handleState, exists := c.handles[handle] + if errno != 0 || !exists { + if errno == 0 { + errno = syscall.EBADF + } + return nil, int32(errno) + } + if handleState.health { + return nil, int32(syscall.EROFS) + } + n, normalized, errno := c.writeHandle(handleState, data, offset) + if errno != 0 { + return nil, int32(errno) + } + if attribute, attrErrno := c.server.filesystem.Getattr(handleState.path); attrErrno == 0 { + c.server.record(fmt.Sprintf("write_result handle=%d reported=%d normalized=%t visible=%d", handle, n, normalized, attribute.Size)) + } + encoder := fskitproto.NewEncoder(4) + encoder.Uint32(uint32(n)) + return encoder.Data(), 0 + case fskitproto.OpFsync, fskitproto.OpFlush, fskitproto.OpRelease: + handle, err := decoder.Uint64() + handleState, exists := c.handles[handle] + if err != nil || decoder.Done() != nil || !exists { + return nil, int32(syscall.EBADF) + } + if handleState.health { + if operation == fskitproto.OpRelease { + delete(c.handles, handle) + } + return nil, 0 + } + var errno syscall.Errno + switch operation { + case fskitproto.OpFsync: + errno = c.server.filesystem.Fsync(handleState.coreHandle) + case fskitproto.OpFlush: + errno = c.server.filesystem.Flush(handleState.coreHandle) + case fskitproto.OpRelease: + c.releaseSharedWindow(handleState) + errno = c.server.filesystem.Release(handleState.coreHandle) + delete(c.handles, handle) + } + return nil, int32(errno) + case fskitproto.OpTruncate: + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return nil, int32(errno) + } + size, err := decoder.Int64() + if err != nil || size < 0 || decoder.Done() != nil { + return nil, int32(syscall.EINVAL) + } + return nil, int32(c.server.filesystem.TruncatePath(name, size)) + case fskitproto.OpMkdir: + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return nil, int32(errno) + } + mode, err := decoder.Uint32() + if err != nil || decoder.Done() != nil { + return nil, int32(syscall.EINVAL) + } + if c.server.filesystem.nativeNamespaceRefreshActive(name) { + _, existing := c.server.filesystem.Getattr(name) + if existing == 0 { + return nil, int32(syscall.EEXIST) + } + return nil, int32(existing) + } + errno = c.server.filesystem.Mkdir(name, mode) + if errno == 0 { + c.server.nodes.acceptVersion(c.server.filesystem.NamespaceVersion()) + } + return nil, int32(errno) + case fskitproto.OpRename: + oldName, errno := decodePathPrefix(decoder) + if errno != 0 { + return nil, int32(errno) + } + newName, errno := decodePath(decoder) + if errno != 0 { + return nil, int32(errno) + } + c.server.record(fmt.Sprintf("path_rename old=%q new=%q", oldName, newName)) + errno = c.server.filesystem.Rename(oldName, newName) + if errno == 0 { + c.server.nodes.rename(oldName, newName, c.server.filesystem.NamespaceVersion()) + } + return nil, int32(errno) + case fskitproto.OpUnlink, fskitproto.OpRmdir: + name, errno := decodePath(decoder) + if errno != 0 { + return nil, int32(errno) + } + if operation == fskitproto.OpUnlink { + errno = c.server.filesystem.Unlink(name) + } else { + errno = c.server.filesystem.Rmdir(name) + } + if errno == 0 { + c.server.nodes.remove(name, c.server.filesystem.NamespaceVersion()) + } + return nil, int32(errno) + case fskitproto.OpStatfs: + if err := decoder.Done(); err != nil { + return nil, int32(syscall.EINVAL) + } + stat, err := nativeFSKitStat(c.server.filesystem.nativeRoot) + if err != nil { + return nil, int32(errnoFor(err)) + } + encoder := fskitproto.NewEncoder(64) + encoder.StatFS(stat) + return encoder.Data(), 0 + case fskitproto.OpSync: + if err := decoder.Done(); err != nil { + return nil, int32(syscall.EINVAL) + } + return nil, int32(c.server.filesystem.SyncAll()) + case fskitproto.OpNamespaceVersion: + if err := decoder.Done(); err != nil { + return nil, int32(syscall.EINVAL) + } + encoder := fskitproto.NewEncoder(8) + encoder.Uint64(c.server.filesystem.NamespaceVersion()) + return encoder.Data(), 0 + case fskitproto.OpSetattr: + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return nil, int32(errno) + } + valid, err := decoder.Uint32() + if err != nil || valid&^(fskitproto.SetAttrMode|fskitproto.SetAttrUID|fskitproto.SetAttrGID|fskitproto.SetAttrAccessTime|fskitproto.SetAttrModifyTime) != 0 { + return nil, int32(syscall.EINVAL) + } + mode, modeErr := decoder.Uint32() + uid, uidErr := decoder.Uint32() + gid, gidErr := decoder.Uint32() + accessTime, accessErr := decoder.Time() + modifyTime, modifyErr := decoder.Time() + if errors.Join(modeErr, uidErr, gidErr, accessErr, modifyErr, decoder.Done()) != nil { + return nil, int32(syscall.EINVAL) + } + request := SetAttrRequest{ + Valid: valid, Mode: mode, UID: uid, GID: gid, + AccessTime: accessTime, ModTime: modifyTime, + } + return nil, int32(c.server.filesystem.SetAttributes(name, request)) + case fskitproto.OpGetXattr: + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return nil, int32(errno) + } + attribute, err := decoder.String(4096) + if err != nil || decoder.Done() != nil { + return nil, int32(syscall.EINVAL) + } + value, errno := c.server.filesystem.GetXattr(name, attribute) + if errno != 0 { + return nil, int32(errno) + } + encoder := fskitproto.NewEncoder(4 + len(value)) + encoder.Bytes(value) + return encoder.Data(), 0 + case fskitproto.OpSetXattr: + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return nil, int32(errno) + } + attribute, err := decoder.String(4096) + policy, policyErr := decoder.Uint32() + value, valueErr := decoder.Bytes(int(c.server.maxPayload) - 8192) + if errors.Join(err, policyErr, valueErr, decoder.Done()) != nil || policy > uint32(fskitproto.XattrDelete) { + return nil, int32(syscall.EINVAL) + } + return nil, int32(c.server.filesystem.SetXattr(name, attribute, value, fskitproto.XattrPolicy(policy))) + case fskitproto.OpListXattrs: + name, errno := decodePath(decoder) + if errno != 0 { + return nil, int32(errno) + } + attributes, errno := c.server.filesystem.ListXattrs(name) + if errno != 0 { + return nil, int32(errno) + } + encoder := fskitproto.NewEncoder(4 + len(attributes)*64) + encoder.Uint32(uint32(len(attributes))) + for _, attribute := range attributes { + encoder.String(attribute) + } + return encoder.Data(), 0 + default: + return nil, int32(syscall.ENOSYS) + } +} + +func (c *nativeFSKitConnection) releaseHandles() { + for _, handle := range c.handles { + c.releaseSharedWindow(handle) + if !handle.health { + _ = c.server.filesystem.Release(handle.coreHandle) + } + } +} + +func (c *nativeFSKitConnection) releaseSharedWindow(handle *nativeFSKitHandle) { + window := handle.sharedWindow + flag := handle.sharedWindowFlag + handle.sharedWindow = nil + handle.sharedWindowFlag = 0 + if window == nil { + return + } + if flag == fskitproto.FlagSharedWindow && c.server.recycleSharedMemoryWindow(window) { + return + } + if flag == fskitproto.FlagSharedFileWindow && c.server.recycleSharedFileWindow(window) { + return + } + _ = window.Close() +} + +func (c *nativeFSKitConnection) addHandle(coreHandle uint64, name string, flags int, snapshotWrite bool) uint64 { + handle := c.nextHandle + c.nextHandle++ + c.handles[handle] = &nativeFSKitHandle{ + coreHandle: coreHandle, path: name, flags: flags, + snapshotWrite: snapshotWrite, snapshotFloor: -1, + } + return handle +} + +func (c *nativeFSKitConnection) addHealthHandle(name string, flags int) uint64 { + handle := c.nextHandle + c.nextHandle++ + c.handles[handle] = &nativeFSKitHandle{path: name, flags: flags, health: true, snapshotFloor: -1} + return handle +} + +func (c *nativeFSKitConnection) writeHandle(handle *nativeFSKitHandle, data []byte, offset int64) (int, bool, syscall.Errno) { + if !handle.snapshotWrite || len(data) == 0 { + n, errno := c.server.filesystem.Write(handle.coreHandle, data, offset) + return n, false, errno + } + currentPath, errno := c.server.filesystem.HandlePath(handle.coreHandle) + if errno != 0 { + return 0, false, errno + } + handle.path = currentPath + attribute, errno := c.server.filesystem.Getattr(currentPath) + if errno != 0 { + return 0, false, errno + } + currentSize := attribute.Size + if offset < 0 || offset > currentSize { + return c.fallbackSnapshotWrite(handle, data, offset) + } + overlap := min(int64(len(data)), currentSize-offset) + current := make([]byte, overlap) + if overlap > 0 { + n, readErrno := c.server.filesystem.Read(handle.coreHandle, current, offset) + if readErrno != 0 { + return 0, false, readErrno + } + current = current[:n] + } + common := commonPrefixBytes(current, data) + if common == len(data) { + return len(data), true, 0 + } + if common == len(current) && offset+int64(common) == currentSize && completeJSONL(data[common:]) { + floor := currentSize + n, writeErrno := c.server.filesystem.Write(handle.coreHandle, data[common:], currentSize) + if writeErrno == 0 { + handle.snapshotFloor = floor + return len(data), true, 0 + } + return n, true, writeErrno + } + if handle.snapshotFloor >= offset && handle.snapshotFloor <= offset+int64(len(data)) { + floorIndex := int(handle.snapshotFloor - offset) + if floorIndex <= len(current) && bytes.Equal(data[:floorIndex], current[:floorIndex]) && completeJSONL(data[floorIndex:]) { + n, writeErrno := c.server.filesystem.Write(handle.coreHandle, data[floorIndex:], currentSize) + if writeErrno == 0 { + return len(data), true, 0 + } + return n, true, writeErrno + } + } + return c.fallbackSnapshotWrite(handle, data, offset) +} + +func (c *nativeFSKitConnection) fallbackSnapshotWrite(handle *nativeFSKitHandle, data []byte, offset int64) (int, bool, syscall.Errno) { + if errno := c.server.filesystem.UseRandomWrites(handle.coreHandle); errno != 0 { + return 0, false, errno + } + handle.flags &^= os.O_APPEND + handle.snapshotWrite = false + handle.snapshotFloor = -1 + n, writeErrno := c.server.filesystem.Write(handle.coreHandle, data, offset) + return n, false, writeErrno +} + +func commonPrefixBytes(left []byte, right []byte) int { + limit := min(len(left), len(right)) + for index := 0; index < limit; index++ { + if left[index] != right[index] { + return index + } + } + return limit +} + +func (s *nativeFSKitServer) readDir(name string) ([]fskitproto.Entry, syscall.Errno) { + names, errno := s.filesystem.ReadDir(name) + if errno != 0 { + return nil, errno + } + entries := make([]fskitproto.Entry, 0, len(names)) + for _, child := range names { + entry, errno := s.entry(path.Join(name, child)) + if errno == syscall.ENOENT { + continue + } + if errno != 0 { + return nil, errno + } + entries = append(entries, entry) + } + if cleanPath(name) == "/" { + entry, errno := s.entry("/" + mountid.Path) + if errno != 0 { + return nil, errno + } + entries = append(entries, entry) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name }) + return entries, 0 +} + +func (s *nativeFSKitServer) entry(name string) (fskitproto.Entry, syscall.Errno) { + cleaned := cleanPath(name) + if cleaned == "/"+mountid.Path { + return fskitproto.Entry{ + Path: cleaned, Name: mountid.Path, NodeID: 3, ParentID: 2, + Type: fskitproto.EntryFile, Mode: 0o400, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), + Size: uint64(len(s.health)), AllocSize: uint64((len(s.health) + 4095) &^ 4095), + ModTime: s.startedAt, ChangeTime: s.startedAt, AccessTime: s.startedAt, + NamespaceID: s.filesystem.NamespaceVersion(), ContentGeneration: 0, + }, 0 + } + attribute, errno := s.filesystem.Getattr(cleaned) + if errno != 0 { + if errno == syscall.ENOENT { + s.nodes.forget(cleaned) + } + return fskitproto.Entry{}, errno + } + entryType := fskitproto.EntryUnknown + switch attribute.Mode & syscall.S_IFMT { + case syscall.S_IFREG: + entryType = fskitproto.EntryFile + case syscall.S_IFDIR: + entryType = fskitproto.EntryDirectory + case syscall.S_IFLNK: + entryType = fskitproto.EntrySymlink + } + nodeID := s.nodes.node(cleaned, attribute.ObjectID) + parentID := uint64(1) + if cleaned == "/" { + parentID = 1 + } else { + parentPath := path.Dir(cleaned) + if parentAttribute, parentErrno := s.filesystem.Getattr(parentPath); parentErrno == 0 { + parentID = s.nodes.node(parentPath, parentAttribute.ObjectID) + } else { + parentID = s.nodes.node(parentPath, "") + } + } + allocated := uint64(0) + if attribute.Size > 0 { + allocated = uint64((attribute.Size + 4095) &^ 4095) + } + return fskitproto.Entry{ + Path: cleaned, Name: path.Base(cleaned), NodeID: nodeID, ParentID: parentID, + Type: entryType, Mode: attribute.Mode & 0o7777, UID: attribute.UID, GID: attribute.GID, + Size: uint64(max(attribute.Size, 0)), AllocSize: allocated, + ModTime: attribute.ModTime, ChangeTime: attribute.ChangeTime, AccessTime: attribute.AccessTime, + NamespaceID: s.filesystem.NamespaceVersion(), ContentGeneration: attribute.DirectoryGeneration, + }, 0 +} + +func (s *nativeFSKitServer) record(message string) { + if s.recorder != nil { + s.recorder(message) + } +} + +func (n *nativeFSKitNodes) syncVersion(version uint64) { + n.mu.Lock() + defer n.mu.Unlock() + n.version = version +} + +func (n *nativeFSKitNodes) acceptVersion(version uint64) { + n.mu.Lock() + n.version = version + n.mu.Unlock() +} + +func (n *nativeFSKitNodes) node(name string, objectID string) uint64 { + n.mu.Lock() + defer n.mu.Unlock() + if node, exists := n.byPath[name]; exists { + if node.objectID == "" && objectID != "" { + node.objectID = objectID + n.byPath[name] = node + } + if objectID == "" || node.objectID == objectID { + return node.id + } + } + nodeID := n.next + n.next++ + n.byPath[name] = nativeFSKitNode{id: nodeID, objectID: objectID} + return nodeID +} + +func (n *nativeFSKitNodes) forget(name string) { + n.mu.Lock() + delete(n.byPath, name) + n.mu.Unlock() +} + +func (n *nativeFSKitNodes) rename(oldName string, newName string, version uint64) { + n.mu.Lock() + defer n.mu.Unlock() + node, exists := n.byPath[oldName] + if exists { + delete(n.byPath, oldName) + n.byPath[newName] = node + } + n.version = version +} + +func (n *nativeFSKitNodes) remove(name string, version uint64) { + n.mu.Lock() + defer n.mu.Unlock() + delete(n.byPath, name) + n.version = version +} + +func decodePath(decoder *fskitproto.Decoder) (string, syscall.Errno) { + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return "", errno + } + if decoder.Done() != nil { + return "", syscall.EINVAL + } + return name, 0 +} + +func decodePathPrefix(decoder *fskitproto.Decoder) (string, syscall.Errno) { + name, err := decoder.String(1 << 20) + if err != nil || name == "" || cleanPath(name) != name { + return "", syscall.EINVAL + } + return name, 0 +} + +func decodeOpen(decoder *fskitproto.Decoder) (string, uint32, syscall.Errno) { + name, errno := decodePathPrefix(decoder) + if errno != 0 { + return "", 0, errno + } + flags, err := decoder.Uint32() + if err != nil || decoder.Done() != nil { + return "", 0, syscall.EINVAL + } + return name, flags, 0 +} + +func decodeRead(decoder *fskitproto.Decoder, maxPayload uint32) (uint64, int64, int, syscall.Errno) { + handle, err := decoder.Uint64() + if err != nil { + return 0, 0, 0, syscall.EINVAL + } + offset, err := decoder.Int64() + if err != nil || offset < 0 { + return 0, 0, 0, syscall.EINVAL + } + length, err := decoder.Uint32() + if err != nil || length > maxPayload-4 || decoder.Done() != nil { + return 0, 0, 0, syscall.EINVAL + } + return handle, offset, int(length), 0 +} + +func decodeWrite(decoder *fskitproto.Decoder, maxPayload uint32) (uint64, int64, []byte, syscall.Errno) { + handle, err := decoder.Uint64() + if err != nil { + return 0, 0, nil, syscall.EINVAL + } + offset, err := decoder.Int64() + if err != nil || offset < 0 { + return 0, 0, nil, syscall.EINVAL + } + data, err := decoder.Bytes(int(maxPayload) - 20) + if err != nil || decoder.Done() != nil { + return 0, 0, nil, syscall.EINVAL + } + return handle, offset, data, 0 +} + +func nativeFSKitOperationName(operation fskitproto.Op) string { + switch operation { + case fskitproto.OpHello: + return "hello" + case fskitproto.OpPing: + return "ping" + case fskitproto.OpGetattr: + return "getattr" + case fskitproto.OpReadDir: + return "readdir" + case fskitproto.OpOpen: + return "open" + case fskitproto.OpCreate: + return "create" + case fskitproto.OpRead: + return "read" + case fskitproto.OpWrite: + return "write" + case fskitproto.OpFsync: + return "fsync" + case fskitproto.OpFlush: + return "flush" + case fskitproto.OpRelease: + return "release" + case fskitproto.OpTruncate: + return "truncate" + case fskitproto.OpMkdir: + return "mkdir" + case fskitproto.OpRename: + return "rename" + case fskitproto.OpUnlink: + return "unlink" + case fskitproto.OpRmdir: + return "rmdir" + case fskitproto.OpStatfs: + return "statfs" + case fskitproto.OpSync: + return "sync" + case fskitproto.OpNamespaceVersion: + return "namespace_version" + case fskitproto.OpSetattr: + return "setattr" + case fskitproto.OpGetXattr: + return "getxattr" + case fskitproto.OpSetXattr: + return "setxattr" + case fskitproto.OpListXattrs: + return "listxattrs" + default: + return "unknown" + } +} diff --git a/internal/mountfs/native_fskit_server_test.go b/internal/mountfs/native_fskit_server_test.go new file mode 100644 index 0000000..34e03ed --- /dev/null +++ b/internal/mountfs/native_fskit_server_test.go @@ -0,0 +1,729 @@ +package mountfs + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "syscall" + "testing" + "time" + + "github.com/samekind/codexfold/internal/fskitproto" + "github.com/samekind/codexfold/internal/mountid" +) + +func TestNativeFSKitServerPreservesJSONLWritesAndNamespaceMutations(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + for _, directory := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(nativeRoot, directory), 0o700); err != nil { + t.Fatal(err) + } + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + defer stop() + defer client.Close() + + for _, directory := range []string{"/sessions/2026", "/sessions/2026/07", "/sessions/2026/07/17"} { + encoder := fskitproto.NewEncoder(128) + encoder.String(directory) + encoder.Uint32(0o700) + if _, err := client.Call(fskitproto.OpMkdir, encoder.Data()); err != nil { + t.Fatalf("mkdir %s: %v", directory, err) + } + } + + filePath := "/sessions/2026/07/17/session.jsonl" + create := fskitproto.NewEncoder(128) + create.String(filePath) + create.Uint32(uint32(os.O_RDWR | os.O_APPEND)) + created, err := client.Call(fskitproto.OpCreate, create.Data()) + if err != nil { + t.Fatal(err) + } + createdDecoder := fskitproto.NewDecoder(created) + handle, err := createdDecoder.Uint64() + if err != nil { + t.Fatal(err) + } + if _, err := createdDecoder.Entry(); err != nil { + t.Fatal(err) + } + if err := createdDecoder.Done(); err != nil { + t.Fatal(err) + } + + first := []byte("{\"record\":1}\n") + second := []byte("{\"record\":2}\n") + writeNativeFSKitTestPayload(t, client, handle, 0, first) + writeNativeFSKitTestPayload(t, client, handle, int64(len(first)), second) + callNativeFSKitHandle(t, client, fskitproto.OpFsync, handle) + + read := fskitproto.NewEncoder(24) + read.Uint64(handle) + read.Int64(0) + read.Uint32(4096) + readPayload, err := client.Call(fskitproto.OpRead, read.Data()) + if err != nil { + t.Fatal(err) + } + readDecoder := fskitproto.NewDecoder(readPayload) + got, err := readDecoder.Bytes(4096) + if err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), first...), second...) + if !bytes.Equal(got, want) { + t.Fatalf("visible bytes = %q, want %q", got, want) + } + callNativeFSKitHandle(t, client, fskitproto.OpRelease, handle) + + for _, directory := range []string{"/archived_sessions/2026", "/archived_sessions/2026/07", "/archived_sessions/2026/07/17"} { + encoder := fskitproto.NewEncoder(128) + encoder.String(directory) + encoder.Uint32(0o700) + if _, err := client.Call(fskitproto.OpMkdir, encoder.Data()); err != nil { + t.Fatalf("mkdir %s: %v", directory, err) + } + } + archivedPath := "/archived_sessions/2026/07/17/session.jsonl" + rename := fskitproto.NewEncoder(256) + rename.String(filePath) + rename.String(archivedPath) + if _, err := client.Call(fskitproto.OpRename, rename.Data()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(nativeRoot, "archived_sessions/2026/07/17/session.jsonl")); err != nil { + t.Fatal(err) + } + + unlink := fskitproto.NewEncoder(128) + unlink.String(archivedPath) + if _, err := client.Call(fskitproto.OpUnlink, unlink.Data()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(nativeRoot, "archived_sessions/2026/07/17/session.jsonl")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("archived path still exists: %v", err) + } +} + +func TestNativeFSKitServerNamespaceRefreshGuardCannotCreateMissingPaths(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + parent := filepath.Join(nativeRoot, "sessions", "2099", "12", "31") + if err := os.MkdirAll(parent, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(nativeRoot, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + defer stop() + defer client.Close() + + filePath := "/sessions/2099/12/31/raced.jsonl" + releaseFile := filesystem.beginNativeNamespaceRefresh(filePath) + create := fskitproto.NewEncoder(128) + create.String(filePath) + create.Uint32(uint32(os.O_RDONLY)) + if _, err := client.Call(fskitproto.OpCreate, create.Data()); fskitproto.ErrorNumber(err) != syscall.ENOENT { + t.Fatalf("guarded missing create error = %v, want ENOENT", err) + } + releaseFile() + if _, err := os.Lstat(filepath.Join(parent, "raced.jsonl")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("guarded refresh created a missing file: %v", err) + } + + directoryPath := "/sessions/2099/12/31/raced-directory" + releaseDirectory := filesystem.beginNativeNamespaceRefresh(directoryPath) + mkdir := fskitproto.NewEncoder(128) + mkdir.String(directoryPath) + mkdir.Uint32(0o700) + if _, err := client.Call(fskitproto.OpMkdir, mkdir.Data()); fskitproto.ErrorNumber(err) != syscall.ENOENT { + t.Fatalf("guarded missing mkdir error = %v, want ENOENT", err) + } + releaseDirectory() + if _, err := os.Lstat(filepath.Join(parent, "raced-directory")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("guarded refresh created a missing directory: %v", err) + } + + created, err := client.Call(fskitproto.OpCreate, create.Data()) + if err != nil { + t.Fatalf("ordinary create after guard: %v", err) + } + decoder := fskitproto.NewDecoder(created) + handle, err := decoder.Uint64() + if err != nil { + t.Fatal(err) + } + if _, err := decoder.Entry(); err != nil || decoder.Done() != nil { + t.Fatalf("decode ordinary create after guard: %v", err) + } + callNativeFSKitHandle(t, client, fskitproto.OpRelease, handle) + if _, err := client.Call(fskitproto.OpMkdir, mkdir.Data()); err != nil { + t.Fatalf("ordinary mkdir after guard: %v", err) + } +} + +func TestNativeFSKitServerNamespaceRefreshLeavesExistingFileUnchanged(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + parent := filepath.Join(nativeRoot, "sessions", "2099", "12", "31") + if err := os.MkdirAll(parent, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(nativeRoot, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(parent, "external.jsonl") + want := []byte("external-content\n") + if err := os.WriteFile(target, want, 0o640); err != nil { + t.Fatal(err) + } + before, err := os.Lstat(target) + if err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + defer stop() + defer client.Close() + + filePath := "/sessions/2099/12/31/external.jsonl" + release := filesystem.beginNativeNamespaceRefresh(filePath) + create := fskitproto.NewEncoder(128) + create.String(filePath) + create.Uint32(uint32(os.O_RDONLY)) + if _, err := client.Call(fskitproto.OpCreate, create.Data()); fskitproto.ErrorNumber(err) != syscall.EEXIST { + release() + t.Fatalf("guarded existing create error = %v, want EEXIST", err) + } + release() + after, err := os.Lstat(target) + if err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) || after.Mode() != before.Mode() || after.Size() != before.Size() || !after.ModTime().Equal(before.ModTime()) { + t.Fatalf("refresh changed native file: bytes=%q mode=%v size=%d mtime=%s", got, after.Mode(), after.Size(), after.ModTime()) + } +} + +func TestNativeFSKitServerStreamsLargeVirtualReadAsOneResponse(t *testing.T) { + root := t.TempDir() + filesystem := NewCanonical() + filesystem.SetNativeRoot(filepath.Join(root, "native")) + line := []byte("{\"stream\":\"bounded\"}\n") + source := bytes.Repeat(line, (3<<20)/len(line)+1) + virtualPath := "/archived_sessions/large-virtual.jsonl" + if err := filesystem.AddSessionAt("large-virtual", virtualPath, mountSessionFixture(t, "large-virtual", source)); err != nil { + t.Fatal(err) + } + client, stop := startNativeFSKitTestServer(t, filesystem, root) + defer stop() + defer client.Close() + + open := fskitproto.NewEncoder(128) + open.String(virtualPath) + open.Uint32(uint32(os.O_RDONLY)) + response, err := client.Call(fskitproto.OpOpen, open.Data()) + if err != nil { + t.Fatal(err) + } + decoder := fskitproto.NewDecoder(response) + handle, err := decoder.Uint64() + if err != nil || decoder.Done() != nil { + t.Fatalf("open handle=%d err=%v", handle, err) + } + defer callNativeFSKitHandle(t, client, fskitproto.OpRelease, handle) + + read := fskitproto.NewEncoder(24) + read.Uint64(handle) + read.Int64(0) + read.Uint32(uint32(len(source))) + response, err = client.Call(fskitproto.OpRead, read.Data()) + if err != nil { + t.Fatal(err) + } + decoder = fskitproto.NewDecoder(response) + got, err := decoder.Bytes(len(source)) + if err != nil || decoder.Done() != nil || !bytes.Equal(got, source) { + t.Fatalf("large virtual response bytes=%d err=%v", len(got), err) + } + + read = fskitproto.NewEncoder(24) + read.Uint64(handle) + read.Int64(int64(len(source) - 13)) + read.Uint32(4096) + response, err = client.Call(fskitproto.OpRead, read.Data()) + if err != nil { + t.Fatal(err) + } + decoder = fskitproto.NewDecoder(response) + got, err = decoder.Bytes(4096) + if err != nil || decoder.Done() != nil || !bytes.Equal(got, source[len(source)-13:]) { + t.Fatalf("large virtual EOF response bytes=%d err=%v", len(got), err) + } +} + +func TestNativeFSKitServerRejectsWrongToken(t *testing.T) { + root := t.TempDir() + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + if err := client.Close(); err != nil { + t.Fatal(err) + } + defer stop() + + resource, err := os.ReadFile(filepath.Join(root, "resource.bin")) + if err != nil { + t.Fatal(err) + } + descriptor, err := fskitproto.DecodeDescriptor(resource) + if err != nil { + t.Fatal(err) + } + descriptor.Token[0] ^= 0xff + if _, err := fskitproto.Dial(descriptor, time.Second); fskitproto.ErrorNumber(err) != syscall.EACCES { + t.Fatalf("wrong token error = %v, want EACCES", err) + } +} + +func TestNativeFSKitServerExposesReadOnlyMountIdentity(t *testing.T) { + root := t.TempDir() + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + defer stop() + defer client.Close() + + readdir := fskitproto.NewEncoder(16) + readdir.String("/") + response, err := client.Call(fskitproto.OpReadDir, readdir.Data()) + if err != nil { + t.Fatal(err) + } + decoder := fskitproto.NewDecoder(response) + count, err := decoder.Uint32() + if err != nil { + t.Fatal(err) + } + found := false + for range count { + entry, err := decoder.Entry() + if err != nil { + t.Fatal(err) + } + if entry.Name == mountid.Path { + found = true + } + } + if err := decoder.Done(); err != nil || !found { + t.Fatalf("mount identity listed=%t err=%v", found, err) + } + + open := fskitproto.NewEncoder(128) + open.String("/" + mountid.Path) + open.Uint32(uint32(os.O_RDONLY)) + response, err = client.Call(fskitproto.OpOpen, open.Data()) + if err != nil { + t.Fatal(err) + } + decoder = fskitproto.NewDecoder(response) + handle, err := decoder.Uint64() + if err != nil || decoder.Done() != nil { + t.Fatalf("identity open handle=%d err=%v", handle, err) + } + read := fskitproto.NewEncoder(24) + read.Uint64(handle) + read.Int64(0) + read.Uint32(4096) + response, err = client.Call(fskitproto.OpRead, read.Data()) + if err != nil { + t.Fatal(err) + } + decoder = fskitproto.NewDecoder(response) + identityBytes, err := decoder.Bytes(4096) + if err != nil || decoder.Done() != nil { + t.Fatalf("decode identity: %v", err) + } + identity, err := mountid.Parse(identityBytes) + if err != nil || identity.BuildSHA256 != strings.Repeat("a", 64) { + t.Fatalf("mount identity = %#v err=%v", identity, err) + } + callNativeFSKitHandle(t, client, fskitproto.OpRelease, handle) + + writeOpen := fskitproto.NewEncoder(128) + writeOpen.String("/" + mountid.Path) + writeOpen.Uint32(uint32(os.O_WRONLY)) + if _, err := client.Call(fskitproto.OpOpen, writeOpen.Data()); fskitproto.ErrorNumber(err) != syscall.EPERM { + t.Fatalf("write identity open error = %v, want EPERM", err) + } +} + +func TestNativeFSKitServerPublishesDirectoryResourceWithScopedSocket(t *testing.T) { + root := shortNativeFSKitTestDir(t, "cfs-r-") + resource := filepath.Join(root, "native-fskit") + filesystem := NewCanonical() + filesystem.SetNativeRoot(filepath.Join(root, "native")) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + options := NativeFSKitServerOptions{ + SocketPath: filepath.Join(resource, "daemon.sock"), ResourcePath: resource, + Token: bytes.Repeat([]byte{0x24}, 32), Generation: 91, BuildSHA256: strings.Repeat("b", 64), + } + go func() { done <- ServeNativeFSKit(ctx, filesystem, options) }() + defer func() { + cancel() + select { + case err := <-done: + if err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("directory resource server shutdown: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("directory resource server did not stop") + } + }() + deadline := time.Now().Add(5 * time.Second) + var client *fskitproto.Client + var err error + for time.Now().Before(deadline) { + client, err = fskitproto.DialResource(resource, 100*time.Millisecond) + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatalf("dial directory resource: %v", err) + } + defer client.Close() + if _, err := os.Stat(filepath.Join(resource, fskitproto.DescriptorFilename)); err != nil { + t.Fatalf("descriptor file: %v", err) + } + if _, err := os.Lstat(options.SocketPath); err != nil { + t.Fatalf("scoped socket: %v", err) + } +} + +func TestNativeFSKitNodesKeepUnchangedObjectsAndNeverReuseReplacements(t *testing.T) { + nodes := nativeFSKitNodes{next: 3, byPath: map[string]nativeFSKitNode{"/": {id: 2}}} + first := nodes.node("/sessions/first", "native:1:10") + nodes.syncVersion(2) + if unchanged := nodes.node("/sessions/first", "native:1:10"); unchanged != first { + t.Fatalf("unchanged object ID = %d, want %d", unchanged, first) + } + replaced := nodes.node("/sessions/first", "native:1:11") + if replaced == first || replaced <= first { + t.Fatalf("replacement object ID = %d, want a fresh ID after %d", replaced, first) + } + nodes.forget("/sessions/first") + recreated := nodes.node("/sessions/first", "native:1:11") + if recreated == replaced || recreated <= replaced { + t.Fatalf("recreated object ID = %d, want a fresh ID after %d", recreated, replaced) + } +} + +func TestNativeFSKitHelloNegotiatesContentGenerationWithoutBreakingLegacyPeers(t *testing.T) { + token := []byte("0123456789abcdef") + connection := &nativeFSKitConnection{server: &nativeFSKitServer{ + filesystem: NewCanonical(), token: token, maxPayload: fskitproto.DefaultMaxPayload, + }} + + legacyRequest := fskitproto.NewEncoder(32) + legacyRequest.Bytes(token) + legacyResponse, status := connection.hello(legacyRequest.Data()) + if status != 0 { + t.Fatalf("legacy hello status = %d", status) + } + legacyDecoder := fskitproto.NewDecoder(legacyResponse) + if _, err := legacyDecoder.Uint32(); err != nil { + t.Fatal(err) + } + if _, err := legacyDecoder.Uint64(); err != nil || legacyDecoder.Done() != nil { + t.Fatalf("legacy hello response changed shape: %v", err) + } + + negotiatedRequest := fskitproto.NewEncoder(36) + negotiatedRequest.Bytes(token) + negotiatedRequest.Uint32(fskitproto.CapabilityContentGeneration) + negotiatedResponse, status := connection.hello(negotiatedRequest.Data()) + if status != 0 { + t.Fatalf("negotiated hello status = %d", status) + } + negotiatedDecoder := fskitproto.NewDecoder(negotiatedResponse) + if _, err := negotiatedDecoder.Uint32(); err != nil { + t.Fatal(err) + } + if _, err := negotiatedDecoder.Uint64(); err != nil { + t.Fatal(err) + } + accepted, err := negotiatedDecoder.Uint32() + if err != nil || negotiatedDecoder.Done() != nil { + t.Fatalf("negotiated hello response decode: accepted=%#x err=%v", accepted, err) + } + if accepted != fskitproto.CapabilityContentGeneration { + t.Fatalf("accepted capabilities = %#x, want %#x", accepted, fskitproto.CapabilityContentGeneration) + } +} + +func TestNativeFSKitServerObjectIDsFollowNativeIdentityNotNamespaceVersion(t *testing.T) { + root := t.TempDir() + for _, namespace := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(root, namespace), 0o700); err != nil { + t.Fatal(err) + } + } + target := filepath.Join(root, "sessions", "target.jsonl") + if err := os.WriteFile(target, []byte("first\n"), 0o600); err != nil { + t.Fatal(err) + } + + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + server := &nativeFSKitServer{ + filesystem: filesystem, + nodes: nativeFSKitNodes{ + version: filesystem.NamespaceVersion(), + next: 4, + byPath: map[string]nativeFSKitNode{"/": {id: 2}}, + }, + } + first, errno := server.entry("/sessions/target.jsonl") + if errno != 0 { + t.Fatalf("first entry: %v", errno) + } + parentBefore, errno := server.entry("/sessions") + if errno != 0 { + t.Fatalf("parent before generation change: %v", errno) + } + filesystem.bumpDirectoryGeneration("/sessions") + filesystem.bumpNamespaceVersion() + server.nodes.syncVersion(filesystem.NamespaceVersion()) + parentChanged, errno := server.entry("/sessions/target.jsonl") + if errno != 0 { + t.Fatalf("entry after parent generation: %v", errno) + } + if parentChanged.NodeID != first.NodeID { + t.Fatalf("parent generation replaced child object ID %d with %d", first.NodeID, parentChanged.NodeID) + } + if parentChanged.ParentID != first.ParentID { + t.Fatalf("parent generation replaced stable parent ID %d with %d", first.ParentID, parentChanged.ParentID) + } + parentAfter, errno := server.entry("/sessions") + if errno != 0 { + t.Fatalf("parent after generation change: %v", errno) + } + if parentAfter.NodeID != parentBefore.NodeID { + t.Fatalf("parent generation replaced parent node ID %d with %d", parentBefore.NodeID, parentAfter.NodeID) + } + if parentAfter.ContentGeneration <= parentBefore.ContentGeneration { + t.Fatalf("parent content generation did not advance: before=%d after=%d", parentBefore.ContentGeneration, parentAfter.ContentGeneration) + } + first = parentChanged + + if err := os.WriteFile(filepath.Join(root, "archived_sessions", "unrelated.jsonl"), []byte("unrelated\n"), 0o600); err != nil { + t.Fatal(err) + } + filesystem.bumpNamespaceVersion() + server.nodes.syncVersion(filesystem.NamespaceVersion()) + unchanged, errno := server.entry("/sessions/target.jsonl") + if errno != 0 { + t.Fatalf("unchanged entry: %v", errno) + } + if unchanged.NodeID != first.NodeID { + t.Fatalf("unrelated namespace change replaced object ID %d with %d", first.NodeID, unchanged.NodeID) + } + + oldFile, err := os.Open(target) + if err != nil { + t.Fatal(err) + } + defer oldFile.Close() + if err := os.Remove(target); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, []byte("second\n"), 0o600); err != nil { + t.Fatal(err) + } + filesystem.bumpNamespaceVersion() + server.nodes.syncVersion(filesystem.NamespaceVersion()) + replaced, errno := server.entry("/sessions/target.jsonl") + if errno != 0 { + t.Fatalf("replacement entry: %v", errno) + } + if replaced.NodeID == first.NodeID || replaced.NodeID <= first.NodeID { + t.Fatalf("replacement object ID = %d, want a fresh ID after %d", replaced.NodeID, first.NodeID) + } +} + +func TestNativeFSKitServerNormalizesFSKitWholeFileSnapshotsIntoJSONLAppends(t *testing.T) { + root := t.TempDir() + nativeRoot := filepath.Join(root, "native") + targetDirectory := filepath.Join(nativeRoot, "sessions", "2026", "07", "17") + if err := os.MkdirAll(targetDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(nativeRoot, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + base := []byte("{\"record\":0}\n") + target := filepath.Join(targetDirectory, "session.jsonl") + if err := os.WriteFile(target, base, 0o600); err != nil { + t.Fatal(err) + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(nativeRoot) + client, stop := startNativeFSKitTestServer(t, filesystem, root) + defer stop() + defer client.Close() + + open := fskitproto.NewEncoder(128) + open.String("/sessions/2026/07/17/session.jsonl") + open.Uint32(uint32(os.O_RDWR|os.O_APPEND) | fskitproto.OpenFlagSnapshot) + response, err := client.Call(fskitproto.OpOpen, open.Data()) + if err != nil { + t.Fatal(err) + } + decoder := fskitproto.NewDecoder(response) + handle, err := decoder.Uint64() + if err != nil || decoder.Done() != nil { + t.Fatalf("open handle=%d err=%v", handle, err) + } + first := append(append([]byte(nil), base...), []byte("{\"record\":1}\n")...) + second := append(append([]byte(nil), base...), []byte("{\"record\":2}\n")...) + writeNativeFSKitTestPayload(t, client, handle, 0, first) + writeNativeFSKitTestPayload(t, client, handle, 0, second) + callNativeFSKitHandle(t, client, fskitproto.OpFsync, handle) + callNativeFSKitHandle(t, client, fskitproto.OpRelease, handle) + + got, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + want := append(append(append([]byte(nil), base...), []byte("{\"record\":1}\n")...), []byte("{\"record\":2}\n")...) + if !bytes.Equal(got, want) { + t.Fatalf("normalized bytes = %q, want %q", got, want) + } + + reopen := fskitproto.NewEncoder(128) + reopen.String("/sessions/2026/07/17/session.jsonl") + reopen.Uint32(uint32(os.O_RDWR|os.O_APPEND) | fskitproto.OpenFlagSnapshot) + response, err = client.Call(fskitproto.OpOpen, reopen.Data()) + if err != nil { + t.Fatal(err) + } + decoder = fskitproto.NewDecoder(response) + handle, err = decoder.Uint64() + if err != nil || decoder.Done() != nil { + t.Fatalf("reopen handle=%d err=%v", handle, err) + } + replacement := bytes.Replace(want, []byte("{\"record\":0}"), []byte("{\"record\":9}"), 1) + writeNativeFSKitTestPayload(t, client, handle, 0, replacement) + callNativeFSKitHandle(t, client, fskitproto.OpFsync, handle) + callNativeFSKitHandle(t, client, fskitproto.OpRelease, handle) + got, err = os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, replacement) { + t.Fatalf("random snapshot bytes = %q, want %q", got, replacement) + } +} + +func startNativeFSKitTestServer(t *testing.T, filesystem *Filesystem, root string) (*fskitproto.Client, func()) { + t.Helper() + socketRoot := shortNativeFSKitTestDir(t, "cfs-") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + options := NativeFSKitServerOptions{ + SocketPath: filepath.Join(socketRoot, "daemon.sock"), ResourcePath: filepath.Join(root, "resource.bin"), + Token: bytes.Repeat([]byte{0x42}, 32), Generation: 77, BuildSHA256: strings.Repeat("a", 64), + } + go func() { done <- ServeNativeFSKit(ctx, filesystem, options) }() + deadline := time.Now().Add(5 * time.Second) + var client *fskitproto.Client + var dialErr error + for time.Now().Before(deadline) { + select { + case serveErr := <-done: + cancel() + t.Fatalf("FSKit test server exited during startup: %v", serveErr) + default: + } + client, dialErr = fskitproto.DialResource(options.ResourcePath, 100*time.Millisecond) + if dialErr == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if dialErr != nil { + cancel() + t.Fatalf("start FSKit test server: %v", dialErr) + } + stop := func() { + cancel() + select { + case err := <-done: + if err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("FSKit server shutdown: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("FSKit server did not stop") + } + } + return client, stop +} + +func shortNativeFSKitTestDir(t *testing.T, pattern string) string { + t.Helper() + base := os.TempDir() + if runtime.GOOS == "darwin" || runtime.GOOS == "linux" { + base = "/tmp" + } + root, err := os.MkdirTemp(base, pattern) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(root) }) + return root +} + +func writeNativeFSKitTestPayload(t *testing.T, client *fskitproto.Client, handle uint64, offset int64, data []byte) { + t.Helper() + encoder := fskitproto.NewEncoder(20 + len(data)) + encoder.Uint64(handle) + encoder.Int64(offset) + encoder.Bytes(data) + response, err := client.Call(fskitproto.OpWrite, encoder.Data()) + if err != nil { + t.Fatal(err) + } + decoder := fskitproto.NewDecoder(response) + written, err := decoder.Uint32() + if err != nil || decoder.Done() != nil || int(written) != len(data) { + t.Fatalf("write response bytes=%d err=%v", written, err) + } +} + +func callNativeFSKitHandle(t *testing.T, client *fskitproto.Client, operation fskitproto.Op, handle uint64) { + t.Helper() + encoder := fskitproto.NewEncoder(8) + encoder.Uint64(handle) + if _, err := client.Call(operation, encoder.Data()); err != nil { + t.Fatal(err) + } +} diff --git a/internal/mountfs/native_fskit_stat_darwin.go b/internal/mountfs/native_fskit_stat_darwin.go new file mode 100644 index 0000000..cd837bc --- /dev/null +++ b/internal/mountfs/native_fskit_stat_darwin.go @@ -0,0 +1,34 @@ +//go:build darwin + +package mountfs + +import ( + "os" + "path/filepath" + + "github.com/samekind/codexfold/internal/fskitproto" + "golang.org/x/sys/unix" +) + +func nativeFSKitStat(root string) (fskitproto.StatFS, error) { + if root == "" { + root = os.TempDir() + } + root, err := filepath.Abs(root) + if err != nil { + return fskitproto.StatFS{}, err + } + var stat unix.Statfs_t + if err := unix.Statfs(root, &stat); err != nil { + return fskitproto.StatFS{}, err + } + blockSize := uint64(stat.Bsize) + total := stat.Blocks * blockSize + available := stat.Bavail * blockSize + free := stat.Bfree * blockSize + return fskitproto.StatFS{ + BlockSize: uint32(stat.Bsize), IOSize: 4 * 1024 * 1024, + TotalBytes: total, AvailableBytes: available, FreeBytes: free, UsedBytes: total - free, + TotalFiles: stat.Files, FreeFiles: stat.Ffree, + }, nil +} diff --git a/internal/mountfs/native_fskit_stat_other.go b/internal/mountfs/native_fskit_stat_other.go new file mode 100644 index 0000000..597959e --- /dev/null +++ b/internal/mountfs/native_fskit_stat_other.go @@ -0,0 +1,19 @@ +//go:build !darwin + +package mountfs + +import ( + "os" + + "github.com/samekind/codexfold/internal/fskitproto" +) + +func nativeFSKitStat(string) (fskitproto.StatFS, error) { + return fskitproto.StatFS{ + BlockSize: 4096, IOSize: 4 * 1024 * 1024, + TotalBytes: 1 << 40, AvailableBytes: 1 << 39, FreeBytes: 1 << 39, UsedBytes: 1 << 39, + TotalFiles: 1 << 32, FreeFiles: 1 << 31, + }, nil +} + +var _ = os.ErrNotExist diff --git a/internal/mountfs/native_namespace_watch_darwin.go b/internal/mountfs/native_namespace_watch_darwin.go new file mode 100644 index 0000000..e7a3126 --- /dev/null +++ b/internal/mountfs/native_namespace_watch_darwin.go @@ -0,0 +1,392 @@ +//go:build darwin + +package mountfs + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +type nativeNamespaceSnapshotEntry struct { + directory bool + objectID string + mode fs.FileMode + uid uint32 + gid uint32 + size int64 + modTime int64 + changeTime int64 +} + +type nativeNamespaceRefreshEntry struct { + route string + directory bool +} + +type nativeNamespaceWatcher struct { + path string + directory bool +} + +type nativeNamespaceSnapshot map[string]nativeNamespaceSnapshotEntry + +func (f *Filesystem) WatchNativeNamespace(ctx context.Context) error { + f.mu.RLock() + root := f.nativeRoot + f.mu.RUnlock() + if root == "" { + return nil + } + queue, err := unix.Kqueue() + if err != nil { + return fmt.Errorf("create native namespace kqueue: %w", err) + } + defer unix.Close(queue) + watchers := make(map[int]nativeNamespaceWatcher) + watchersByPath := make(map[string]int) + closeWatchers := func() { + for descriptor := range watchers { + _ = unix.Close(descriptor) + } + clear(watchers) + clear(watchersByPath) + } + defer closeWatchers() + rescan := func() (nativeNamespaceSnapshot, error) { + seen := make(map[string]struct{}) + snapshot, err := scanNativeNamespace(root, func(name string, directory bool) error { + name = filepath.Clean(name) + seen[name] = struct{}{} + if descriptor, exists := watchersByPath[name]; exists { + watcher := watchers[descriptor] + watcher.directory = directory + watchers[descriptor] = watcher + return nil + } + descriptor, err := unix.Open(name, unix.O_EVTONLY|unix.O_CLOEXEC, 0) + if err != nil { + if errors.Is(err, unix.ENOENT) { + return nil + } + return err + } + event := unix.Kevent_t{ + Ident: uint64(descriptor), Filter: unix.EVFILT_VNODE, + Flags: unix.EV_ADD | unix.EV_ENABLE | unix.EV_CLEAR, + Fflags: unix.NOTE_WRITE | unix.NOTE_EXTEND | unix.NOTE_DELETE | unix.NOTE_RENAME | + unix.NOTE_ATTRIB | unix.NOTE_LINK | unix.NOTE_REVOKE, + } + if _, err := unix.Kevent(queue, []unix.Kevent_t{event}, nil, nil); err != nil { + _ = unix.Close(descriptor) + return err + } + watchers[descriptor] = nativeNamespaceWatcher{path: name, directory: directory} + watchersByPath[name] = descriptor + return nil + }) + if err != nil { + return nil, err + } + for name, descriptor := range watchersByPath { + if _, exists := seen[name]; exists { + continue + } + _ = unix.Close(descriptor) + delete(watchers, descriptor) + delete(watchersByPath, name) + } + return snapshot, nil + } + snapshot, err := rescan() + if err != nil { + return err + } + pendingRefreshes := make(map[string]nativeNamespaceRefreshEntry) + f.bumpNamespaceVersion() + events := make([]unix.Kevent_t, 64) + for { + if err := ctx.Err(); err != nil { + return err + } + timeout := unix.NsecToTimespec((500 * time.Millisecond).Nanoseconds()) + count, err := unix.Kevent(queue, nil, events, &timeout) + if err != nil { + if errors.Is(err, unix.EINTR) { + continue + } + return fmt.Errorf("wait for native namespace event: %w", err) + } + if count > 0 { + changedSet := make(map[string]struct{}) + changedFiles := make(map[string]nativeNamespaceWatcher) + internalFiles := make(map[string]nativeNamespaceWatcher) + externalChange := false + fullRescan := false + for _, event := range events[:count] { + watcher, exists := watchers[int(event.Ident)] + if !exists { + continue + } + route, ok := nativeNamespaceRoute(root, watcher.path) + if !ok { + continue + } + if !watcher.directory && f.nativeInternalMutationSuppressed(watcher.path) { + internalFiles[route] = watcher + continue + } + externalChange = true + if watcher.directory { + changedSet[route] = struct{}{} + fullRescan = true + } else { + changedFiles[route] = watcher + } + if !watcher.directory || event.Fflags&(unix.NOTE_DELETE|unix.NOTE_RENAME) != 0 { + changedSet[filepath.ToSlash(filepath.Dir(route))] = struct{}{} + } + if event.Fflags&(unix.NOTE_DELETE|unix.NOTE_RENAME|unix.NOTE_REVOKE) != 0 { + fullRescan = true + } + } + current := cloneNativeNamespaceSnapshot(snapshot) + if fullRescan { + current, err = rescan() + if err != nil { + return err + } + } else { + for route, watcher := range changedFiles { + entry, exists, entryErr := nativeNamespaceEntry(watcher.path) + if entryErr != nil { + return fmt.Errorf("refresh native namespace entry %s: %w", route, entryErr) + } + if exists { + current[route] = entry + } else { + delete(current, route) + } + } + for route, watcher := range internalFiles { + entry, exists, entryErr := nativeNamespaceEntry(watcher.path) + if entryErr != nil { + return fmt.Errorf("refresh internal native namespace entry %s: %w", route, entryErr) + } + if exists { + current[route] = entry + } else { + delete(current, route) + } + } + } + comparison := snapshot + if len(internalFiles) != 0 { + comparison = cloneNativeNamespaceSnapshot(snapshot) + for route := range internalFiles { + if entry, exists := current[route]; exists { + comparison[route] = entry + } else { + delete(comparison, route) + } + } + } + if externalChange { + mergeNativeNamespaceRefreshEntries(pendingRefreshes, nativeNamespaceRefreshDelta(comparison, current)) + } + snapshot = current + if !externalChange { + continue + } + changed := make([]string, 0, len(changedSet)) + for route := range changedSet { + changed = append(changed, route) + } + f.bumpDirectoryGenerations(changed) + f.bumpNamespaceVersion() + } + // A mount can be briefly unavailable while FSKit starts or remounts. Keep + // failed name-cache repairs pending instead of terminating the daemon. + retryNativeNamespaceRefreshes(pendingRefreshes, f.refreshNativeNamespacePath) + } +} + +func scanNativeNamespace(root string, watchEntry func(string, bool) error) (nativeNamespaceSnapshot, error) { + snapshot := make(nativeNamespaceSnapshot) + for _, namespace := range []string{"sessions", "archived_sessions"} { + base := filepath.Join(root, namespace) + err := filepath.WalkDir(base, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + if errors.Is(walkErr, os.ErrNotExist) { + return nil + } + return walkErr + } + route, ok := nativeNamespaceRoute(root, name) + if !ok { + return nil + } + rootDirectory := route == "/sessions" || route == "/archived_sessions" + if !rootDirectory && !nativeNamespaceRefreshCandidate(route) { + if entry.IsDir() { + return fs.SkipDir + } + return nil + } + info, err := entry.Info() + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() && !info.Mode().IsRegular() { + return nil + } + if watchEntry != nil { + if err := watchEntry(name, info.IsDir()); err != nil { + return err + } + info, err = os.Lstat(name) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + } + if rootDirectory { + return nil + } + snapshot[route] = nativeNamespaceEntryFromInfo(info) + return nil + }) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("scan native namespace %s: %w", namespace, err) + } + } + return snapshot, nil +} + +func nativeNamespaceEntry(name string) (nativeNamespaceSnapshotEntry, bool, error) { + info, err := os.Lstat(name) + if errors.Is(err, os.ErrNotExist) { + return nativeNamespaceSnapshotEntry{}, false, nil + } + if err != nil { + return nativeNamespaceSnapshotEntry{}, false, err + } + if !info.IsDir() && !info.Mode().IsRegular() { + return nativeNamespaceSnapshotEntry{}, false, nil + } + return nativeNamespaceEntryFromInfo(info), true, nil +} + +func nativeNamespaceEntryFromInfo(info os.FileInfo) nativeNamespaceSnapshotEntry { + objectID := fileObjectIdentity(info) + if objectID == "" { + objectID = fmt.Sprintf("fallback:%d:%d", info.Size(), info.ModTime().UnixNano()) + } + entry := nativeNamespaceSnapshotEntry{directory: info.IsDir(), objectID: objectID} + if info.IsDir() { + return entry + } + uid, gid, _, changeTime := fileOwnershipAndTimes(info) + entry.mode = info.Mode() + entry.uid = uid + entry.gid = gid + entry.size = info.Size() + entry.modTime = info.ModTime().UnixNano() + entry.changeTime = changeTime.UnixNano() + return entry +} + +func cloneNativeNamespaceSnapshot(snapshot nativeNamespaceSnapshot) nativeNamespaceSnapshot { + cloned := make(nativeNamespaceSnapshot, len(snapshot)) + for route, entry := range snapshot { + cloned[route] = entry + } + return cloned +} + +func nativeNamespaceRefreshCandidate(route string) bool { + if !canonicalNamespacePath(route) { + return false + } + name := path.Base(route) + return name != ".DS_Store" && !strings.HasPrefix(name, ".") +} + +func nativeNamespaceRefreshDelta(previous nativeNamespaceSnapshot, current nativeNamespaceSnapshot) []nativeNamespaceRefreshEntry { + entries := make([]nativeNamespaceRefreshEntry, 0) + for route, entry := range current { + old, exists := previous[route] + if exists && old == entry { + continue + } + entries = append(entries, nativeNamespaceRefreshEntry{route: route, directory: entry.directory}) + } + sortNativeNamespaceRefreshEntries(entries) + return entries +} + +func mergeNativeNamespaceRefreshEntries( + pending map[string]nativeNamespaceRefreshEntry, + entries []nativeNamespaceRefreshEntry, +) { + for _, entry := range entries { + pending[entry.route] = entry + } +} + +func retryNativeNamespaceRefreshes( + pending map[string]nativeNamespaceRefreshEntry, + refresh func(string, bool) error, +) { + entries := make([]nativeNamespaceRefreshEntry, 0, len(pending)) + for _, entry := range pending { + entries = append(entries, entry) + } + sortNativeNamespaceRefreshEntries(entries) + for _, entry := range entries { + if err := refresh(entry.route, entry.directory); err == nil { + delete(pending, entry.route) + } + } +} + +func sortNativeNamespaceRefreshEntries(entries []nativeNamespaceRefreshEntry) { + sort.Slice(entries, func(left int, right int) bool { + leftDepth := strings.Count(entries[left].route, "/") + rightDepth := strings.Count(entries[right].route, "/") + if leftDepth != rightDepth { + return leftDepth < rightDepth + } + if entries[left].directory != entries[right].directory { + return entries[left].directory + } + return entries[left].route < entries[right].route + }) +} + +func nativeNamespaceRoute(root string, nativePath string) (string, bool) { + relative, err := filepath.Rel(filepath.Clean(root), filepath.Clean(nativePath)) + if err != nil || relative == ".." || relative == "." || filepath.IsAbs(relative) || + len(relative) > 3 && relative[:3] == ".."+string(filepath.Separator) { + return "", false + } + route := "/" + filepath.ToSlash(relative) + if !canonicalNamespacePath(route) { + return "", false + } + return route, true +} diff --git a/internal/mountfs/native_namespace_watch_darwin_test.go b/internal/mountfs/native_namespace_watch_darwin_test.go new file mode 100644 index 0000000..760b24c --- /dev/null +++ b/internal/mountfs/native_namespace_watch_darwin_test.go @@ -0,0 +1,211 @@ +//go:build darwin + +package mountfs + +import ( + "context" + "errors" + "os" + "path/filepath" + "slices" + "testing" + "time" +) + +func TestNativeNamespaceSnapshotRefreshesOnlyVisibleNewAndReplacedEntries(t *testing.T) { + root := t.TempDir() + for _, namespace := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(root, namespace), 0o700); err != nil { + t.Fatal(err) + } + } + baseline, err := scanNativeNamespace(root, nil) + if err != nil { + t.Fatal(err) + } + directory := filepath.Join(root, "sessions", "2099", "12", "31") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(directory, "external.jsonl") + if err := os.WriteFile(target, []byte("one\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(directory, "._external.jsonl"), []byte("sidecar"), 0o600); err != nil { + t.Fatal(err) + } + hiddenDirectory := filepath.Join(directory, ".hidden") + if err := os.MkdirAll(hiddenDirectory, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(hiddenDirectory, "ignored.jsonl"), []byte("hidden"), 0o600); err != nil { + t.Fatal(err) + } + current, err := scanNativeNamespace(root, nil) + if err != nil { + t.Fatal(err) + } + want := []nativeNamespaceRefreshEntry{ + {route: "/sessions/2099", directory: true}, + {route: "/sessions/2099/12", directory: true}, + {route: "/sessions/2099/12/31", directory: true}, + {route: "/sessions/2099/12/31/external.jsonl", directory: false}, + } + if got := nativeNamespaceRefreshDelta(baseline, current); !slices.Equal(got, want) { + t.Fatalf("new namespace refresh entries = %#v, want %#v", got, want) + } + + if err := os.WriteFile(target, []byte("same-inode-update\n"), 0o600); err != nil { + t.Fatal(err) + } + modified, err := scanNativeNamespace(root, nil) + if err != nil { + t.Fatal(err) + } + want = []nativeNamespaceRefreshEntry{{route: "/sessions/2099/12/31/external.jsonl", directory: false}} + if got := nativeNamespaceRefreshDelta(current, modified); !slices.Equal(got, want) { + t.Fatalf("same-inode data refresh entries = %#v, want %#v", got, want) + } + + oldTarget := filepath.Join(root, "old-external.jsonl") + if err := os.Rename(target, oldTarget); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, []byte("replacement\n"), 0o600); err != nil { + t.Fatal(err) + } + replaced, err := scanNativeNamespace(root, nil) + if err != nil { + t.Fatal(err) + } + want = []nativeNamespaceRefreshEntry{{route: "/sessions/2099/12/31/external.jsonl", directory: false}} + if got := nativeNamespaceRefreshDelta(modified, replaced); !slices.Equal(got, want) { + t.Fatalf("replacement namespace refresh entries = %#v, want %#v", got, want) + } +} + +func TestWatchNativeNamespaceBumpsVersionForExternalEntryChanges(t *testing.T) { + root := t.TempDir() + for _, namespace := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(root, namespace), 0o700); err != nil { + t.Fatal(err) + } + } + filesystem := NewCanonical() + filesystem.SetNativeRoot(root) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- filesystem.WatchNativeNamespace(ctx) }() + initial := filesystem.NamespaceVersion() + readyDeadline := time.Now().Add(5 * time.Second) + for filesystem.NamespaceVersion() == initial && time.Now().Before(readyDeadline) { + time.Sleep(10 * time.Millisecond) + } + if filesystem.NamespaceVersion() == initial { + t.Fatal("native namespace watcher did not become ready") + } + baseline := filesystem.NamespaceVersion() + sessionsBefore, errno := filesystem.Getattr("/sessions") + if errno != 0 { + t.Fatalf("sessions Getattr before change errno=%v", errno) + } + archivedBefore, errno := filesystem.Getattr("/archived_sessions") + if errno != 0 { + t.Fatalf("archived Getattr before change errno=%v", errno) + } + target := filepath.Join(root, "sessions", "external.jsonl") + if err := os.WriteFile(target, []byte("{}\n"), 0o600); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for filesystem.NamespaceVersion() == baseline && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if filesystem.NamespaceVersion() == baseline { + t.Fatal("external namespace creation did not bump the version") + } + sessionsAfter, errno := filesystem.Getattr("/sessions") + if errno != 0 { + t.Fatalf("sessions Getattr after change errno=%v", errno) + } + if sessionsAfter.ObjectID != sessionsBefore.ObjectID { + t.Fatalf("changed directory replaced stable object ID %q with %q", sessionsBefore.ObjectID, sessionsAfter.ObjectID) + } + if sessionsAfter.DirectoryGeneration <= sessionsBefore.DirectoryGeneration { + t.Fatalf("changed directory generation did not advance: before=%d after=%d", sessionsBefore.DirectoryGeneration, sessionsAfter.DirectoryGeneration) + } + archivedAfter, errno := filesystem.Getattr("/archived_sessions") + if errno != 0 { + t.Fatalf("archived Getattr after change errno=%v", errno) + } + if archivedAfter.ObjectID != archivedBefore.ObjectID { + t.Fatalf("unrelated directory identity changed from %q to %q", archivedBefore.ObjectID, archivedAfter.ObjectID) + } + + baseline = filesystem.NamespaceVersion() + file, err := os.OpenFile(target, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.Write([]byte("{\"updated\":true}\n")); err != nil { + file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + deadline = time.Now().Add(5 * time.Second) + for filesystem.NamespaceVersion() == baseline && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if filesystem.NamespaceVersion() == baseline { + t.Fatal("same-inode external append did not bump the version") + } + attribute, errno := filesystem.Getattr("/sessions/external.jsonl") + if errno != 0 || attribute.Size != int64(len("{}\n{\"updated\":true}\n")) { + t.Fatalf("same-inode external append size=%d errno=%v", attribute.Size, errno) + } + cancel() + select { + case err := <-done: + if err != nil && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("native namespace watcher did not stop") + } +} + +func TestRetryNativeNamespaceRefreshesKeepsTransientFailuresPending(t *testing.T) { + pending := make(map[string]nativeNamespaceRefreshEntry) + mergeNativeNamespaceRefreshEntries(pending, []nativeNamespaceRefreshEntry{ + {route: "/sessions/2099/12/31/external.jsonl", directory: false}, + {route: "/sessions/2099", directory: true}, + }) + + var firstAttempt []string + retryNativeNamespaceRefreshes(pending, func(route string, directory bool) error { + firstAttempt = append(firstAttempt, route) + if route == "/sessions/2099" { + return errors.New("mount is temporarily unavailable") + } + return nil + }) + wantFirstAttempt := []string{"/sessions/2099", "/sessions/2099/12/31/external.jsonl"} + if !slices.Equal(firstAttempt, wantFirstAttempt) { + t.Fatalf("first refresh order = %v, want %v", firstAttempt, wantFirstAttempt) + } + if len(pending) != 1 || !pending["/sessions/2099"].directory { + t.Fatalf("pending refreshes after transient failure = %#v", pending) + } + + retryNativeNamespaceRefreshes(pending, func(route string, directory bool) error { + if route != "/sessions/2099" || !directory { + t.Fatalf("retried refresh = %q directory=%t", route, directory) + } + return nil + }) + if len(pending) != 0 { + t.Fatalf("pending refreshes after recovery = %#v", pending) + } +} diff --git a/internal/mountfs/native_namespace_watch_other.go b/internal/mountfs/native_namespace_watch_other.go new file mode 100644 index 0000000..180636e --- /dev/null +++ b/internal/mountfs/native_namespace_watch_other.go @@ -0,0 +1,10 @@ +//go:build !darwin + +package mountfs + +import "context" + +func (f *Filesystem) WatchNativeNamespace(ctx context.Context) error { + <-ctx.Done() + return ctx.Err() +} diff --git a/internal/mountfs/native_preflight.go b/internal/mountfs/native_preflight.go new file mode 100644 index 0000000..eba864f --- /dev/null +++ b/internal/mountfs/native_preflight.go @@ -0,0 +1,105 @@ +package mountfs + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "unicode/utf8" +) + +type NativePreflightReport struct { + Files int `json:"files"` + Bytes int64 `json:"bytes"` + ValidatedFiles int `json:"validated_files"` + IncrementalFiles int `json:"incremental_files"` + CachedFiles int `json:"cached_files"` + ValidatedBytes int64 `json:"validated_bytes"` + CachePath string `json:"cache_path,omitempty"` + CacheRebuilt bool `json:"cache_rebuilt,omitempty"` +} + +func (f *Filesystem) ValidateNativeWriterRollouts(ctx context.Context) (NativePreflightReport, error) { + f.mu.RLock() + root := f.nativeRoot + f.mu.RUnlock() + if root == "" { + return NativePreflightReport{}, nil + } + return validateNativeWriterRollouts(ctx, root) +} + +func validateNativeWriterRollouts(ctx context.Context, nativeRoot string) (NativePreflightReport, error) { + return validateNativeWriterRolloutsCached(ctx, nativeRoot) +} + +func ValidateNativeRollout(ctx context.Context, filePath string) (int64, error) { + path := filepath.Clean(filePath) + before, err := os.Lstat(path) + if err != nil { + return 0, fmt.Errorf("inspect native rollout %s: %w", path, err) + } + if !before.Mode().IsRegular() { + return 0, fmt.Errorf("native rollout %s is not a regular file", path) + } + validated, err := validateNativeJSONL(ctx, path) + if err != nil { + return validated, err + } + after, err := os.Lstat(path) + if err != nil { + return validated, fmt.Errorf("reinspect native rollout %s: %w", path, err) + } + if !after.Mode().IsRegular() || !os.SameFile(before, after) || before.Size() != after.Size() || before.ModTime() != after.ModTime() { + return validated, fmt.Errorf("native rollout changed during validation: %s", path) + } + return validated, nil +} + +func validateNativeJSONL(ctx context.Context, filePath string) (int64, error) { + file, err := os.Open(filePath) + if err != nil { + return 0, err + } + reader := bufio.NewReaderSize(file, 1<<20) + var lineNumber int64 + var bytesRead int64 + for { + if err := ctx.Err(); err != nil { + _ = file.Close() + return bytesRead, err + } + line, readErr := reader.ReadBytes('\n') + bytesRead += int64(len(line)) + if len(line) != 0 { + lineNumber++ + } + if errors.Is(readErr, io.EOF) { + if len(line) != 0 { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight %s line %d is missing its final newline", filePath, lineNumber) + } + break + } + if readErr != nil { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight read %s line %d: %w", filePath, lineNumber, readErr) + } + if !utf8.Valid(line) { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight %s line %d is not valid UTF-8", filePath, lineNumber) + } + if !json.Valid(line) { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight %s line %d is not valid JSON", filePath, lineNumber) + } + } + if err := file.Close(); err != nil { + return bytesRead, err + } + return bytesRead, nil +} diff --git a/internal/mountfs/native_preflight_audit.go b/internal/mountfs/native_preflight_audit.go new file mode 100644 index 0000000..3b8267c --- /dev/null +++ b/internal/mountfs/native_preflight_audit.go @@ -0,0 +1,72 @@ +package mountfs + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" +) + +type NativePreflightIssue struct { + Path string `json:"path"` + Message string `json:"message"` +} + +type NativePreflightAudit struct { + NativePreflightReport + Issues []NativePreflightIssue `json:"issues,omitempty"` +} + +func AuditNativeWriterRollouts(ctx context.Context, nativeRoot string) (NativePreflightAudit, error) { + root := filepath.Clean(nativeRoot) + activeRoot := filepath.Join(root, "sessions") + report := NativePreflightAudit{} + err := filepath.WalkDir(activeRoot, func(filePath string, entry os.DirEntry, walkErr error) error { + if err := ctx.Err(); err != nil { + return err + } + if walkErr != nil { + report.Issues = append(report.Issues, NativePreflightIssue{Path: filePath, Message: walkErr.Error()}) + if entry != nil && entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + report.Issues = append(report.Issues, NativePreflightIssue{Path: filePath, Message: "symlink is not allowed in the active native rollout tree"}) + return nil + } + if entry.IsDir() { + return nil + } + if !entry.Type().IsRegular() { + report.Issues = append(report.Issues, NativePreflightIssue{Path: filePath, Message: "non-regular file is not allowed in the active native rollout tree"}) + return nil + } + if strings.HasPrefix(entry.Name(), "._") || !strings.HasSuffix(entry.Name(), ".jsonl") { + return nil + } + info, err := entry.Info() + if err != nil { + report.Issues = append(report.Issues, NativePreflightIssue{Path: filePath, Message: err.Error()}) + return nil + } + report.Files++ + report.Bytes += info.Size() + validated, err := validateNativeJSONL(ctx, filePath) + report.ValidatedBytes += validated + report.ValidatedFiles++ + if err != nil { + report.Issues = append(report.Issues, NativePreflightIssue{Path: filePath, Message: err.Error()}) + } + return nil + }) + if os.IsNotExist(err) { + return NativePreflightAudit{}, nil + } + if err != nil { + return report, fmt.Errorf("audit active native rollouts: %w", err) + } + return report, nil +} diff --git a/internal/mountfs/native_preflight_cache.go b/internal/mountfs/native_preflight_cache.go new file mode 100644 index 0000000..5af29fa --- /dev/null +++ b/internal/mountfs/native_preflight_cache.go @@ -0,0 +1,276 @@ +package mountfs + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "unicode/utf8" +) + +const ( + nativePreflightCacheVersion = 1 + nativeFingerprintWindow = 64 << 10 +) + +type nativePreflightCache struct { + Version int `json:"version"` + Entries map[string]nativePreflightEntry `json:"entries"` +} + +type nativePreflightEntry struct { + Size int64 `json:"size"` + ModTimeNS int64 `json:"mod_time_ns"` + HeadSHA256 string `json:"head_sha256"` + TailSHA256 string `json:"tail_sha256"` +} + +func validateNativeWriterRolloutsCached(ctx context.Context, nativeRoot string) (NativePreflightReport, error) { + root := filepath.Clean(nativeRoot) + activeRoot := filepath.Join(root, "sessions") + cachePath := filepath.Join(root, ".codexfold-native-preflight-v1.json") + cache, cacheBytes, rebuilt := loadNativePreflightCache(cachePath) + next := nativePreflightCache{Version: nativePreflightCacheVersion, Entries: make(map[string]nativePreflightEntry)} + report := NativePreflightReport{CachePath: cachePath, CacheRebuilt: rebuilt} + + err := filepath.WalkDir(activeRoot, func(filePath string, directoryEntry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := ctx.Err(); err != nil { + return err + } + if directoryEntry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("native writer preflight rejects symlink %s", filePath) + } + if directoryEntry.IsDir() { + return nil + } + if !directoryEntry.Type().IsRegular() { + return fmt.Errorf("native writer preflight rejects non-regular file %s", filePath) + } + if strings.HasPrefix(directoryEntry.Name(), "._") || !strings.HasSuffix(directoryEntry.Name(), ".jsonl") { + return nil + } + info, err := directoryEntry.Info() + if err != nil { + return err + } + relative, err := filepath.Rel(root, filePath) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return fmt.Errorf("native writer preflight path escaped native root: %s", filePath) + } + key := filepath.ToSlash(relative) + current := nativePreflightEntry{Size: info.Size(), ModTimeNS: info.ModTime().UnixNano()} + report.Files++ + report.Bytes += info.Size() + + cached, exists := cache.Entries[key] + if exists && cached.Size == current.Size && cached.ModTimeNS == current.ModTimeNS { + next.Entries[key] = cached + report.CachedFiles++ + return nil + } + + var validatedBytes int64 + incremental := exists && current.Size > cached.Size && nativePrefixMatches(filePath, cached) + if incremental { + validatedBytes, err = validateNativeJSONLFrom(ctx, filePath, cached.Size) + } else { + validatedBytes, err = validateNativeJSONL(ctx, filePath) + } + if err != nil { + return err + } + after, err := os.Stat(filePath) + if err != nil { + return err + } + if after.Size() != current.Size || after.ModTime().UnixNano() != current.ModTimeNS { + return fmt.Errorf("native writer preflight target changed during validation: %s", filePath) + } + current.HeadSHA256, current.TailSHA256, err = nativeFingerprints(filePath, current.Size) + if err != nil { + return err + } + next.Entries[key] = current + report.ValidatedBytes += validatedBytes + if incremental { + report.IncrementalFiles++ + } else { + report.ValidatedFiles++ + } + return nil + }) + if errors.Is(err, os.ErrNotExist) { + err = nil + } + if err != nil { + return report, err + } + if err := writeNativePreflightCache(cachePath, next, cacheBytes); err != nil { + return report, err + } + return report, nil +} + +func loadNativePreflightCache(path string) (nativePreflightCache, []byte, bool) { + empty := nativePreflightCache{Version: nativePreflightCacheVersion, Entries: make(map[string]nativePreflightEntry)} + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return empty, nil, false + } + if err != nil { + return empty, nil, true + } + var cache nativePreflightCache + if json.Unmarshal(data, &cache) != nil || cache.Version != nativePreflightCacheVersion || cache.Entries == nil { + return empty, data, true + } + for key, entry := range cache.Entries { + if key == "" || filepath.IsAbs(key) || strings.HasPrefix(filepath.Clean(key), "..") || entry.Size < 0 || entry.ModTimeNS < 0 || + !validNativeFingerprint(entry.HeadSHA256) || !validNativeFingerprint(entry.TailSHA256) { + return empty, data, true + } + } + return cache, data, false +} + +func validNativeFingerprint(value string) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == sha256.Size +} + +func writeNativePreflightCache(path string, cache nativePreflightCache, previous []byte) error { + data, err := json.Marshal(cache) + if err != nil { + return err + } + if bytes.Equal(data, previous) { + return nil + } + root := filepath.Dir(path) + if err := os.MkdirAll(root, 0o700); err != nil { + return err + } + temporary, err := os.CreateTemp(root, ".native-preflight-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return err + } + return syncDirectory(root) +} + +func nativePrefixMatches(path string, cached nativePreflightEntry) bool { + head, tail, err := nativeFingerprints(path, cached.Size) + return err == nil && head == cached.HeadSHA256 && tail == cached.TailSHA256 +} + +func nativeFingerprints(path string, logicalSize int64) (string, string, error) { + if logicalSize < 0 { + return "", "", errors.New("negative native fingerprint size") + } + file, err := os.Open(path) + if err != nil { + return "", "", err + } + defer file.Close() + window := min(logicalSize, int64(nativeFingerprintWindow)) + head := make([]byte, int(window)) + if _, err := file.ReadAt(head, 0); err != nil && !errors.Is(err, io.EOF) { + return "", "", err + } + tail := make([]byte, int(window)) + if _, err := file.ReadAt(tail, logicalSize-window); err != nil && !errors.Is(err, io.EOF) { + return "", "", err + } + headDigest := sha256.Sum256(head) + tailDigest := sha256.Sum256(tail) + return hex.EncodeToString(headDigest[:]), hex.EncodeToString(tailDigest[:]), nil +} + +func validateNativeJSONLFrom(ctx context.Context, filePath string, offset int64) (int64, error) { + file, err := os.Open(filePath) + if err != nil { + return 0, err + } + if offset < 0 { + _ = file.Close() + return 0, errors.New("negative native preflight offset") + } + if offset > 0 { + boundary := []byte{0} + if _, err := file.ReadAt(boundary, offset-1); err != nil || boundary[0] != '\n' { + _ = file.Close() + return 0, fmt.Errorf("native writer preflight %s cached offset %d is not a JSONL boundary", filePath, offset) + } + } + if _, err := file.Seek(offset, io.SeekStart); err != nil { + _ = file.Close() + return 0, err + } + reader := bufio.NewReaderSize(file, 1<<20) + var lineNumber int64 + var bytesRead int64 + for { + if err := ctx.Err(); err != nil { + _ = file.Close() + return bytesRead, err + } + line, readErr := reader.ReadBytes('\n') + bytesRead += int64(len(line)) + if len(line) != 0 { + lineNumber++ + } + if errors.Is(readErr, io.EOF) { + if len(line) != 0 { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight %s line %d after byte %d is missing its final newline", filePath, lineNumber, offset) + } + break + } + if readErr != nil { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight read %s after byte %d: %w", filePath, offset, readErr) + } + if !utf8.Valid(line) { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight %s line %d after byte %d is not valid UTF-8", filePath, lineNumber, offset) + } + if !json.Valid(line) { + _ = file.Close() + return bytesRead, fmt.Errorf("native writer preflight %s line %d after byte %d is not valid JSON", filePath, lineNumber, offset) + } + } + if err := file.Close(); err != nil { + return bytesRead, err + } + return bytesRead, nil +} diff --git a/internal/mountfs/native_preflight_test.go b/internal/mountfs/native_preflight_test.go new file mode 100644 index 0000000..9a027c4 --- /dev/null +++ b/internal/mountfs/native_preflight_test.go @@ -0,0 +1,210 @@ +package mountfs + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestNativeWriterPreflightValidatesOnlyActiveRollouts(t *testing.T) { + root := t.TempDir() + active := filepath.Join(root, "sessions", "2026", "07", "16") + archived := filepath.Join(root, "archived_sessions") + if err := os.MkdirAll(active, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(archived, 0o700); err != nil { + t.Fatal(err) + } + valid := []byte("{\"record\":0}\n{\"record\":1}\n") + if err := os.WriteFile(filepath.Join(active, "rollout-valid.jsonl"), valid, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(active, "._rollout-valid.jsonl"), []byte("not-json"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(archived, "rollout-old.jsonl"), []byte("not-json"), 0o600); err != nil { + t.Fatal(err) + } + report, err := validateNativeWriterRollouts(context.Background(), root) + if err != nil { + t.Fatal(err) + } + if report.Files != 1 || report.Bytes != int64(len(valid)) { + t.Fatalf("preflight report = %#v", report) + } +} + +func TestNativeWriterPreflightRejectsInvalidJSON(t *testing.T) { + root, _ := nativePreflightFixture(t, []byte("{\"record\":0}\nnot-json\n")) + _, err := validateNativeWriterRollouts(context.Background(), root) + if err == nil || !strings.Contains(err.Error(), "line 2") || !strings.Contains(err.Error(), "not valid JSON") { + t.Fatalf("invalid JSON preflight error = %v", err) + } +} + +func TestNativeWriterPreflightRejectsInvalidUTF8(t *testing.T) { + root, _ := nativePreflightFixture(t, []byte{'{', '"', 'x', '"', ':', '"', 0xff, '"', '}', '\n'}) + _, err := validateNativeWriterRollouts(context.Background(), root) + if err == nil || !strings.Contains(err.Error(), "not valid UTF-8") { + t.Fatalf("invalid UTF-8 preflight error = %v", err) + } +} + +func TestNativeWriterPreflightRejectsMissingFinalNewline(t *testing.T) { + root, _ := nativePreflightFixture(t, []byte("{\"record\":0}")) + _, err := validateNativeWriterRollouts(context.Background(), root) + if err == nil || !strings.Contains(err.Error(), "missing its final newline") { + t.Fatalf("missing newline preflight error = %v", err) + } +} + +func TestValidateNativeRolloutRejectsNonRegularFiles(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "target.jsonl") + if err := os.WriteFile(target, []byte("{\"record\":0}\n"), 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "link.jsonl") + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + if _, err := ValidateNativeRollout(context.Background(), link); err == nil || !strings.Contains(err.Error(), "not a regular file") { + t.Fatalf("symlink validation error = %v", err) + } +} + +func TestNativeWriterPreflightRejectsSymlink(t *testing.T) { + root, path := nativePreflightFixture(t, []byte("{\"record\":0}\n")) + link := filepath.Join(filepath.Dir(path), "rollout-link.jsonl") + if err := os.Symlink(path, link); err != nil { + t.Fatal(err) + } + _, err := validateNativeWriterRollouts(context.Background(), root) + if err == nil || !strings.Contains(err.Error(), "rejects symlink") { + t.Fatalf("symlink preflight error = %v", err) + } +} + +func TestNativeWriterPreflightHonorsCancellation(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "sessions"), 0o700); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := validateNativeWriterRollouts(ctx, root) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled preflight error = %v", err) + } +} + +func TestNativeWriterPreflightCachesAndValidatesOnlyNewTail(t *testing.T) { + root, path := nativePreflightFixture(t, []byte("{\"record\":0}\n")) + first, err := validateNativeWriterRollouts(context.Background(), root) + if err != nil { + t.Fatal(err) + } + if first.ValidatedFiles != 1 || first.CachedFiles != 0 || first.IncrementalFiles != 0 { + t.Fatalf("first preflight = %#v", first) + } + second, err := validateNativeWriterRollouts(context.Background(), root) + if err != nil { + t.Fatal(err) + } + if second.CachedFiles != 1 || second.ValidatedFiles != 0 || second.ValidatedBytes != 0 { + t.Fatalf("cached preflight = %#v", second) + } + + tail := []byte("{\"record\":1}\n") + file, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.Write(tail); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + incremental, err := validateNativeWriterRollouts(context.Background(), root) + if err != nil { + t.Fatal(err) + } + if incremental.IncrementalFiles != 1 || incremental.ValidatedFiles != 0 || incremental.ValidatedBytes != int64(len(tail)) { + t.Fatalf("incremental preflight = %#v", incremental) + } +} + +func TestNativeWriterPreflightFullyRevalidatesSameSizeMutation(t *testing.T) { + root, path := nativePreflightFixture(t, []byte("{\"record\":0}\n")) + if _, err := validateNativeWriterRollouts(context.Background(), root); err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteAt([]byte("X"), 1); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + future := time.Now().Add(time.Second) + if err := os.Chtimes(path, future, future); err != nil { + t.Fatal(err) + } + if _, err := validateNativeWriterRollouts(context.Background(), root); err == nil || !strings.Contains(err.Error(), "not valid JSON") { + t.Fatalf("same-size mutation preflight error = %v", err) + } +} + +func TestNativeWriterPreflightRebuildsCorruptCache(t *testing.T) { + root, _ := nativePreflightFixture(t, []byte("{\"record\":0}\n")) + cachePath := filepath.Join(root, ".codexfold-native-preflight-v1.json") + if err := os.WriteFile(cachePath, []byte("broken-cache"), 0o600); err != nil { + t.Fatal(err) + } + report, err := validateNativeWriterRollouts(context.Background(), root) + if err != nil { + t.Fatal(err) + } + if !report.CacheRebuilt || report.ValidatedFiles != 1 { + t.Fatalf("rebuilt preflight = %#v", report) + } + if _, _, rebuilt := loadNativePreflightCache(cachePath); rebuilt { + t.Fatal("rewritten preflight cache is still invalid") + } +} + +func TestNativeWriterPreflightExternalRoot(t *testing.T) { + root := os.Getenv("CODEXFOLD_NATIVE_PREFLIGHT_ROOT") + if root == "" { + t.Skip("set CODEXFOLD_NATIVE_PREFLIGHT_ROOT to scan an external native root") + } + report, err := validateNativeWriterRollouts(context.Background(), root) + if err != nil { + t.Fatal(err) + } + t.Logf("validated active native rollouts: files=%d bytes=%d", report.Files, report.Bytes) +} + +func nativePreflightFixture(t *testing.T, data []byte) (string, string) { + t.Helper() + root := t.TempDir() + path := filepath.Join(root, "sessions", "2026", "07", "16", "rollout-fixture.jsonl") + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + return root, path +} diff --git a/internal/mountfs/native_sendfile_darwin.go b/internal/mountfs/native_sendfile_darwin.go new file mode 100644 index 0000000..afb7c84 --- /dev/null +++ b/internal/mountfs/native_sendfile_darwin.go @@ -0,0 +1,290 @@ +//go:build darwin + +package mountfs + +import ( + "crypto/rand" + "errors" + "fmt" + "io" + "net" + "os" + "unsafe" + + "github.com/samekind/codexfold/internal/fskitproto" + "golang.org/x/sys/unix" +) + +type nativeSharedReadObject struct { + file *os.File + unlink func() error +} + +type nativeSharedReadWindow struct { + file *os.File + mapping []byte + capacity int +} + +var createNativeSharedReadObject = newNativePOSIXSharedMemory +var createNativeSharedFileObject = newNativeRegularFile + +func nativeFileStreamingAvailable() bool { + return true +} + +func nativeSharedReadFDAvailable() bool { + return true +} + +func nativeSharedFileWindowAvailable() bool { + return true +} + +func sendNativeFile(connection net.Conn, file *os.File, offset int64, count int) (int, error) { + if file == nil || offset < 0 || count < 0 { + return 0, errors.New("invalid native sendfile range") + } + unixConnection, ok := connection.(*net.UnixConn) + if !ok { + return 0, errors.New("FSKit connection is not a Unix socket") + } + rawConnection, err := unixConnection.SyscallConn() + if err != nil { + return 0, err + } + written := 0 + var callbackErr error + rawErr := rawConnection.Write(func(outputFD uintptr) bool { + for written < count { + currentOffset := offset + int64(written) + n, sendErr := unix.Sendfile(int(outputFD), int(file.Fd()), ¤tOffset, count-written) + if n > 0 { + written += n + } + if sendErr == unix.EINTR { + continue + } + if sendErr == unix.EAGAIN || sendErr == unix.EWOULDBLOCK { + return false + } + if sendErr != nil { + callbackErr = sendErr + return true + } + if n == 0 { + callbackErr = io.ErrUnexpectedEOF + return true + } + } + return true + }) + if rawErr != nil { + return written, rawErr + } + if callbackErr != nil { + return written, callbackErr + } + return written, nil +} + +func sendNativeReadFD(connection net.Conn, file *os.File) error { + return sendReadFD(connection, file, fskitproto.NativeReadFDMarker) +} + +func sendSharedReadFD(connection net.Conn, file *os.File) error { + return sendReadFD(connection, file, fskitproto.SharedReadFDMarker) +} + +func sendSharedWindowFD(connection net.Conn, file *os.File) error { + return sendReadFD(connection, file, fskitproto.SharedWindowFDMarker) +} + +func sendSharedFileWindowFD(connection net.Conn, file *os.File) error { + return sendReadFD(connection, file, fskitproto.SharedFileWindowFDMarker) +} + +func sendReadFD(connection net.Conn, file *os.File, markerByte byte) error { + if file == nil { + return errors.New("read descriptor is nil") + } + unixConnection, ok := connection.(*net.UnixConn) + if !ok { + return errors.New("FSKit connection is not a Unix socket") + } + rawConnection, err := unixConnection.SyscallConn() + if err != nil { + return err + } + oob := unix.UnixRights(int(file.Fd())) + marker := []byte{markerByte} + var callbackErr error + rawErr := rawConnection.Write(func(outputFD uintptr) bool { + for { + n, sendErr := unix.SendmsgN(int(outputFD), marker, oob, nil, 0) + if sendErr == unix.EINTR { + continue + } + if sendErr == unix.EAGAIN || sendErr == unix.EWOULDBLOCK { + return false + } + if sendErr != nil { + callbackErr = sendErr + return true + } + if n != len(marker) { + callbackErr = io.ErrShortWrite + } + return true + } + }) + if rawErr != nil { + return rawErr + } + return callbackErr +} + +func prepareNativeSharedReadFD(length int, populate func([]byte) (int, error)) (_ *os.File, populated int, resultErr error) { + if length <= 0 || populate == nil { + return nil, 0, errors.New("invalid shared read mapping request") + } + window, err := newNativeSharedReadWindow(length) + if err != nil { + return nil, 0, err + } + populated, populateErr := populate(window.mapping[:length]) + if populateErr != nil { + return nil, populated, errors.Join(fmt.Errorf("populate shared read file: %w", populateErr), window.Close()) + } + if populated < 0 || populated > length { + return nil, populated, errors.Join( + fmt.Errorf("shared read populated %d bytes into %d-byte mapping", populated, length), + window.Close(), + ) + } + if err := unix.Munmap(window.mapping); err != nil { + return nil, populated, errors.Join(fmt.Errorf("unmap shared read file: %w", err), window.Close()) + } + window.mapping = nil + file := window.file + window.file = nil + return file, populated, nil +} + +func newNativeSharedReadWindow(length int) (*nativeSharedReadWindow, error) { + return newNativeSharedReadWindowWith(length, createNativeSharedReadObject) +} + +func newNativeSharedFileWindow(length int) (*nativeSharedReadWindow, error) { + return newNativeSharedReadWindowWith(length, createNativeSharedFileObject) +} + +func newNativeSharedReadWindowWith(length int, create func() (nativeSharedReadObject, error)) (*nativeSharedReadWindow, error) { + if length <= 0 { + return nil, errors.New("invalid shared read window size") + } + if create == nil { + return nil, errors.New("shared read object creator is nil") + } + object, err := create() + if err != nil { + return nil, fmt.Errorf("create shared read object: %w", err) + } + if object.file == nil || object.unlink == nil { + if object.file != nil { + _ = object.file.Close() + } + return nil, errors.New("shared read object is incomplete") + } + file := object.file + if err := object.unlink(); err != nil { + return nil, errors.Join(fmt.Errorf("unlink shared read object: %w", err), file.Close(), object.unlink()) + } + mappedLength := nativeSharedReadMappedLength(length) + if err := file.Truncate(int64(mappedLength)); err != nil { + return nil, errors.Join(fmt.Errorf("size shared read file: %w", err), file.Close()) + } + mapping, err := unix.Mmap(int(file.Fd()), 0, mappedLength, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED) + if err != nil { + return nil, errors.Join(fmt.Errorf("map shared read file: %w", err), file.Close()) + } + return &nativeSharedReadWindow{file: file, mapping: mapping, capacity: length}, nil +} + +func newNativeRegularFile() (nativeSharedReadObject, error) { + file, err := os.CreateTemp("", ".codexfold-shared-window-*") + if err != nil { + return nativeSharedReadObject{}, err + } + if err := file.Chmod(0o600); err != nil { + return nativeSharedReadObject{}, errors.Join(err, file.Close(), os.Remove(file.Name())) + } + path := file.Name() + return nativeSharedReadObject{ + file: file, + unlink: func() error { + return os.Remove(path) + }, + }, nil +} + +func (w *nativeSharedReadWindow) Close() error { + if w == nil { + return nil + } + var result error + if w.mapping != nil { + result = errors.Join(result, unix.Munmap(w.mapping)) + w.mapping = nil + } + if w.file != nil { + result = errors.Join(result, w.file.Close()) + w.file = nil + } + return result +} + +func nativeSharedReadMappedLength(length int) int { + pageBytes := os.Getpagesize() + return (length + pageBytes - 1) / pageBytes * pageBytes +} + +func newNativePOSIXSharedMemory() (nativeSharedReadObject, error) { + var entropy [10]byte + if _, err := rand.Read(entropy[:]); err != nil { + return nativeSharedReadObject{}, err + } + name := fmt.Sprintf("/cfs-%x", entropy[:]) + pointer, err := unix.BytePtrFromString(name) + if err != nil { + return nativeSharedReadObject{}, err + } + descriptor, _, errno := unix.Syscall( + unix.SYS_SHM_OPEN, + uintptr(unsafe.Pointer(pointer)), + uintptr(unix.O_RDWR|unix.O_CREAT|unix.O_EXCL), + uintptr(0o600), + ) + if errno != 0 { + return nativeSharedReadObject{}, errno + } + file := os.NewFile(descriptor, name) + if file == nil { + _ = unix.Close(int(descriptor)) + return nativeSharedReadObject{}, errors.New("construct POSIX shared memory file") + } + return nativeSharedReadObject{ + file: file, + unlink: func() error { + namePointer, err := unix.BytePtrFromString(name) + if err != nil { + return err + } + _, _, errno := unix.Syscall(unix.SYS_SHM_UNLINK, uintptr(unsafe.Pointer(namePointer)), 0, 0) + if errno != 0 { + return errno + } + return nil + }, + }, nil +} diff --git a/internal/mountfs/native_sendfile_other.go b/internal/mountfs/native_sendfile_other.go new file mode 100644 index 0000000..7e029cb --- /dev/null +++ b/internal/mountfs/native_sendfile_other.go @@ -0,0 +1,61 @@ +//go:build !darwin + +package mountfs + +import ( + "errors" + "net" + "os" +) + +type nativeSharedReadWindow struct { + file *os.File + mapping []byte + capacity int +} + +func nativeFileStreamingAvailable() bool { + return false +} + +func nativeSharedReadFDAvailable() bool { + return false +} + +func nativeSharedFileWindowAvailable() bool { + return false +} + +func sendNativeFile(_ net.Conn, _ *os.File, _ int64, _ int) (int, error) { + return 0, errors.New("native sendfile is unavailable on this platform") +} + +func sendNativeReadFD(_ net.Conn, _ *os.File) error { + return errors.New("native descriptor transfer is unavailable on this platform") +} + +func sendSharedReadFD(_ net.Conn, _ *os.File) error { + return errors.New("shared descriptor transfer is unavailable on this platform") +} + +func sendSharedWindowFD(_ net.Conn, _ *os.File) error { + return errors.New("shared window descriptor transfer is unavailable on this platform") +} + +func sendSharedFileWindowFD(_ net.Conn, _ *os.File) error { + return errors.New("shared file window descriptor transfer is unavailable on this platform") +} + +func prepareNativeSharedReadFD(_ int, _ func([]byte) (int, error)) (*os.File, int, error) { + return nil, 0, errors.New("shared descriptor transfer is unavailable on this platform") +} + +func newNativeSharedReadWindow(_ int) (*nativeSharedReadWindow, error) { + return nil, errors.New("shared read windows are unavailable on this platform") +} + +func newNativeSharedFileWindow(_ int) (*nativeSharedReadWindow, error) { + return nil, errors.New("shared file windows are unavailable on this platform") +} + +func (w *nativeSharedReadWindow) Close() error { return nil } diff --git a/internal/mountfs/testdata/codex-real-resume-write.trace b/internal/mountfs/testdata/codex-real-resume-write.trace new file mode 100644 index 0000000..b85940a --- /dev/null +++ b/internal/mountfs/testdata/codex-real-resume-write.trace @@ -0,0 +1,27 @@ +# Sanitized Codex resume write sequence captured on macOS on 2026-07-16. +# Fields are operation, absolute offset, and byte count. No path or content is retained. +open 2 +write 51836 553 +fsync +write 52389 233 +fsync +write 52622 705 +fsync +write 53327 402 +fsync +write 53729 311 +fsync +write 54040 141 +fsync +write 54181 1435 +fsync +write 55616 261 +fsync +write 55877 445 +fsync +write 56322 583 +fsync +write 56905 350 +fsync +flush +release diff --git a/internal/mountfs/xattr_darwin.go b/internal/mountfs/xattr_darwin.go new file mode 100644 index 0000000..61e9e7a --- /dev/null +++ b/internal/mountfs/xattr_darwin.go @@ -0,0 +1,67 @@ +//go:build darwin + +package mountfs + +import ( + "bytes" + "syscall" + + "github.com/samekind/codexfold/internal/fskitproto" + "golang.org/x/sys/unix" +) + +func platformSetXattr(path string, attribute string, value []byte, policy fskitproto.XattrPolicy) error { + flags := 0 + switch policy { + case fskitproto.XattrAlwaysSet: + case fskitproto.XattrMustCreate: + flags = unix.XATTR_CREATE + case fskitproto.XattrMustReplace: + flags = unix.XATTR_REPLACE + default: + return syscall.EINVAL + } + return unix.Setxattr(path, attribute, value, flags) +} + +func platformGetXattr(path string, attribute string) ([]byte, error) { + size, err := unix.Getxattr(path, attribute, nil) + if err != nil { + return nil, err + } + value := make([]byte, size) + n, err := unix.Getxattr(path, attribute, value) + if err != nil { + return nil, err + } + return value[:n], nil +} + +func platformListXattrs(path string) ([]string, error) { + size, err := unix.Listxattr(path, nil) + if err != nil { + return nil, err + } + if size == 0 { + return []string{}, nil + } + buffer := make([]byte, size) + n, err := unix.Listxattr(path, buffer) + if err != nil { + return nil, err + } + parts := bytes.Split(bytes.TrimRight(buffer[:n], "\x00"), []byte{0}) + result := make([]string, 0, len(parts)) + for _, part := range parts { + if len(part) != 0 { + result = append(result, string(part)) + } + } + return result, nil +} + +func platformRemoveXattr(path string, attribute string) error { + return unix.Removexattr(path, attribute) +} + +func xattrMissingErrno() syscall.Errno { return syscall.ENOATTR } diff --git a/internal/mountfs/xattr_linux.go b/internal/mountfs/xattr_linux.go new file mode 100644 index 0000000..8d3e882 --- /dev/null +++ b/internal/mountfs/xattr_linux.go @@ -0,0 +1,67 @@ +//go:build linux + +package mountfs + +import ( + "bytes" + "syscall" + + "github.com/samekind/codexfold/internal/fskitproto" + "golang.org/x/sys/unix" +) + +func platformSetXattr(path string, attribute string, value []byte, policy fskitproto.XattrPolicy) error { + flags := 0 + switch policy { + case fskitproto.XattrAlwaysSet: + case fskitproto.XattrMustCreate: + flags = unix.XATTR_CREATE + case fskitproto.XattrMustReplace: + flags = unix.XATTR_REPLACE + default: + return syscall.EINVAL + } + return unix.Setxattr(path, attribute, value, flags) +} + +func platformGetXattr(path string, attribute string) ([]byte, error) { + size, err := unix.Getxattr(path, attribute, nil) + if err != nil { + return nil, err + } + value := make([]byte, size) + n, err := unix.Getxattr(path, attribute, value) + if err != nil { + return nil, err + } + return value[:n], nil +} + +func platformListXattrs(path string) ([]string, error) { + size, err := unix.Listxattr(path, nil) + if err != nil { + return nil, err + } + if size == 0 { + return []string{}, nil + } + buffer := make([]byte, size) + n, err := unix.Listxattr(path, buffer) + if err != nil { + return nil, err + } + parts := bytes.Split(bytes.TrimRight(buffer[:n], "\x00"), []byte{0}) + result := make([]string, 0, len(parts)) + for _, part := range parts { + if len(part) != 0 { + result = append(result, string(part)) + } + } + return result, nil +} + +func platformRemoveXattr(path string, attribute string) error { + return unix.Removexattr(path, attribute) +} + +func xattrMissingErrno() syscall.Errno { return syscall.ENODATA } diff --git a/internal/mountfs/xattr_other.go b/internal/mountfs/xattr_other.go new file mode 100644 index 0000000..94c5e99 --- /dev/null +++ b/internal/mountfs/xattr_other.go @@ -0,0 +1,15 @@ +//go:build !darwin && !linux + +package mountfs + +import ( + "syscall" + + "github.com/samekind/codexfold/internal/fskitproto" +) + +func platformSetXattr(string, string, []byte, fskitproto.XattrPolicy) error { return syscall.ENOTSUP } +func platformGetXattr(string, string) ([]byte, error) { return nil, syscall.ENOTSUP } +func platformListXattrs(string) ([]string, error) { return nil, syscall.ENOTSUP } +func platformRemoveXattr(string, string) error { return syscall.ENOTSUP } +func xattrMissingErrno() syscall.Errno { return syscall.ENOENT } diff --git a/internal/mountid/identity.go b/internal/mountid/identity.go new file mode 100644 index 0000000..24f6612 --- /dev/null +++ b/internal/mountid/identity.go @@ -0,0 +1,62 @@ +package mountid + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "strings" + + "github.com/samekind/codexfold/internal/buildid" +) + +const ( + Path = ".codexfold-health" + prefixV1 = "codexfold-v1:" + prefixV2 = "codexfold-v2:" +) + +type Identity struct { + Version int + Nonce string + BuildSHA256 string +} + +func New(buildSHA256 string) (string, error) { + if !buildid.ValidSHA256(buildSHA256) { + return "", errors.New("mount identity build SHA-256 is invalid") + } + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return "", err + } + return prefixV2 + hex.EncodeToString(random[:]) + ":" + buildSHA256, nil +} + +func Validate(value []byte) error { + _, err := Parse(value) + return err +} + +func Parse(value []byte) (Identity, error) { + text := string(value) + if strings.HasPrefix(text, prefixV1) { + nonce := strings.TrimPrefix(text, prefixV1) + if !validNonce(nonce) { + return Identity{}, errors.New("mount identity payload is invalid") + } + return Identity{Version: 1, Nonce: nonce}, nil + } + if !strings.HasPrefix(text, prefixV2) { + return Identity{}, errors.New("mount identity prefix is invalid") + } + parts := strings.Split(strings.TrimPrefix(text, prefixV2), ":") + if len(parts) != 2 || !validNonce(parts[0]) || !buildid.ValidSHA256(parts[1]) { + return Identity{}, errors.New("mount identity payload is invalid") + } + return Identity{Version: 2, Nonce: parts[0], BuildSHA256: parts[1]}, nil +} + +func validNonce(value string) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == 16 +} diff --git a/internal/mountid/identity_test.go b/internal/mountid/identity_test.go new file mode 100644 index 0000000..9547f53 --- /dev/null +++ b/internal/mountid/identity_test.go @@ -0,0 +1,40 @@ +package mountid + +import ( + "strings" + "testing" +) + +func TestVersionTwoIdentityCarriesBuildSHA256(t *testing.T) { + build := strings.Repeat("a", 64) + value, err := New(build) + if err != nil { + t.Fatal(err) + } + identity, err := Parse([]byte(value)) + if err != nil { + t.Fatal(err) + } + if identity.Version != 2 || identity.BuildSHA256 != build || len(identity.Nonce) != 32 { + t.Fatalf("identity = %#v", identity) + } +} + +func TestLegacyIdentityRemainsProbeCompatibleWithoutBuild(t *testing.T) { + identity, err := Parse([]byte("codexfold-v1:" + strings.Repeat("a", 32))) + if err != nil { + t.Fatal(err) + } + if identity.Version != 1 || identity.BuildSHA256 != "" { + t.Fatalf("legacy identity = %#v", identity) + } +} + +func TestIdentityRejectsInvalidBuild(t *testing.T) { + if _, err := New("invalid"); err == nil { + t.Fatal("invalid build SHA-256 was accepted") + } + if _, err := Parse([]byte("codexfold-v2:" + strings.Repeat("a", 32) + ":invalid")); err == nil { + t.Fatal("invalid v2 payload was accepted") + } +} diff --git a/internal/pack/build.go b/internal/pack/build.go new file mode 100644 index 0000000..fd6b71b --- /dev/null +++ b/internal/pack/build.go @@ -0,0 +1,351 @@ +package pack + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/klauspost/compress/zstd" + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/storage" +) + +type BuildOptions struct { + BlockBytes int64 + PackBytes int64 + Budget storage.Checker + BeforePublish func() error +} + +type BuildResult struct { + Generation string `json:"generation"` + ObjectCount int `json:"object_count"` + BlockCount int `json:"block_count"` + PackCount int `json:"pack_count"` + RawBytes int64 `json:"raw_bytes"` + StoredBytes int64 `json:"stored_bytes"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` +} + +type packWriter struct { + directory string + limit int64 + sequence int + file *os.File + name string + offset int64 +} + +func Build(ctx context.Context, storeDir string, options BuildOptions) (BuildResult, error) { + if storeDir == "" { + return BuildResult{}, errors.New("pack store directory is required") + } + if options.BlockBytes <= 0 { + options.BlockBytes = defaultBlockBytes + } + if options.PackBytes <= 0 { + options.PackBytes = defaultPackBytes + } + if options.BlockBytes > int64(int(^uint(0)>>1)) { + return BuildResult{}, errors.New("pack block size exceeds platform integer size") + } + refs, err := referencedObjects(storeDir) + if err != nil { + return BuildResult{}, err + } + budget := options.Budget + if budget == nil { + guard, err := storage.DefaultGuard(storeDir) + if err != nil { + return BuildResult{}, err + } + budget = guard + } + estimatedBytes, err := estimatedGenerationBytes(refs) + if err != nil { + return BuildResult{}, err + } + storageAssessment, err := budget.Check(ctx, storage.Projection{ + Operation: "pack-build", + AdditionalPersistentBytes: estimatedBytes, + TemporaryBytes: estimatedBytes, + TemporaryPersistentOverlapBytes: estimatedBytes, + }) + if err != nil { + return BuildResult{}, err + } + packsDir := filepath.Join(storeDir, "packs") + if err := os.MkdirAll(packsDir, 0o755); err != nil { + return BuildResult{}, fmt.Errorf("create packs directory: %w", err) + } + temporaryDir, err := os.MkdirTemp(packsDir, ".generation-") + if err != nil { + return BuildResult{}, fmt.Errorf("create temporary pack generation: %w", err) + } + defer func() { _ = os.RemoveAll(temporaryDir) }() + generation := "gen-" + strings.TrimPrefix(filepath.Base(temporaryDir), ".generation-") + index := Index{Version: IndexVersion, Kind: IndexKind, Generation: generation, CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), BlockBytes: options.BlockBytes, Objects: make([]Object, 0, len(refs))} + result := BuildResult{Generation: generation, ObjectCount: len(refs)} + writer := &packWriter{directory: temporaryDir, limit: options.PackBytes} + encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault)) + if err != nil { + return BuildResult{}, fmt.Errorf("create pack encoder: %w", err) + } + defer encoder.Close() + store := fold.NewObjectStore(storeDir) + buffer := make([]byte, int(options.BlockBytes)) + for _, ref := range refs { + if err := ctx.Err(); err != nil { + return BuildResult{}, err + } + stream, err := store.OpenStream(ref) + if err != nil { + return BuildResult{}, err + } + object := Object{SHA256: ref.SHA256, RawBytes: ref.RawBytes} + objectHash := sha256.New() + var rawOffset int64 + for { + n, readErr := io.ReadFull(stream, buffer) + if n > 0 { + raw := buffer[:n] + _, _ = objectHash.Write(raw) + compressed := encoder.EncodeAll(raw, nil) + stored := compressed + encoding := EncodingZstd + if preferRawBlock(raw, compressed) { + stored = raw + encoding = EncodingRaw + } + packName, packOffset, writeErr := writer.write(stored) + if writeErr != nil { + _ = stream.Close() + return BuildResult{}, writeErr + } + blockHash := sha256.Sum256(raw) + object.Blocks = append(object.Blocks, Block{Pack: packName, PackOffset: packOffset, StoredBytes: int64(len(stored)), RawOffset: rawOffset, RawBytes: int64(n), SHA256: hex.EncodeToString(blockHash[:]), Encoding: encoding}) + rawOffset += int64(n) + result.BlockCount++ + result.RawBytes += int64(n) + result.StoredBytes += int64(len(stored)) + } + if errors.Is(readErr, io.EOF) || errors.Is(readErr, io.ErrUnexpectedEOF) { + break + } + if readErr != nil { + _ = stream.Close() + return BuildResult{}, fmt.Errorf("read loose object %s: %w", ref.SHA256, readErr) + } + } + if err := stream.Close(); err != nil { + return BuildResult{}, err + } + if rawOffset != ref.RawBytes || hex.EncodeToString(objectHash.Sum(nil)) != ref.SHA256 { + return BuildResult{}, fmt.Errorf("loose object %s failed stream verification", ref.SHA256) + } + index.Objects = append(index.Objects, object) + } + if err := writer.close(); err != nil { + return BuildResult{}, err + } + result.PackCount = writer.sequence + if err := writeIndex(temporaryDir, index); err != nil { + return BuildResult{}, err + } + if err := verifyGeneration(ctx, temporaryDir, index); err != nil { + return BuildResult{}, fmt.Errorf("verify candidate pack generation: %w", err) + } + finalDir := filepath.Join(packsDir, generation) + if err := os.Rename(temporaryDir, finalDir); err != nil { + return BuildResult{}, fmt.Errorf("publish pack generation directory: %w", err) + } + if err := syncDirectory(packsDir); err != nil { + return BuildResult{}, err + } + if options.BeforePublish != nil { + if err := options.BeforePublish(); err != nil { + return BuildResult{}, err + } + } + if err := publishCurrent(packsDir, generation); err != nil { + return BuildResult{}, err + } + result.Storage = storage.CompleteAccounting(ctx, storageAssessment, storeDir) + return result, nil +} + +func preferRawBlock(raw []byte, compressed []byte) bool { + if len(compressed) >= len(raw) { + return true + } + return len(raw) <= 16<<10 && len(compressed)*2 >= len(raw) +} + +func estimatedGenerationBytes(refs []fold.ObjectRef) (int64, error) { + const fixedOverhead = int64(1 << 20) + var rawBytes int64 + for _, ref := range refs { + if ref.RawBytes < 0 || rawBytes > math.MaxInt64-ref.RawBytes { + return 0, errors.New("pack generation byte estimate overflow") + } + rawBytes += ref.RawBytes + } + compressionOverhead := rawBytes/16 + fixedOverhead + if rawBytes > math.MaxInt64-compressionOverhead { + return 0, errors.New("pack generation byte estimate overflow") + } + return rawBytes + compressionOverhead, nil +} + +func referencedObjects(storeDir string) ([]fold.ObjectRef, error) { + refs := make(map[string]fold.ObjectRef) + err := filepath.WalkDir(filepath.Join(storeDir, "manifests"), func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + return nil + } + manifest, err := fold.LoadManifestPath(path) + if err != nil { + return err + } + for _, part := range manifest.Parts { + if existing, ok := refs[part.Object.SHA256]; ok && existing.RawBytes != part.Object.RawBytes { + return fmt.Errorf("object %s has conflicting raw lengths", part.Object.SHA256) + } + refs[part.Object.SHA256] = part.Object + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("read manifests for pack build: %w", err) + } + digests := make([]string, 0, len(refs)) + for digest := range refs { + digests = append(digests, digest) + } + sort.Strings(digests) + result := make([]fold.ObjectRef, 0, len(digests)) + for _, digest := range digests { + result = append(result, refs[digest]) + } + return result, nil +} + +func (w *packWriter) write(data []byte) (string, int64, error) { + if w.file == nil || (w.offset > 0 && w.offset+int64(len(data)) > w.limit) { + if err := w.rotate(); err != nil { + return "", 0, err + } + } + offset := w.offset + if _, err := w.file.Write(data); err != nil { + return "", 0, fmt.Errorf("write pack file: %w", err) + } + w.offset += int64(len(data)) + return w.name, offset, nil +} + +func (w *packWriter) rotate() error { + if err := w.closeCurrent(); err != nil { + return err + } + w.sequence++ + w.name = fmt.Sprintf("pack-%06d.pack", w.sequence) + file, err := os.OpenFile(filepath.Join(w.directory, w.name), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("create pack file: %w", err) + } + w.file = file + w.offset = 0 + return nil +} + +func (w *packWriter) closeCurrent() error { + if w.file == nil { + return nil + } + if err := w.file.Sync(); err != nil { + _ = w.file.Close() + return fmt.Errorf("sync pack file: %w", err) + } + if err := w.file.Close(); err != nil { + return fmt.Errorf("close pack file: %w", err) + } + w.file = nil + return nil +} + +func (w *packWriter) close() error { return w.closeCurrent() } + +func writeIndex(directory string, index Index) error { + data, err := json.Marshal(index) + if err != nil { + return fmt.Errorf("encode pack index: %w", err) + } + data = append(data, '\n') + path := filepath.Join(directory, "index.json") + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("create pack index: %w", err) + } + if _, err := file.Write(data); err != nil { + _ = file.Close() + return fmt.Errorf("write pack index: %w", err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + return fmt.Errorf("sync pack index: %w", err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("close pack index: %w", err) + } + return syncDirectory(directory) +} + +func publishCurrent(packsDir string, generation string) error { + temporary, err := os.CreateTemp(packsDir, ".CURRENT-") + if err != nil { + return fmt.Errorf("create temporary CURRENT: %w", err) + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if _, err := temporary.WriteString(generation + "\n"); err != nil { + _ = temporary.Close() + return fmt.Errorf("write CURRENT: %w", err) + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return fmt.Errorf("sync CURRENT: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close CURRENT: %w", err) + } + if err := replaceFile(temporaryPath, filepath.Join(packsDir, "CURRENT")); err != nil { + return fmt.Errorf("publish CURRENT: %w", err) + } + return syncDirectory(packsDir) +} + +func syncDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return fmt.Errorf("open directory for sync %s: %w", path, err) + } + defer directory.Close() + if err := directory.Sync(); err != nil { + return fmt.Errorf("sync directory %s: %w", path, err) + } + return nil +} diff --git a/internal/pack/cache.go b/internal/pack/cache.go new file mode 100644 index 0000000..fe2810e --- /dev/null +++ b/internal/pack/cache.go @@ -0,0 +1,64 @@ +package pack + +import ( + "container/list" + "sync" +) + +type cacheEntry struct { + key string + data []byte +} + +type blockCache struct { + mu sync.Mutex + budget int64 + used int64 + items map[string]*list.Element + lru list.List +} + +func newBlockCache(budget int64) *blockCache { + if budget < 0 { + budget = 0 + } + return &blockCache{budget: budget, items: make(map[string]*list.Element)} +} + +func (c *blockCache) get(key string) ([]byte, bool) { + c.mu.Lock() + defer c.mu.Unlock() + element, ok := c.items[key] + if !ok { + return nil, false + } + c.lru.MoveToFront(element) + return element.Value.(cacheEntry).data, true +} + +func (c *blockCache) put(key string, data []byte) { + if int64(len(data)) > c.budget || c.budget == 0 { + return + } + if cap(data) != len(data) { + owned := make([]byte, len(data)) + copy(owned, data) + data = owned + } + c.mu.Lock() + defer c.mu.Unlock() + if existing, ok := c.items[key]; ok { + c.lru.MoveToFront(existing) + return + } + element := c.lru.PushFront(cacheEntry{key: key, data: data}) + c.items[key] = element + c.used += int64(len(data)) + for c.used > c.budget { + oldest := c.lru.Back() + entry := oldest.Value.(cacheEntry) + delete(c.items, entry.key) + c.used -= int64(len(entry.data)) + c.lru.Remove(oldest) + } +} diff --git a/internal/pack/doctor.go b/internal/pack/doctor.go new file mode 100644 index 0000000..b173a33 --- /dev/null +++ b/internal/pack/doctor.go @@ -0,0 +1,94 @@ +package pack + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + + "github.com/samekind/codexfold/internal/fold" +) + +type DoctorIssue struct { + ObjectSHA256 string `json:"object_sha256,omitempty"` + Message string `json:"message"` +} + +type DoctorResult struct { + Generation string `json:"generation,omitempty"` + ObjectCount int `json:"object_count"` + VerifiedCount int `json:"verified_count"` + IssueCount int `json:"issue_count"` + Issues []DoctorIssue `json:"issues,omitempty"` +} + +func Doctor(ctx context.Context, storeDir string) (DoctorResult, error) { + resolver, err := Open(storeDir, OpenOptions{CacheBytes: -1}) + if err != nil { + return DoctorResult{IssueCount: 1, Issues: []DoctorIssue{{Message: err.Error()}}}, nil + } + defer resolver.Close() + result := DoctorResult{Generation: resolver.index.Generation, ObjectCount: len(resolver.index.Objects)} + for _, object := range resolver.index.Objects { + hasher := sha256.New() + buffer := make([]byte, 128<<10) + var offset int64 + failed := false + for offset < object.RawBytes { + n, readErr := resolver.ReadAt(ctx, fold.ObjectRef{SHA256: object.SHA256, RawBytes: object.RawBytes}, buffer, offset) + if n > 0 { + _, _ = hasher.Write(buffer[:n]) + offset += int64(n) + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + result.Issues = append(result.Issues, DoctorIssue{ObjectSHA256: object.SHA256, Message: readErr.Error()}) + failed = true + break + } + if n == 0 { + break + } + } + if !failed && (offset != object.RawBytes || hex.EncodeToString(hasher.Sum(nil)) != object.SHA256) { + result.Issues = append(result.Issues, DoctorIssue{ObjectSHA256: object.SHA256, Message: fmt.Sprintf("object reconstruction mismatch at %d of %d bytes", offset, object.RawBytes)}) + failed = true + } + if !failed { + result.VerifiedCount++ + } + } + result.IssueCount = len(result.Issues) + return result, nil +} + +func verifyGeneration(ctx context.Context, directory string, index Index) error { + resolver, err := openGeneration(directory, 0, false) + if err != nil { + return err + } + defer resolver.Close() + for _, object := range index.Objects { + hasher := sha256.New() + buffer := make([]byte, 128<<10) + var offset int64 + for offset < object.RawBytes { + n, readErr := resolver.ReadAt(ctx, fold.ObjectRef{SHA256: object.SHA256, RawBytes: object.RawBytes}, buffer, offset) + if n > 0 { + _, _ = hasher.Write(buffer[:n]) + offset += int64(n) + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + return readErr + } + if n == 0 { + break + } + } + if offset != object.RawBytes || hex.EncodeToString(hasher.Sum(nil)) != object.SHA256 { + return fmt.Errorf("packed object %s verification failed", object.SHA256) + } + } + return nil +} diff --git a/internal/pack/format.go b/internal/pack/format.go new file mode 100644 index 0000000..aec8ecc --- /dev/null +++ b/internal/pack/format.go @@ -0,0 +1,98 @@ +package pack + +import ( + "fmt" + "path/filepath" +) + +const ( + IndexVersion = 2 + IndexKind = "pack-v2" + legacyIndexVersion = 1 + legacyIndexKind = "pack-v1" + EncodingZstd = "zstd" + EncodingRaw = "raw" + defaultBlockBytes = int64(256 << 10) + defaultPackBytes = int64(512 << 20) + defaultCacheBytes = int64(64 << 20) +) + +type Index struct { + Version int `json:"version"` + Kind string `json:"kind"` + Generation string `json:"generation"` + CreatedAt string `json:"created_at"` + BlockBytes int64 `json:"block_bytes"` + Objects []Object `json:"objects"` +} + +type Object struct { + SHA256 string `json:"sha256"` + RawBytes int64 `json:"raw_bytes"` + Blocks []Block `json:"blocks"` +} + +type Block struct { + Pack string `json:"pack"` + PackOffset int64 `json:"pack_offset"` + StoredBytes int64 `json:"stored_bytes"` + RawOffset int64 `json:"raw_offset"` + RawBytes int64 `json:"raw_bytes"` + SHA256 string `json:"sha256"` + Encoding string `json:"encoding,omitempty"` +} + +func validateIndex(index Index) error { + current := index.Version == IndexVersion && index.Kind == IndexKind + legacy := index.Version == legacyIndexVersion && index.Kind == legacyIndexKind + if !current && !legacy { + return fmt.Errorf("unsupported pack index version=%d kind=%q", index.Version, index.Kind) + } + if !safeGeneration(index.Generation) || index.BlockBytes <= 0 { + return fmt.Errorf("invalid pack index generation or block size") + } + seen := make(map[string]struct{}, len(index.Objects)) + for objectIndex, object := range index.Objects { + if len(object.SHA256) != 64 || object.RawBytes < 0 { + return fmt.Errorf("invalid pack object %d", objectIndex) + } + if _, ok := seen[object.SHA256]; ok { + return fmt.Errorf("duplicate pack object %s", object.SHA256) + } + seen[object.SHA256] = struct{}{} + var expectedOffset int64 + for blockIndex, block := range object.Blocks { + if filepath.Base(block.Pack) != block.Pack || block.Pack == "." || block.Pack == ".." || block.PackOffset < 0 || block.StoredBytes <= 0 || block.StoredBytes > maxInt64() || block.RawBytes <= 0 || block.RawOffset != expectedOffset || len(block.SHA256) != 64 { + return fmt.Errorf("invalid block %d for object %s", blockIndex, object.SHA256) + } + if block.Encoding != EncodingZstd && (!current || block.Encoding != EncodingRaw) { + return fmt.Errorf("invalid block encoding %q for object %s", block.Encoding, object.SHA256) + } + expectedOffset += block.RawBytes + } + if expectedOffset != object.RawBytes { + return fmt.Errorf("object %s block bytes %d, want %d", object.SHA256, expectedOffset, object.RawBytes) + } + } + return nil +} + +func normalizeLegacyIndex(index *Index) { + if index.Version != legacyIndexVersion || index.Kind != legacyIndexKind { + return + } + for objectIndex := range index.Objects { + for blockIndex := range index.Objects[objectIndex].Blocks { + block := &index.Objects[objectIndex].Blocks[blockIndex] + if block.Encoding == "" { + block.Encoding = EncodingZstd + } + } + } +} + +func maxInt64() int64 { return int64(^uint(0) >> 1) } + +func safeGeneration(generation string) bool { + return generation != "" && generation != "." && generation != ".." && filepath.Base(generation) == generation +} diff --git a/internal/pack/nocache_darwin.go b/internal/pack/nocache_darwin.go new file mode 100644 index 0000000..41463c7 --- /dev/null +++ b/internal/pack/nocache_darwin.go @@ -0,0 +1,14 @@ +//go:build darwin + +package pack + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func configureNoCache(file *os.File) (bool, error) { + _, err := unix.FcntlInt(file.Fd(), unix.F_NOCACHE, 1) + return err == nil, err +} diff --git a/internal/pack/nocache_other.go b/internal/pack/nocache_other.go new file mode 100644 index 0000000..29e79a3 --- /dev/null +++ b/internal/pack/nocache_other.go @@ -0,0 +1,7 @@ +//go:build !darwin + +package pack + +import "os" + +func configureNoCache(*os.File) (bool, error) { return false, nil } diff --git a/internal/pack/pack_test.go b/internal/pack/pack_test.go new file mode 100644 index 0000000..da6fc62 --- /dev/null +++ b/internal/pack/pack_test.go @@ -0,0 +1,569 @@ +package pack + +import ( + "bytes" + "context" + cryptorand "crypto/rand" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/storage" +) + +func TestBuildAndResolverReadExactRandomRanges(t *testing.T) { + root := t.TempDir() + large := bytes.Repeat([]byte("large-object-block-"), 50000) + refs := putObjects(t, root, []byte("shared-small-object"), large) + writeManifest(t, root, "first", []fold.ObjectRef{refs[0], refs[1], refs[0]}) + writeManifest(t, root, "fork", []fold.ObjectRef{refs[0], refs[1]}) + + result, err := Build(context.Background(), root, BuildOptions{BlockBytes: 256 << 10, PackBytes: 1 << 20}) + if err != nil { + t.Fatalf("Build returned error: %v", err) + } + if result.ObjectCount != 2 || result.BlockCount < 3 || result.PackCount < 1 { + t.Fatalf("unexpected build result: %#v", result) + } + current, err := CurrentGeneration(root) + if err != nil || current != result.Generation { + t.Fatalf("current generation = %q, %v; want %q", current, err, result.Generation) + } + loose := fold.NewObjectStore(root) + for _, ref := range refs { + if err := os.Remove(loose.ObjectPath(ref.SHA256)); err != nil { + t.Fatalf("remove loose object after pack build: %v", err) + } + } + + resolver, err := Open(root, OpenOptions{CacheBytes: 512 << 10}) + if err != nil { + t.Fatalf("Open returned error: %v", err) + } + t.Cleanup(func() { _ = resolver.Close() }) + if resolver.Generation() != result.Generation { + t.Fatalf("resolver generation = %q, want %q", resolver.Generation(), result.Generation) + } + + for _, test := range []struct { + name string + ref fold.ObjectRef + data []byte + off int64 + size int + }{ + {name: "small", ref: refs[0], data: []byte("shared-small-object"), off: 2, size: 8}, + {name: "large-first", ref: refs[1], data: large, off: 17, size: 333}, + {name: "large-block-boundary", ref: refs[1], data: large, off: (256 << 10) - 31, size: 1000}, + {name: "large-tail", ref: refs[1], data: large, off: int64(len(large) - 101), size: 200}, + } { + t.Run(test.name, func(t *testing.T) { + buffer := make([]byte, test.size) + n, readErr := resolver.ReadAt(context.Background(), test.ref, buffer, test.off) + end := int(test.off) + test.size + if end > len(test.data) { + end = len(test.data) + } + want := test.data[int(test.off):end] + if !bytes.Equal(buffer[:n], want) { + t.Fatalf("ReadAt bytes differ: got=%d want=%d", n, len(want)) + } + if len(want) < test.size && !errors.Is(readErr, io.EOF) { + t.Fatalf("ReadAt error = %v, want EOF", readErr) + } + }) + } +} + +func TestResolverReusesDecoderAcrossDistinctBlocks(t *testing.T) { + root := t.TempDir() + value := bytes.Repeat([]byte("decoder-pool-block-data"), 20000) + refs := putObjects(t, root, value) + writeManifest(t, root, "session", refs) + if _, err := Build(context.Background(), root, BuildOptions{BlockBytes: 64 << 10, PackBytes: 1 << 20}); err != nil { + t.Fatalf("Build: %v", err) + } + resolver, err := Open(root, OpenOptions{CacheBytes: 1 << 20}) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { _ = resolver.Close() }) + + created := 0 + resolver.decoderFactory = func() (*zstd.Decoder, error) { + created++ + return newPackDecoder() + } + for _, offset := range []int64{0, 70 << 10} { + if _, err := resolver.ReadAt(context.Background(), refs[0], make([]byte, 1), offset); err != nil { + t.Fatalf("ReadAt(%d): %v", offset, err) + } + } + if created != 1 { + t.Fatalf("decoder creations = %d, want 1", created) + } +} + +func TestBlockCacheDoesNotRetainOversizedCallerBacking(t *testing.T) { + cache := newBlockCache(64) + value := make([]byte, 16, 4096) + copy(value, []byte("sixteen-byte-val")) + cache.put("object:0", value) + + cached, ok := cache.get("object:0") + if !ok { + t.Fatal("cache rejected a value within its byte budget") + } + if len(cached) != len(value) || cap(cached) != len(cached) { + t.Fatalf("cached slice len=%d cap=%d, want exact owned storage", len(cached), cap(cached)) + } + value[0] = 'X' + if cached[0] == value[0] { + t.Fatal("cache retained the caller's oversized backing array") + } + if cache.used != int64(len(cached)) { + t.Fatalf("cache used=%d, want %d", cache.used, len(cached)) + } +} + +func TestBuildSelectsRawAndZstdBlocksAndResolverReadsBoth(t *testing.T) { + root := t.TempDir() + raw := make([]byte, 8<<10) + if _, err := cryptorand.Read(raw); err != nil { + t.Fatal(err) + } + compressible := bytes.Repeat([]byte("highly-compressible-value"), 400) + refs := putObjects(t, root, raw, compressible) + writeManifest(t, root, "session", refs) + if _, err := Build(context.Background(), root, BuildOptions{}); err != nil { + t.Fatalf("Build: %v", err) + } + index := loadCurrentIndex(t, root) + encodings := make(map[string]string, len(index.Objects)) + for _, object := range index.Objects { + if len(object.Blocks) != 1 { + t.Fatalf("object %s blocks = %d, want 1", object.SHA256, len(object.Blocks)) + } + encodings[object.SHA256] = object.Blocks[0].Encoding + } + if encodings[refs[0].SHA256] != EncodingRaw || encodings[refs[1].SHA256] != EncodingZstd { + t.Fatalf("encodings = %#v", encodings) + } + + resolver, err := Open(root, OpenOptions{}) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer resolver.Close() + for index, want := range [][]byte{raw, compressible} { + got := make([]byte, len(want)) + if n, err := resolver.ReadAt(context.Background(), refs[index], got, 0); n != len(want) || err != nil { + t.Fatalf("ReadAt(%d) = %d, %v", index, n, err) + } + if !bytes.Equal(got, want) { + t.Fatalf("ReadAt(%d) bytes differ", index) + } + } +} + +func TestResolverOpensLegacyV1ZstdIndex(t *testing.T) { + root := t.TempDir() + value := bytes.Repeat([]byte("legacy-zstd-data"), 2000) + refs := putObjects(t, root, value) + writeManifest(t, root, "session", refs) + result, err := Build(context.Background(), root, BuildOptions{}) + if err != nil { + t.Fatalf("Build: %v", err) + } + index := loadCurrentIndex(t, root) + index.Version = legacyIndexVersion + index.Kind = legacyIndexKind + for objectIndex := range index.Objects { + for blockIndex := range index.Objects[objectIndex].Blocks { + block := &index.Objects[objectIndex].Blocks[blockIndex] + if block.Encoding != EncodingZstd { + t.Fatalf("legacy fixture block encoding = %q, want zstd", block.Encoding) + } + block.Encoding = "" + } + } + encoded, err := json.Marshal(index) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "packs", result.Generation, "index.json"), encoded, 0o600); err != nil { + t.Fatal(err) + } + resolver, err := Open(root, OpenOptions{}) + if err != nil { + t.Fatalf("Open legacy v1: %v", err) + } + defer resolver.Close() + got := make([]byte, len(value)) + if n, err := resolver.ReadAt(context.Background(), refs[0], got, 0); n != len(value) || err != nil || !bytes.Equal(got, value) { + t.Fatalf("legacy ReadAt = %d, %v, equal=%t", n, err, bytes.Equal(got, value)) + } +} + +func TestResolverSupportsOSCacheBypassOption(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, []byte("cache-bypass-object")) + writeManifest(t, root, "session", refs) + if _, err := Build(context.Background(), root, BuildOptions{}); err != nil { + t.Fatal(err) + } + resolver, err := Open(root, OpenOptions{BypassOSCache: true}) + if err != nil { + t.Fatal(err) + } + defer resolver.Close() + buffer := make([]byte, refs[0].RawBytes) + if _, err := resolver.ReadAt(context.Background(), refs[0], buffer, 0); err != nil { + t.Fatal(err) + } + if string(buffer) != "cache-bypass-object" { + t.Fatalf("unexpected bytes: %q", buffer) + } +} + +func TestResolverHoldsGenerationLeaseUntilClose(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, []byte("leased-generation")) + writeManifest(t, root, "session", refs) + result, err := Build(context.Background(), root, BuildOptions{}) + if err != nil { + t.Fatal(err) + } + resolver, err := Open(root, OpenOptions{}) + if err != nil { + t.Fatal(err) + } + leaseDirectory := filepath.Join(root, "packs", result.Generation, "leases") + active, err := storage.DirectoryHasActiveLease(leaseDirectory, false) + if err != nil || !active { + t.Fatalf("resolver generation lease: active=%t err=%v", active, err) + } + if err := resolver.Close(); err != nil { + t.Fatal(err) + } + active, err = storage.DirectoryHasActiveLease(leaseDirectory, true) + if err != nil || active { + t.Fatalf("closed resolver generation lease: active=%t err=%v", active, err) + } +} + +func TestResolversShareGenerationResourcesUntilLastClose(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, []byte("shared-generation-resources")) + writeManifest(t, root, "session", refs) + result, err := Build(context.Background(), root, BuildOptions{}) + if err != nil { + t.Fatal(err) + } + first, err := Open(root, OpenOptions{}) + if err != nil { + t.Fatal(err) + } + second, err := Open(root, OpenOptions{}) + if err != nil { + _ = first.Close() + t.Fatal(err) + } + if first.cache != second.cache || first.packs["pack-000001.pack"] != second.packs["pack-000001.pack"] { + _ = first.Close() + _ = second.Close() + t.Fatal("resolvers for one generation did not share cache and pack descriptors") + } + if err := first.Close(); err != nil { + t.Fatal(err) + } + leaseDirectory := filepath.Join(root, "packs", result.Generation, "leases") + active, err := storage.DirectoryHasActiveLease(leaseDirectory, false) + if err != nil || !active { + t.Fatalf("shared lease after first close: active=%t err=%v", active, err) + } + buffer := make([]byte, refs[0].RawBytes) + if _, err := second.ReadAt(context.Background(), refs[0], buffer, 0); err != nil { + t.Fatalf("second resolver after first close: %v", err) + } + if err := second.Close(); err != nil { + t.Fatal(err) + } + active, err = storage.DirectoryHasActiveLease(leaseDirectory, true) + if err != nil || active { + t.Fatalf("shared lease after final close: active=%t err=%v", active, err) + } +} + +func TestDoctorDoesNotPopulateActiveResolverCache(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, bytes.Repeat([]byte("doctor-cache-isolation"), 1000)) + writeManifest(t, root, "session", refs) + if _, err := Build(context.Background(), root, BuildOptions{}); err != nil { + t.Fatal(err) + } + resolver, err := Open(root, OpenOptions{}) + if err != nil { + t.Fatal(err) + } + defer resolver.Close() + if resolver.cache.used != 0 { + t.Fatalf("new resolver cache bytes = %d", resolver.cache.used) + } + report, err := Doctor(context.Background(), root) + if err != nil || report.IssueCount != 0 { + t.Fatalf("Doctor: report=%#v err=%v", report, err) + } + if resolver.cache.used != 0 { + t.Fatalf("doctor populated active resolver cache with %d bytes", resolver.cache.used) + } +} + +func TestOpenDoesNotRecreateGenerationMissingFromCurrent(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "packs"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "packs", "CURRENT"), []byte("gen-missing\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Open(root, OpenOptions{}); err == nil { + t.Fatal("resolver unexpectedly opened a missing generation") + } + if _, err := os.Lstat(filepath.Join(root, "packs", "gen-missing")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing pack generation was recreated: %v", err) + } +} + +func TestValidatePackedBlockBoundsUsesCachedPackSizes(t *testing.T) { + objectDigest := strings.Repeat("a", 64) + blockDigest := strings.Repeat("b", 64) + index := Index{ + Objects: []Object{{ + SHA256: objectDigest, + Blocks: []Block{ + {Pack: "pack-000001.bin", PackOffset: 0, StoredBytes: 10, SHA256: blockDigest}, + {Pack: "pack-000001.bin", PackOffset: 10, StoredBytes: 20, SHA256: blockDigest}, + }, + }}, + } + if err := validatePackedBlockBounds(index, map[string]int64{"pack-000001.bin": 30}); err != nil { + t.Fatalf("valid packed bounds: %v", err) + } + if err := validatePackedBlockBounds(index, map[string]int64{"pack-000001.bin": 29}); err == nil { + t.Fatal("truncated pack bounds were accepted") + } + if err := validatePackedBlockBounds(index, map[string]int64{}); err == nil { + t.Fatal("missing pack bounds were accepted") + } +} + +func TestBuildInterruptionKeepsPreviousGenerationCurrent(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, []byte("first-generation")) + writeManifest(t, root, "session", refs) + first, err := Build(context.Background(), root, BuildOptions{}) + if err != nil { + t.Fatalf("first Build returned error: %v", err) + } + + stop := errors.New("stop before publish") + if _, err := Build(context.Background(), root, BuildOptions{BeforePublish: func() error { return stop }}); !errors.Is(err, stop) { + t.Fatalf("interrupted Build error = %v, want %v", err, stop) + } + current, err := os.ReadFile(filepath.Join(root, "packs", "CURRENT")) + if err != nil { + t.Fatalf("read CURRENT: %v", err) + } + if string(bytes.TrimSpace(current)) != first.Generation { + t.Fatalf("CURRENT = %q, want %q", bytes.TrimSpace(current), first.Generation) + } + + resolver, err := Open(root, OpenOptions{}) + if err != nil { + t.Fatalf("Open previous generation: %v", err) + } + defer resolver.Close() + buffer := make([]byte, refs[0].RawBytes) + if _, err := resolver.ReadAt(context.Background(), refs[0], buffer, 0); err != nil && !errors.Is(err, io.EOF) { + t.Fatalf("read previous generation: %v", err) + } +} + +func TestBuildBudgetRejectsBeforeCreatingCandidateGeneration(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, []byte("budgeted-pack-object")) + writeManifest(t, root, "session", refs) + checker := rejectingChecker{} + if _, err := Build(context.Background(), root, BuildOptions{Budget: &checker}); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("Build error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 { + t.Fatalf("budget checks = %d, want 1", checker.Calls) + } + if _, err := os.Stat(filepath.Join(root, "packs")); !os.IsNotExist(err) { + t.Fatalf("candidate packs directory exists after preflight rejection: %v", err) + } +} + +func TestBuildReportsStorageAccounting(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, []byte("accounted-pack")) + writeManifest(t, root, "session", refs) + result, err := Build(context.Background(), root, BuildOptions{}) + if err != nil { + t.Fatal(err) + } + if result.Storage == nil || result.Storage.Budget.ProjectedPeakBytes <= result.Storage.Budget.CurrentPhysicalBytes || result.Storage.After.Packs.ApparentBytes == 0 { + t.Fatalf("pack storage accounting is incomplete: %#v", result.Storage) + } +} + +func TestResolverAndDoctorDetectPackCorruption(t *testing.T) { + root := t.TempDir() + refs := putObjects(t, root, bytes.Repeat([]byte("protected"), 10000)) + writeManifest(t, root, "session", refs) + if _, err := Build(context.Background(), root, BuildOptions{}); err != nil { + t.Fatalf("Build returned error: %v", err) + } + + index := loadCurrentIndex(t, root) + block := index.Objects[0].Blocks[0] + packPath := filepath.Join(root, "packs", index.Generation, block.Pack) + file, err := os.OpenFile(packPath, os.O_RDWR, 0) + if err != nil { + t.Fatalf("open pack: %v", err) + } + if _, err := file.WriteAt([]byte{0xff}, block.PackOffset); err != nil { + _ = file.Close() + t.Fatalf("corrupt pack: %v", err) + } + _ = file.Close() + + resolver, err := Open(root, OpenOptions{}) + if err != nil { + t.Fatalf("Open returned error: %v", err) + } + defer resolver.Close() + if _, err := resolver.ReadAt(context.Background(), refs[0], make([]byte, 16), 0); err == nil { + t.Fatal("ReadAt should reject a corrupt pack block") + } + report, err := Doctor(context.Background(), root) + if err != nil { + t.Fatalf("Doctor returned error: %v", err) + } + if report.IssueCount == 0 { + t.Fatalf("Doctor did not report corruption: %#v", report) + } +} + +func TestBuildIncludesObjectsReferencedByGenerationManifests(t *testing.T) { + root := t.TempDir() + data := []byte("generation-only-object") + store := fold.NewObjectStore(root) + ref, _, err := store.Put(data, true) + if err != nil { + t.Fatal(err) + } + if err := store.SyncPending(context.Background()); err != nil { + t.Fatal(err) + } + manifest := fold.Manifest{ + Version: fold.ManifestVersion, Kind: fold.ManifestKind, + Session: fold.ManifestSession{ID: "session", RolloutPath: "session.jsonl", Archived: true}, + Source: fold.ManifestSource{Bytes: int64(len(data)), SHA256: ref.SHA256}, + Parts: []fold.Part{{Kind: fold.PartResidual, Object: ref}}, + } + manifestPath := filepath.Join(root, "manifests", "generations", "session", "2.json") + if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(manifestPath, encoded, 0o600); err != nil { + t.Fatal(err) + } + result, err := Build(context.Background(), root, BuildOptions{}) + if err != nil { + t.Fatalf("Build: %v", err) + } + if result.ObjectCount != 1 { + t.Fatalf("object count = %d, want 1", result.ObjectCount) + } +} + +func putObjects(t *testing.T, root string, values ...[]byte) []fold.ObjectRef { + t.Helper() + store := fold.NewObjectStore(root) + refs := make([]fold.ObjectRef, 0, len(values)) + for _, value := range values { + ref, _, err := store.Put(value, true) + if err != nil { + t.Fatalf("Put returned error: %v", err) + } + refs = append(refs, ref) + } + if err := store.SyncPending(context.Background()); err != nil { + t.Fatalf("SyncPending returned error: %v", err) + } + return refs +} + +func writeManifest(t *testing.T, root string, sessionID string, refs []fold.ObjectRef) { + t.Helper() + manifest := fold.Manifest{ + Version: fold.ManifestVersion, + Kind: fold.ManifestKind, + Session: fold.ManifestSession{ID: sessionID, RolloutPath: filepath.Join(root, sessionID+".jsonl")}, + Parts: make([]fold.Part, 0, len(refs)), + } + for _, ref := range refs { + manifest.Source.Bytes += ref.RawBytes + manifest.Parts = append(manifest.Parts, fold.Part{Kind: fold.PartResidual, Object: ref}) + } + manifest.Source.SHA256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + data, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal manifest: %v", err) + } + if err := os.MkdirAll(filepath.Join(root, "manifests"), 0o755); err != nil { + t.Fatalf("create manifests: %v", err) + } + if err := os.WriteFile(fold.ManifestPath(root, sessionID), data, 0o644); err != nil { + t.Fatalf("write manifest: %v", err) + } +} + +func loadCurrentIndex(t *testing.T, root string) Index { + t.Helper() + current, err := os.ReadFile(filepath.Join(root, "packs", "CURRENT")) + if err != nil { + t.Fatalf("read CURRENT: %v", err) + } + data, err := os.ReadFile(filepath.Join(root, "packs", string(bytes.TrimSpace(current)), "index.json")) + if err != nil { + t.Fatalf("read index: %v", err) + } + var index Index + if err := json.Unmarshal(data, &index); err != nil { + t.Fatalf("decode index: %v", err) + } + return index +} + +type rejectingChecker struct { + Calls int +} + +func (c *rejectingChecker) Check(context.Context, storage.Projection) (storage.Assessment, error) { + c.Calls++ + return storage.Assessment{}, storage.ErrBudgetExceeded +} diff --git a/internal/pack/replace_unix.go b/internal/pack/replace_unix.go new file mode 100644 index 0000000..f935f6b --- /dev/null +++ b/internal/pack/replace_unix.go @@ -0,0 +1,9 @@ +//go:build !windows + +package pack + +import "os" + +func replaceFile(source string, target string) error { + return os.Rename(source, target) +} diff --git a/internal/pack/replace_windows.go b/internal/pack/replace_windows.go new file mode 100644 index 0000000..845910c --- /dev/null +++ b/internal/pack/replace_windows.go @@ -0,0 +1,36 @@ +//go:build windows + +package pack + +import ( + "fmt" + "syscall" + "unsafe" +) + +const ( + moveFileReplaceExisting = 0x1 + moveFileWriteThrough = 0x8 +) + +var moveFileExW = syscall.NewLazyDLL("kernel32.dll").NewProc("MoveFileExW") + +func replaceFile(source string, target string) error { + sourcePointer, err := syscall.UTF16PtrFromString(source) + if err != nil { + return err + } + targetPointer, err := syscall.UTF16PtrFromString(target) + if err != nil { + return err + } + result, _, callErr := moveFileExW.Call( + uintptr(unsafe.Pointer(sourcePointer)), + uintptr(unsafe.Pointer(targetPointer)), + moveFileReplaceExisting|moveFileWriteThrough, + ) + if result == 0 { + return fmt.Errorf("replace file: %w", callErr) + } + return nil +} diff --git a/internal/pack/resolver.go b/internal/pack/resolver.go new file mode 100644 index 0000000..510ffd4 --- /dev/null +++ b/internal/pack/resolver.go @@ -0,0 +1,391 @@ +package pack + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + + "github.com/klauspost/compress/zstd" + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/storage" +) + +type OpenOptions struct { + CacheBytes int64 + BypassOSCache bool +} + +type Resolver struct { + directory string + index Index + objects map[string]Object + packs map[string]*os.File + cache *blockCache + decoders chan *zstd.Decoder + decoderFactory func() (*zstd.Decoder, error) + lease *storage.Lease + bypassOSCacheApplied bool + shared *resolverResources + closeOnce sync.Once + closeErr error +} + +type resolverResourceKey struct { + directory string + cacheBytes int64 + bypassOSCache bool +} + +type resolverResources struct { + key resolverResourceKey + directory string + index Index + objects map[string]Object + packs map[string]*os.File + cache *blockCache + decoders chan *zstd.Decoder + lease *storage.Lease + bypassOSCacheApplied bool + references int +} + +var resolverRegistry = struct { + sync.Mutex + resources map[resolverResourceKey]*resolverResources +}{resources: make(map[resolverResourceKey]*resolverResources)} + +func Open(storeDir string, options OpenOptions) (*Resolver, error) { + generation, err := CurrentGeneration(storeDir) + if err != nil { + return nil, err + } + if options.CacheBytes == 0 { + options.CacheBytes = defaultCacheBytes + } + return openGeneration(filepath.Join(storeDir, "packs", generation), options.CacheBytes, options.BypassOSCache) +} + +func CurrentGeneration(storeDir string) (string, error) { + current, err := os.ReadFile(filepath.Join(storeDir, "packs", "CURRENT")) + if err != nil { + return "", fmt.Errorf("read pack CURRENT: %w", err) + } + generation := strings.TrimSpace(string(current)) + if !safeGeneration(generation) { + return "", fmt.Errorf("unsafe pack generation %q", generation) + } + return generation, nil +} + +func openGeneration(directory string, cacheBytes int64, bypassOSCache bool) (*Resolver, error) { + directory = filepath.Clean(directory) + if cacheBytes < 0 { + cacheBytes = 0 + } + key := resolverResourceKey{directory: directory, cacheBytes: cacheBytes, bypassOSCache: bypassOSCache} + resolverRegistry.Lock() + defer resolverRegistry.Unlock() + if shared := resolverRegistry.resources[key]; shared != nil { + shared.references++ + return resolverFromResources(shared), nil + } + shared, err := loadResolverResources(key) + if err != nil { + return nil, err + } + shared.references = 1 + resolverRegistry.resources[key] = shared + return resolverFromResources(shared), nil +} + +func loadResolverResources(key resolverResourceKey) (*resolverResources, error) { + directory := key.directory + lease, err := storage.AcquireLease(filepath.Join(directory, "leases"), "resolver") + if err != nil { + return nil, fmt.Errorf("acquire pack generation lease: %w", err) + } + var shared *resolverResources + keepResources := false + defer func() { + if keepResources { + return + } + if shared != nil { + _ = closeResolverResources(shared) + } else { + _ = lease.Close() + } + }() + data, err := os.ReadFile(filepath.Join(directory, "index.json")) + if err != nil { + return nil, fmt.Errorf("read pack index: %w", err) + } + var index Index + if err := json.Unmarshal(data, &index); err != nil { + return nil, fmt.Errorf("decode pack index: %w", err) + } + normalizeLegacyIndex(&index) + directoryName := filepath.Base(directory) + if directoryName != index.Generation && !strings.HasPrefix(directoryName, ".generation-") { + return nil, fmt.Errorf("pack index generation %q does not match directory %q", index.Generation, filepath.Base(directory)) + } + if err := validateIndex(index); err != nil { + return nil, err + } + decoderPoolSize := runtime.GOMAXPROCS(0) + if decoderPoolSize < 1 { + decoderPoolSize = 1 + } + if decoderPoolSize > 32 { + decoderPoolSize = 32 + } + shared = &resolverResources{ + key: key, + directory: directory, index: index, objects: make(map[string]Object, len(index.Objects)), + packs: make(map[string]*os.File), cache: newBlockCache(key.cacheBytes), + decoders: make(chan *zstd.Decoder, decoderPoolSize), lease: lease, + } + for _, object := range index.Objects { + shared.objects[object.SHA256] = object + for _, block := range object.Blocks { + if _, ok := shared.packs[block.Pack]; ok { + continue + } + file, err := os.Open(filepath.Join(directory, block.Pack)) + if err != nil { + return nil, fmt.Errorf("open pack %s: %w", block.Pack, err) + } + if key.bypassOSCache { + applied, err := configureNoCache(file) + if err != nil { + _ = file.Close() + return nil, fmt.Errorf("disable OS cache for pack %s: %w", block.Pack, err) + } + shared.bypassOSCacheApplied = shared.bypassOSCacheApplied || applied + } + shared.packs[block.Pack] = file + } + } + packSizes, err := resolverPackSizes(shared.packs) + if err != nil { + return nil, err + } + if err := validatePackedBlockBounds(index, packSizes); err != nil { + return nil, err + } + keepResources = true + return shared, nil +} + +func resolverFromResources(shared *resolverResources) *Resolver { + return &Resolver{ + directory: shared.directory, index: shared.index, objects: shared.objects, + packs: shared.packs, cache: shared.cache, decoders: shared.decoders, + decoderFactory: newPackDecoder, lease: shared.lease, + bypassOSCacheApplied: shared.bypassOSCacheApplied, shared: shared, + } +} + +func resolverPackSizes(packs map[string]*os.File) (map[string]int64, error) { + sizes := make(map[string]int64, len(packs)) + for name, file := range packs { + info, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("stat pack %s: %w", name, err) + } + sizes[name] = info.Size() + } + return sizes, nil +} + +func validatePackedBlockBounds(index Index, packSizes map[string]int64) error { + for _, object := range index.Objects { + for blockIndex, block := range object.Blocks { + packSize, exists := packSizes[block.Pack] + if !exists { + return fmt.Errorf("packed block %s:%d references unopened pack %s", object.SHA256, blockIndex, block.Pack) + } + if block.PackOffset > packSize || block.StoredBytes > packSize-block.PackOffset { + return fmt.Errorf("packed block %s:%d exceeds %s size", object.SHA256, blockIndex, block.Pack) + } + } + } + return nil +} + +func (r *Resolver) OSCacheBypassApplied() bool { return r.bypassOSCacheApplied } + +func (r *Resolver) Generation() string { return r.index.Generation } + +func (r *Resolver) ReadAt(ctx context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { + if offset < 0 { + return 0, errors.New("negative object read offset") + } + if len(destination) == 0 { + return 0, nil + } + object, ok := r.objects[ref.SHA256] + if !ok { + return 0, fmt.Errorf("object %s is not packed", ref.SHA256) + } + if object.RawBytes != ref.RawBytes { + return 0, fmt.Errorf("object %s raw size %d, want %d", ref.SHA256, object.RawBytes, ref.RawBytes) + } + if offset >= object.RawBytes { + return 0, io.EOF + } + written := 0 + for written < len(destination) && offset < object.RawBytes { + if err := ctx.Err(); err != nil { + return written, err + } + blockIndex := sort.Search(len(object.Blocks), func(index int) bool { + block := object.Blocks[index] + return block.RawOffset+block.RawBytes > offset + }) + if blockIndex == len(object.Blocks) { + return written, fmt.Errorf("object %s has no block for offset %d", object.SHA256, offset) + } + block := object.Blocks[blockIndex] + data, err := r.readBlock(object.SHA256, blockIndex, block) + if err != nil { + return written, err + } + inside := offset - block.RawOffset + copied := copy(destination[written:], data[inside:]) + written += copied + offset += int64(copied) + } + if written < len(destination) { + return written, io.EOF + } + return written, nil +} + +func (r *Resolver) readBlock(objectDigest string, blockIndex int, block Block) ([]byte, error) { + key := fmt.Sprintf("%s:%d", objectDigest, blockIndex) + if data, ok := r.cache.get(key); ok { + return data, nil + } + file := r.packs[block.Pack] + if file == nil { + return nil, fmt.Errorf("pack %s is not open", block.Pack) + } + stored := make([]byte, int(block.StoredBytes)) + if _, err := file.ReadAt(stored, block.PackOffset); err != nil { + return nil, fmt.Errorf("read packed block %s:%d: %w", objectDigest, blockIndex, err) + } + var data []byte + if block.Encoding == EncodingRaw { + data = stored + } else { + if block.RawBytes > int64(^uint(0)>>1) { + return nil, fmt.Errorf("packed block %s:%d raw size exceeds platform limit", objectDigest, blockIndex) + } + decoder, err := r.acquireDecoder() + if err != nil { + return nil, fmt.Errorf("create pack decoder: %w", err) + } + data, err = decoder.DecodeAll(stored, make([]byte, 0, int(block.RawBytes))) + if err != nil { + decoder.Close() + return nil, fmt.Errorf("decode packed block %s:%d: %w", objectDigest, blockIndex, err) + } + r.releaseDecoder(decoder) + } + if int64(len(data)) != block.RawBytes { + return nil, fmt.Errorf("packed block %s:%d raw size %d, want %d", objectDigest, blockIndex, len(data), block.RawBytes) + } + digest := sha256.Sum256(data) + if hex.EncodeToString(digest[:]) != block.SHA256 { + return nil, fmt.Errorf("packed block %s:%d SHA-256 mismatch", objectDigest, blockIndex) + } + r.cache.put(key, data) + return data, nil +} + +func newPackDecoder() (*zstd.Decoder, error) { + return zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderLowmem(true)) +} + +func (r *Resolver) acquireDecoder() (*zstd.Decoder, error) { + select { + case decoder := <-r.decoders: + return decoder, nil + default: + return r.decoderFactory() + } +} + +func (r *Resolver) releaseDecoder(decoder *zstd.Decoder) { + select { + case r.decoders <- decoder: + default: + decoder.Close() + } +} + +func (r *Resolver) Close() error { + r.closeOnce.Do(func() { + r.closeErr = releaseResolverResources(r.shared) + }) + return r.closeErr +} + +func releaseResolverResources(shared *resolverResources) error { + if shared == nil { + return nil + } + resolverRegistry.Lock() + registered := resolverRegistry.resources[shared.key] + if registered != shared || shared.references <= 0 { + resolverRegistry.Unlock() + return errors.New("pack resolver resource reference is not registered") + } + shared.references-- + if shared.references > 0 { + resolverRegistry.Unlock() + return nil + } + delete(resolverRegistry.resources, shared.key) + resolverRegistry.Unlock() + return closeResolverResources(shared) +} + +func closeResolverResources(shared *resolverResources) error { + if shared == nil { + return nil + } + var closeErr error + names := make([]string, 0, len(shared.packs)) + for name := range shared.packs { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if err := shared.packs[name].Close(); err != nil && closeErr == nil { + closeErr = err + } + } + for { + select { + case decoder := <-shared.decoders: + decoder.Close() + default: + if shared.lease != nil { + closeErr = errors.Join(closeErr, shared.lease.Close()) + } + return closeErr + } + } +} diff --git a/internal/prune/remove_contained.go b/internal/prune/remove_contained.go index ab0962d..b6724b1 100644 --- a/internal/prune/remove_contained.go +++ b/internal/prune/remove_contained.go @@ -15,9 +15,9 @@ import ( "strings" "time" - "github.com/jstar0/codexfold/internal/codex" - "github.com/jstar0/codexfold/internal/contain" - "github.com/jstar0/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/contain" + "github.com/samekind/codexfold/internal/fold" _ "modernc.org/sqlite" ) diff --git a/internal/prune/remove_contained_test.go b/internal/prune/remove_contained_test.go index a54feee..67b4150 100644 --- a/internal/prune/remove_contained_test.go +++ b/internal/prune/remove_contained_test.go @@ -9,8 +9,8 @@ import ( "path/filepath" "testing" - "github.com/jstar0/codexfold/internal/codex" - "github.com/jstar0/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/fold" _ "modernc.org/sqlite" ) @@ -168,7 +168,10 @@ func newRemovalFixture(t *testing.T) removalFixture { } contained := codex.Session{ID: "contained", Title: "Contained", CWD: "/workspace", RolloutPath: containedPath, Archived: true} container := codex.Session{ID: "container", Title: "Container", CWD: "/workspace", RolloutPath: containerPath} - if _, err := fold.Fold(context.Background(), contained, fold.FoldOptions{StoreDir: store, Apply: true, FieldThreshold: 4}); err != nil { + if _, err := fold.Fold(context.Background(), fold.Session{ + ID: contained.ID, Title: contained.Title, CWD: contained.CWD, + RolloutPath: contained.RolloutPath, Archived: contained.Archived, + }, fold.FoldOptions{StoreDir: store, Apply: true, FieldThreshold: 4}); err != nil { t.Fatalf("fold contained fixture: %v", err) } diff --git a/internal/reconcile/budget.go b/internal/reconcile/budget.go new file mode 100644 index 0000000..89d9688 --- /dev/null +++ b/internal/reconcile/budget.go @@ -0,0 +1,18 @@ +package reconcile + +import ( + "errors" + "math" +) + +func estimatedOutputBytes(sourceBytes int64) (int64, error) { + const fixedOverhead = int64(1 << 20) + if sourceBytes < 0 { + return 0, errors.New("output byte estimate cannot be negative") + } + overhead := sourceBytes/16 + fixedOverhead + if sourceBytes > math.MaxInt64-overhead { + return 0, errors.New("output byte estimate overflow") + } + return sourceBytes + overhead, nil +} diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go new file mode 100644 index 0000000..47c2e2c --- /dev/null +++ b/internal/reconcile/reconcile.go @@ -0,0 +1,481 @@ +package reconcile + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "time" + + "github.com/samekind/codexfold/internal/storage" +) + +type SourceSummary struct { + Path string `json:"path"` + Bytes int64 `json:"bytes"` + Records int64 `json:"records"` + SHA256 string `json:"sha256"` + FirstTimestamp string `json:"first_timestamp"` + LastTimestamp string `json:"last_timestamp"` + TimestampRegressions int64 `json:"timestamp_regressions"` +} + +type Result struct { + Base SourceSummary `json:"base"` + Branch SourceSummary `json:"branch"` + SharedRecords int64 `json:"shared_records"` + BaseOnlyRecords int64 `json:"base_only_records"` + AddedFromBranch int64 `json:"added_from_branch"` + OutputRecords int64 `json:"output_records"` + OutputBytes int64 `json:"output_bytes,omitempty"` + OutputSHA256 string `json:"output_sha256,omitempty"` + OutputPath string `json:"output_path,omitempty"` + OutputRegressions int64 `json:"output_timestamp_regressions,omitempty"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` +} + +type MergeOptions struct { + Context context.Context + Budget storage.Checker +} + +type recordKey struct { + digest [sha256.Size]byte + length int64 +} + +type recordRef struct { + source int + sequence int64 + offset int64 + length int64 + timestamp time.Time + key recordKey +} + +type scannedSource struct { + summary SourceSummary + records []recordRef + counts map[recordKey]int64 +} + +func Analyze(basePath, branchPath string) (Result, error) { + base, err := scanPath(basePath, 0) + if err != nil { + return Result{}, fmt.Errorf("scan base: %w", err) + } + branch, err := scanPath(branchPath, 1) + if err != nil { + return Result{}, fmt.Errorf("scan branch: %w", err) + } + result, _ := reconcileRecords(base, branch) + return result, nil +} + +func Merge(basePath, branchPath, outputPath string) (Result, error) { + return MergeWithOptions(basePath, branchPath, outputPath, MergeOptions{}) +} + +func MergeWithOptions(basePath, branchPath, outputPath string, options MergeOptions) (Result, error) { + if outputPath == "" { + return Result{}, errors.New("output path is required") + } + baseAbs, err := filepath.Abs(basePath) + if err != nil { + return Result{}, err + } + branchAbs, err := filepath.Abs(branchPath) + if err != nil { + return Result{}, err + } + outputAbs, err := filepath.Abs(outputPath) + if err != nil { + return Result{}, err + } + if outputAbs == baseAbs || outputAbs == branchAbs { + return Result{}, errors.New("output must not replace either source") + } + if _, err := os.Lstat(outputAbs); err == nil { + return Result{}, fmt.Errorf("output already exists: %s", outputAbs) + } else if !errors.Is(err, os.ErrNotExist) { + return Result{}, err + } + + base, err := scanPath(baseAbs, 0) + if err != nil { + return Result{}, fmt.Errorf("scan base: %w", err) + } + branch, err := scanPath(branchAbs, 1) + if err != nil { + return Result{}, fmt.Errorf("scan branch: %w", err) + } + result, records := reconcileRecords(base, branch) + sort.SliceStable(records, func(i, j int) bool { + if records[i].timestamp.Equal(records[j].timestamp) { + if records[i].source == records[j].source { + return records[i].sequence < records[j].sequence + } + return records[i].source < records[j].source + } + return records[i].timestamp.Before(records[j].timestamp) + }) + ctx := options.Context + if ctx == nil { + ctx = context.Background() + } + var outputBytes int64 + for _, record := range records { + if record.length < 0 || outputBytes > int64(^uint64(0)>>1)-record.length { + return Result{}, errors.New("reconciled output byte count overflow") + } + outputBytes += record.length + } + budget := options.Budget + if budget == nil { + budget = storage.VolumeGuard{Path: filepath.Dir(outputAbs)} + } + storageAssessment, err := budget.Check(ctx, storage.Projection{ + Operation: "reconcile-rollout", AdditionalPersistentBytes: outputBytes, + TemporaryBytes: outputBytes, TemporaryPersistentOverlapBytes: outputBytes, + }) + if err != nil { + return Result{}, err + } + + baseFile, err := os.Open(baseAbs) + if err != nil { + return Result{}, err + } + defer baseFile.Close() + branchFile, err := os.Open(branchAbs) + if err != nil { + return Result{}, err + } + defer branchFile.Close() + + if err := os.MkdirAll(filepath.Dir(outputAbs), 0o700); err != nil { + return Result{}, err + } + temp, err := os.CreateTemp(filepath.Dir(outputAbs), ".codexfold-reconcile-*.tmp") + if err != nil { + return Result{}, err + } + tempPath := temp.Name() + committed := false + defer func() { + if !committed { + _ = temp.Close() + _ = os.Remove(tempPath) + } + }() + if err := temp.Chmod(0o600); err != nil { + return Result{}, err + } + + outputHasher := sha256.New() + writer := io.MultiWriter(temp, outputHasher) + for _, record := range records { + if err := ctx.Err(); err != nil { + return Result{}, err + } + source := baseFile + if record.source == 1 { + source = branchFile + } + if _, err := io.Copy(writer, io.NewSectionReader(source, record.offset, record.length)); err != nil { + return Result{}, fmt.Errorf("write merged record: %w", err) + } + } + if err := temp.Sync(); err != nil { + return Result{}, err + } + if err := temp.Close(); err != nil { + return Result{}, err + } + if err := verifyUnchanged(baseAbs, base.summary); err != nil { + return Result{}, fmt.Errorf("base changed during merge: %w", err) + } + if err := verifyUnchanged(branchAbs, branch.summary); err != nil { + return Result{}, fmt.Errorf("branch changed during merge: %w", err) + } + if err := os.Rename(tempPath, outputAbs); err != nil { + return Result{}, err + } + committed = true + if err := syncDir(filepath.Dir(outputAbs)); err != nil { + return Result{}, err + } + + output, err := scanPath(outputAbs, 2) + if err != nil { + return Result{}, fmt.Errorf("verify output: %w", err) + } + result.OutputPath = outputAbs + result.OutputBytes = output.summary.Bytes + result.OutputSHA256 = hex.EncodeToString(outputHasher.Sum(nil)) + result.OutputRegressions = output.summary.TimestampRegressions + if output.summary.Records != result.OutputRecords || output.summary.SHA256 != result.OutputSHA256 || output.summary.TimestampRegressions != 0 { + return Result{}, errors.New("merged output verification failed") + } + result.Storage = storage.CompleteAccounting(ctx, storageAssessment, "") + return result, nil +} + +func reconcileRecords(base, branch scannedSource) (Result, []recordRef) { + remaining := make(map[recordKey]int64, len(base.counts)) + for key, count := range base.counts { + remaining[key] = count + } + records := make([]recordRef, 0, len(base.records)+len(branch.records)) + records = append(records, base.records...) + var shared int64 + var added int64 + for _, record := range branch.records { + if remaining[record.key] > 0 { + remaining[record.key]-- + shared++ + continue + } + records = append(records, record) + added++ + } + return Result{ + Base: base.summary, + Branch: branch.summary, + SharedRecords: shared, + BaseOnlyRecords: base.summary.Records - shared, + AddedFromBranch: added, + OutputRecords: base.summary.Records + added, + }, records +} + +func scanPath(path string, source int) (scannedSource, error) { + file, err := os.Open(path) + if err != nil { + return scannedSource{}, err + } + defer file.Close() + before, err := file.Stat() + if err != nil { + return scannedSource{}, err + } + + result := scannedSource{ + summary: SourceSummary{Path: path}, + counts: make(map[recordKey]int64), + } + reader := bufio.NewReaderSize(file, 1024*1024) + fileHasher := sha256.New() + var offset int64 + var previous time.Time + for sequence := int64(0); ; sequence++ { + start := offset + recordHasher := sha256.New() + timestampExtractor := newTimestampExtractor() + hasData := false + reachedEOF := false + for { + fragment, readErr := reader.ReadSlice('\n') + if len(fragment) > 0 { + hasData = true + offset += int64(len(fragment)) + _, _ = fileHasher.Write(fragment) + _, _ = recordHasher.Write(fragment) + if err := timestampExtractor.Write(fragment); err != nil { + return scannedSource{}, err + } + } + switch { + case readErr == nil: + goto recordComplete + case errors.Is(readErr, bufio.ErrBufferFull): + continue + case errors.Is(readErr, io.EOF): + reachedEOF = true + goto recordComplete + default: + return scannedSource{}, readErr + } + } + + recordComplete: + if !hasData { + break + } + timestamp, err := timestampExtractor.Timestamp() + if err != nil { + return scannedSource{}, fmt.Errorf("record %d at byte %d: %w", sequence+1, start, err) + } + if !previous.IsZero() && timestamp.Before(previous) { + result.summary.TimestampRegressions++ + } + if result.summary.Records == 0 { + result.summary.FirstTimestamp = timestamp.Format(time.RFC3339Nano) + } + previous = timestamp + result.summary.LastTimestamp = timestamp.Format(time.RFC3339Nano) + var digest [sha256.Size]byte + copy(digest[:], recordHasher.Sum(nil)) + key := recordKey{digest: digest, length: offset - start} + result.records = append(result.records, recordRef{ + source: source, + sequence: sequence, + offset: start, + length: offset - start, + timestamp: timestamp, + key: key, + }) + result.counts[key]++ + result.summary.Records++ + if reachedEOF { + break + } + } + after, err := file.Stat() + if err != nil { + return scannedSource{}, err + } + if before.Size() != after.Size() || !before.ModTime().Equal(after.ModTime()) { + return scannedSource{}, errors.New("source changed while scanning") + } + result.summary.Bytes = offset + result.summary.SHA256 = hex.EncodeToString(fileHasher.Sum(nil)) + return result, nil +} + +type timestampExtractor struct { + depth int + inString bool + escaped bool + capture bool + token []byte + topKey string + expectation extractorExpectation + timestamp string +} + +type extractorExpectation uint8 + +const ( + expectKey extractorExpectation = iota + expectColon + expectValue +) + +func newTimestampExtractor() *timestampExtractor { + return ×tampExtractor{expectation: expectKey} +} + +func (e *timestampExtractor) Write(data []byte) error { + for _, char := range data { + if e.inString { + if e.escaped { + e.escaped = false + if e.capture { + e.token = append(e.token, char) + } + continue + } + if char == '\\' { + e.escaped = true + continue + } + if char == '"' { + e.inString = false + if e.capture { + switch e.expectation { + case expectKey: + e.topKey = string(e.token) + e.expectation = expectColon + case expectValue: + if e.topKey == "timestamp" { + e.timestamp = string(e.token) + } + } + } + e.token = e.token[:0] + e.capture = false + continue + } + if e.capture { + e.token = append(e.token, char) + } + continue + } + + switch char { + case '"': + e.inString = true + if e.depth == 1 && (e.expectation == expectKey || (e.expectation == expectValue && e.topKey == "timestamp")) { + e.capture = true + e.token = e.token[:0] + } + case '{', '[': + e.depth++ + case '}', ']': + if e.depth > 0 { + e.depth-- + } + case ':': + if e.depth == 1 && e.expectation == expectColon { + e.expectation = expectValue + } + case ',': + if e.depth == 1 { + e.expectation = expectKey + e.topKey = "" + } + } + } + return nil +} + +func (e *timestampExtractor) Timestamp() (time.Time, error) { + if e.timestamp == "" { + return time.Time{}, errors.New("top-level timestamp not found in record") + } + timestamp, err := time.Parse(time.RFC3339Nano, e.timestamp) + if err != nil { + return time.Time{}, fmt.Errorf("invalid timestamp %q: %w", e.timestamp, err) + } + return timestamp, nil +} + +func verifyUnchanged(path string, expected SourceSummary) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return err + } + if info.Size() != expected.Bytes { + return fmt.Errorf("size is %d, expected %d", info.Size(), expected.Bytes) + } + hasher := sha256.New() + if _, err := io.Copy(hasher, file); err != nil { + return err + } + actual := hex.EncodeToString(hasher.Sum(nil)) + if actual != expected.SHA256 { + return fmt.Errorf("sha256 is %s, expected %s", actual, expected.SHA256) + } + return nil +} + +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go new file mode 100644 index 0000000..9b66dcd --- /dev/null +++ b/internal/reconcile/reconcile_test.go @@ -0,0 +1,168 @@ +package reconcile + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/samekind/codexfold/internal/storage" +) + +type reconcileRejectingChecker struct { + Calls int + Projection storage.Projection +} + +func (c *reconcileRejectingChecker) Check(_ context.Context, projection storage.Projection) (storage.Assessment, error) { + c.Calls++ + c.Projection = projection + return storage.Assessment{}, storage.ErrBudgetExceeded +} + +func TestMergeInsertsBranchOnlyRecordsByTimestamp(t *testing.T) { + dir := t.TempDir() + base := writeRollout(t, dir, "base.jsonl", []string{ + record("2026-07-13T01:00:00Z", "a"), + record("2026-07-13T01:02:00Z", "c"), + }) + branch := writeRollout(t, dir, "branch.jsonl", []string{ + record("2026-07-13T01:00:00Z", "a"), + record("2026-07-13T01:01:00Z", "b"), + record("2026-07-13T01:02:00Z", "c"), + }) + output := filepath.Join(dir, "merged.jsonl") + + result, err := Merge(base, branch, output) + if err != nil { + t.Fatal(err) + } + if result.SharedRecords != 2 || result.AddedFromBranch != 1 || result.OutputRecords != 3 { + t.Fatalf("unexpected result: %#v", result) + } + if result.Storage == nil || result.Storage.Budget.ProjectedPeakBytes <= 0 || result.Storage.ActualReclaimedBytes != 0 { + t.Fatalf("merge storage accounting is incomplete: %#v", result.Storage) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + want := strings.Join([]string{ + record("2026-07-13T01:00:00Z", "a"), + record("2026-07-13T01:01:00Z", "b"), + record("2026-07-13T01:02:00Z", "c"), + }, "\n") + "\n" + if string(data) != want { + t.Fatalf("merged bytes:\n%s\nwant:\n%s", data, want) + } +} + +func TestMergeBudgetRejectsBeforeCreatingOutput(t *testing.T) { + dir := t.TempDir() + base := writeRollout(t, dir, "base.jsonl", []string{record("2026-07-13T01:00:00Z", "a")}) + branch := writeRollout(t, dir, "branch.jsonl", []string{record("2026-07-13T01:01:00Z", "b")}) + output := filepath.Join(dir, "output", "merged.jsonl") + checker := &reconcileRejectingChecker{} + if _, err := MergeWithOptions(base, branch, output, MergeOptions{Budget: checker}); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("MergeWithOptions error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 || checker.Projection.Operation != "reconcile-rollout" || checker.Projection.AdditionalPersistentBytes <= 0 { + t.Fatalf("unexpected merge budget projection: %#v", checker) + } + if _, err := os.Stat(filepath.Dir(output)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("merge output directory exists after preflight rejection: %v", err) + } +} + +func TestMergePreservesExcessDuplicateOccurrence(t *testing.T) { + dir := t.TempDir() + line := record("2026-07-13T01:00:00Z", "same") + base := writeRollout(t, dir, "base.jsonl", []string{line}) + branch := writeRollout(t, dir, "branch.jsonl", []string{line, line}) + output := filepath.Join(dir, "merged.jsonl") + + result, err := Merge(base, branch, output) + if err != nil { + t.Fatal(err) + } + if result.SharedRecords != 1 || result.AddedFromBranch != 1 || result.OutputRecords != 2 { + t.Fatalf("unexpected result: %#v", result) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if string(data) != line+"\n"+line+"\n" { + t.Fatalf("duplicate occurrence was not preserved: %q", data) + } +} + +func TestMergeSortsTimestampRegression(t *testing.T) { + dir := t.TempDir() + base := writeRollout(t, dir, "base.jsonl", []string{ + record("2026-07-13T01:01:00Z", "later"), + record("2026-07-13T01:00:00Z", "earlier"), + }) + branch := writeRollout(t, dir, "branch.jsonl", []string{ + record("2026-07-13T01:00:30Z", "branch"), + }) + + report, err := Analyze(base, branch) + if err != nil { + t.Fatal(err) + } + if report.Base.TimestampRegressions != 1 { + t.Fatalf("regressions = %d, want 1", report.Base.TimestampRegressions) + } + output := filepath.Join(dir, "merged.jsonl") + if _, err := Merge(base, branch, output); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"value":"earlier"`) || !strings.HasPrefix(string(data), record("2026-07-13T01:00:00Z", "earlier")) { + t.Fatalf("merge did not sort records: %q", data) + } +} + +func TestAnalyzeRejectsMissingTimestamp(t *testing.T) { + dir := t.TempDir() + base := writeRollout(t, dir, "base.jsonl", []string{`{"type":"event_msg"}`}) + branch := writeRollout(t, dir, "branch.jsonl", []string{record("2026-07-13T01:00:00Z", "ok")}) + + if _, err := Analyze(base, branch); err == nil { + t.Fatal("analyze accepted a record without a timestamp") + } +} + +func TestAnalyzeFindsTopLevelTimestampAfterLargePayload(t *testing.T) { + dir := t.TempDir() + line := `{"payload":{"text":"` + strings.Repeat("x", 16*1024) + `","timestamp":"2000-01-01T00:00:00Z"},"timestamp":"2026-07-13T01:00:00Z","type":"session_meta"}` + base := writeRollout(t, dir, "base.jsonl", []string{line}) + branch := writeRollout(t, dir, "branch.jsonl", []string{line}) + + report, err := Analyze(base, branch) + if err != nil { + t.Fatal(err) + } + if report.Base.FirstTimestamp != "2026-07-13T01:00:00Z" { + t.Fatalf("timestamp = %s", report.Base.FirstTimestamp) + } +} + +func record(timestamp, value string) string { + return `{"timestamp":"` + timestamp + `","type":"event_msg","payload":{"value":"` + value + `"}}` +} + +func writeRollout(t *testing.T, dir, name string, lines []string) string { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil { + t.Fatal(err) + } + return path +} diff --git a/internal/reconcile/repair.go b/internal/reconcile/repair.go new file mode 100644 index 0000000..bf44c40 --- /dev/null +++ b/internal/reconcile/repair.go @@ -0,0 +1,494 @@ +package reconcile + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "time" + + "github.com/samekind/codexfold/internal/storage" +) + +const maxRepairBufferedBytes = int64(64 * 1024 * 1024) + +var recordStartPattern = regexp.MustCompile(`\{"timestamp"\s*:`) + +type RepairResult struct { + SourcePath string `json:"source_path"` + SourceBytes int64 `json:"source_bytes"` + SourceSHA256 string `json:"source_sha256"` + PhysicalLines int64 `json:"physical_lines"` + InvalidPhysicalLines int64 `json:"invalid_physical_lines"` + ReconstructedRecords int64 `json:"reconstructed_records"` + SourceConversationRecords int64 `json:"source_conversation_records"` + PreservedConversationRecords int64 `json:"preserved_conversation_records"` + ReconstructedConversationRecords int64 `json:"reconstructed_conversation_records"` + ConversationIntegrityVerified bool `json:"conversation_integrity_verified"` + OutputPath string `json:"output_path"` + OutputBytes int64 `json:"output_bytes"` + OutputRecords int64 `json:"output_records"` + OutputSHA256 string `json:"output_sha256"` + TimestampRegressions int64 `json:"timestamp_regressions"` + MaximumBufferedBytes int64 `json:"maximum_buffered_bytes"` + OrphanBytes int64 `json:"orphan_bytes,omitempty"` + OrphanLines int64 `json:"orphan_lines,omitempty"` + Storage *storage.MutationAccounting `json:"storage,omitempty"` +} + +type RepairOptions struct { + AllowOrphans bool + OrphanPath string + Context context.Context + Budget storage.Checker +} + +type repairFrame struct { + partial []byte + pending []repairRecord + startedLine int64 +} + +type repairRecord struct { + data []byte + sourceValid bool +} + +type repairWriter struct { + writer io.Writer + stack []repairFrame + bufferedBytes int64 + maximumBuffered int64 + outputRecords int64 + reconstructed int64 + previousTimestamp time.Time + timestampRegressions int64 + orphanWriter *bufio.Writer + allowOrphans bool + orphanBytes int64 + orphanLines int64 + sourceConversationChain conversationChain + preservedConversationChain conversationChain + sourceConversationRecords int64 + preservedConversationRecords int64 + reconstructedConversationRecords int64 +} + +func Repair(sourcePath, outputPath string) (RepairResult, error) { + return RepairWithOptions(sourcePath, outputPath, RepairOptions{}) +} + +func RepairWithOptions(sourcePath, outputPath string, options RepairOptions) (RepairResult, error) { + if outputPath == "" { + return RepairResult{}, errors.New("output path is required") + } + sourceAbs, err := filepath.Abs(sourcePath) + if err != nil { + return RepairResult{}, err + } + outputAbs, err := filepath.Abs(outputPath) + if err != nil { + return RepairResult{}, err + } + if sourceAbs == outputAbs { + return RepairResult{}, errors.New("output must not replace the source") + } + if _, err := os.Lstat(outputAbs); err == nil { + return RepairResult{}, fmt.Errorf("output already exists: %s", outputAbs) + } else if !errors.Is(err, os.ErrNotExist) { + return RepairResult{}, err + } + if options.AllowOrphans { + if options.OrphanPath == "" { + return RepairResult{}, errors.New("orphan path is required when allow orphans is enabled") + } + orphanAbs, err := filepath.Abs(options.OrphanPath) + if err != nil { + return RepairResult{}, err + } + if orphanAbs == sourceAbs || orphanAbs == outputAbs { + return RepairResult{}, errors.New("orphan output must be separate from source and repaired output") + } + if _, err := os.Lstat(orphanAbs); err == nil { + return RepairResult{}, fmt.Errorf("orphan output already exists: %s", orphanAbs) + } else if !errors.Is(err, os.ErrNotExist) { + return RepairResult{}, err + } + options.OrphanPath = orphanAbs + } + + source, err := os.Open(sourceAbs) + if err != nil { + return RepairResult{}, err + } + defer source.Close() + before, err := source.Stat() + if err != nil { + return RepairResult{}, err + } + ctx := options.Context + if ctx == nil { + ctx = context.Background() + } + estimatedBytes, err := estimatedOutputBytes(before.Size()) + if err != nil { + return RepairResult{}, err + } + persistentBytes := estimatedBytes + if options.AllowOrphans { + if persistentBytes > int64(^uint64(0)>>1)-estimatedBytes { + return RepairResult{}, errors.New("repair output byte estimate overflow") + } + persistentBytes += estimatedBytes + } + budget := options.Budget + if budget == nil { + budget = storage.VolumeGuard{Path: filepath.Dir(outputAbs)} + } + storageAssessment, err := budget.Check(ctx, storage.Projection{ + Operation: "repair-rollout", AdditionalPersistentBytes: persistentBytes, + TemporaryBytes: estimatedBytes, TemporaryPersistentOverlapBytes: estimatedBytes, + }) + if err != nil { + return RepairResult{}, err + } + if options.AllowOrphans && options.Budget == nil { + if _, err := (storage.VolumeGuard{Path: filepath.Dir(options.OrphanPath)}).Check(ctx, storage.Projection{Operation: "repair-orphans", AdditionalPersistentBytes: estimatedBytes}); err != nil { + return RepairResult{}, err + } + } + if err := os.MkdirAll(filepath.Dir(outputAbs), 0o700); err != nil { + return RepairResult{}, err + } + temp, err := os.CreateTemp(filepath.Dir(outputAbs), ".codexfold-repair-*.tmp") + if err != nil { + return RepairResult{}, err + } + tempPath := temp.Name() + committed := false + defer func() { + if !committed { + _ = temp.Close() + _ = os.Remove(tempPath) + } + }() + if err := temp.Chmod(0o600); err != nil { + return RepairResult{}, err + } + var orphanFile *os.File + var orphanWriter *bufio.Writer + if options.AllowOrphans { + if err := os.MkdirAll(filepath.Dir(options.OrphanPath), 0o700); err != nil { + return RepairResult{}, err + } + orphanFile, err = os.OpenFile(options.OrphanPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return RepairResult{}, err + } + defer orphanFile.Close() + defer func() { + if !committed { + _ = os.Remove(options.OrphanPath) + } + }() + orphanWriter = bufio.NewWriterSize(orphanFile, 64*1024) + } + + result := RepairResult{SourcePath: sourceAbs, OutputPath: outputAbs} + sourceHasher := sha256.New() + outputHasher := sha256.New() + processor := repairWriter{writer: io.MultiWriter(temp, outputHasher), orphanWriter: orphanWriter, allowOrphans: options.AllowOrphans} + reader := bufio.NewReaderSize(source, 1024*1024) + for { + if err := ctx.Err(); err != nil { + return RepairResult{}, err + } + line, readErr := reader.ReadBytes('\n') + if len(line) > 0 { + result.PhysicalLines++ + result.SourceBytes += int64(len(line)) + _, _ = sourceHasher.Write(line) + line = bytes.TrimSuffix(line, []byte{'\n'}) + line = bytes.TrimSuffix(line, []byte{'\r'}) + if json.Valid(line) { + if err := processor.acceptValid(line); err != nil { + return RepairResult{}, fmt.Errorf("physical line %d: %w", result.PhysicalLines, err) + } + } else { + result.InvalidPhysicalLines++ + if err := processor.acceptFragment(line, result.PhysicalLines); err != nil { + return RepairResult{}, fmt.Errorf("physical line %d: %w", result.PhysicalLines, err) + } + } + } + if errors.Is(readErr, io.EOF) { + break + } + if readErr != nil { + return RepairResult{}, readErr + } + } + if len(processor.stack) != 0 && !options.AllowOrphans { + return RepairResult{}, fmt.Errorf("unresolved interrupted record started at physical line %d", processor.stack[0].startedLine) + } + if len(processor.stack) != 0 { + if err := processor.salvageUnresolved(); err != nil { + return RepairResult{}, err + } + } + if err := processor.verifyConversationIntegrity(); err != nil { + return RepairResult{}, err + } + if processor.timestampRegressions != 0 && !options.AllowOrphans { + return RepairResult{}, fmt.Errorf("repaired record order still has %d timestamp regressions", processor.timestampRegressions) + } + after, err := source.Stat() + if err != nil { + return RepairResult{}, err + } + if before.Size() != after.Size() || !before.ModTime().Equal(after.ModTime()) { + return RepairResult{}, errors.New("source changed while repairing") + } + if err := temp.Sync(); err != nil { + return RepairResult{}, err + } + if err := temp.Close(); err != nil { + return RepairResult{}, err + } + if orphanWriter != nil { + if err := orphanWriter.Flush(); err != nil { + return RepairResult{}, err + } + if err := orphanFile.Sync(); err != nil { + return RepairResult{}, err + } + } + verified, err := scanPath(tempPath, 2) + if err != nil { + return RepairResult{}, fmt.Errorf("verify repaired output: %w", err) + } + if (!options.AllowOrphans && verified.summary.TimestampRegressions != 0) || verified.summary.Records != processor.outputRecords { + return RepairResult{}, errors.New("repaired output verification failed") + } + outputDigest := hex.EncodeToString(outputHasher.Sum(nil)) + if verified.summary.SHA256 != outputDigest { + return RepairResult{}, errors.New("repaired output digest verification failed") + } + if err := os.Rename(tempPath, outputAbs); err != nil { + return RepairResult{}, err + } + committed = true + if err := syncDir(filepath.Dir(outputAbs)); err != nil { + return RepairResult{}, err + } + + result.SourceSHA256 = hex.EncodeToString(sourceHasher.Sum(nil)) + result.ReconstructedRecords = processor.reconstructed + result.SourceConversationRecords = processor.sourceConversationRecords + result.PreservedConversationRecords = processor.preservedConversationRecords + result.ReconstructedConversationRecords = processor.reconstructedConversationRecords + result.ConversationIntegrityVerified = true + result.OutputBytes = verified.summary.Bytes + result.OutputRecords = verified.summary.Records + result.OutputSHA256 = outputDigest + result.TimestampRegressions = verified.summary.TimestampRegressions + result.MaximumBufferedBytes = processor.maximumBuffered + result.OrphanBytes = processor.orphanBytes + result.OrphanLines = processor.orphanLines + result.Storage = storage.CompleteAccounting(ctx, storageAssessment, "") + return result, nil +} + +func (w *repairWriter) acceptValid(record []byte) error { + w.trackSourceConversation(record) + if len(w.stack) == 0 { + return w.writeRecord(repairRecord{data: record, sourceValid: true}) + } + copyOfRecord := append([]byte(nil), record...) + top := &w.stack[len(w.stack)-1] + top.pending = append(top.pending, repairRecord{data: copyOfRecord, sourceValid: true}) + return w.addBuffered(int64(len(copyOfRecord))) +} + +func (w *repairWriter) acceptFragment(line []byte, physicalLine int64) error { + starts := recordStartPattern.FindAllIndex(line, -1) + if len(starts) == 0 { + if len(w.stack) == 0 { + return w.handleOrphan(line) + } + return w.appendFragment(line) + } + + cursor := 0 + for _, match := range starts { + start := match[0] + if start > cursor { + if len(w.stack) == 0 { + if len(bytes.TrimSpace(line[cursor:start])) != 0 { + if err := w.handleOrphan(line[cursor:start]); err != nil { + return err + } + } + } else if err := w.appendFragment(line[cursor:start]); err != nil { + return err + } + } + w.stack = append(w.stack, repairFrame{startedLine: physicalLine}) + cursor = start + } + return w.appendFragment(line[cursor:]) +} + +func (w *repairWriter) appendFragment(fragment []byte) error { + if len(w.stack) == 0 { + return w.handleOrphan(fragment) + } + top := &w.stack[len(w.stack)-1] + top.partial = append(top.partial, fragment...) + if err := w.addBuffered(int64(len(fragment))); err != nil { + return err + } + if json.Valid(top.partial) { + return w.finishTop() + } + return nil +} + +func (w *repairWriter) handleOrphan(fragment []byte) error { + if len(bytes.TrimSpace(fragment)) == 0 { + return nil + } + if !w.allowOrphans { + return errors.New("orphan JSON fragment has no active interrupted record") + } + return w.writeOrphan(fragment) +} + +func (w *repairWriter) writeOrphan(fragment []byte) error { + if w.orphanWriter == nil { + return errors.New("orphan writer is not configured") + } + if containsCompleteConversationRecord(fragment) { + return errors.New("refusing to orphan a complete user or assistant conversation record") + } + if _, err := w.orphanWriter.Write(fragment); err != nil { + return err + } + if err := w.orphanWriter.WriteByte('\n'); err != nil { + return err + } + w.orphanBytes += int64(len(fragment)) + w.orphanLines++ + return nil +} + +func (w *repairWriter) finishTop() error { + index := len(w.stack) - 1 + frame := w.stack[index] + w.stack = w.stack[:index] + w.reconstructed++ + records := make([]repairRecord, 0, 1+len(frame.pending)) + records = append(records, repairRecord{data: frame.partial}) + records = append(records, frame.pending...) + if len(w.stack) != 0 { + parent := &w.stack[len(w.stack)-1] + parent.pending = append(parent.pending, records...) + return nil + } + for _, record := range records { + if err := w.writeRecord(record); err != nil { + return err + } + w.bufferedBytes -= int64(len(record.data)) + } + return nil +} + +func (w *repairWriter) salvageUnresolved() error { + for _, frame := range w.stack { + if err := w.writeOrphan(frame.partial); err != nil { + return err + } + for _, pending := range frame.pending { + if err := w.writeRecord(pending); err != nil { + return err + } + } + } + w.stack = nil + w.bufferedBytes = 0 + return nil +} + +func (w *repairWriter) writeRecord(record repairRecord) error { + if !json.Valid(record.data) { + return errors.New("attempted to emit invalid JSON record") + } + extractor := newTimestampExtractor() + if err := extractor.Write(record.data); err != nil { + return err + } + timestamp, err := extractor.Timestamp() + if err != nil { + return err + } + if !w.previousTimestamp.IsZero() && timestamp.Before(w.previousTimestamp) { + w.timestampRegressions++ + } + w.previousTimestamp = timestamp + if _, err := w.writer.Write(record.data); err != nil { + return err + } + if _, err := w.writer.Write([]byte{'\n'}); err != nil { + return err + } + w.outputRecords++ + w.trackOutputConversation(record) + return nil +} + +func (w *repairWriter) trackSourceConversation(record []byte) { + if conversationRecordKind(record) == "" { + return + } + w.sourceConversationChain = w.sourceConversationChain.append(record) + w.sourceConversationRecords++ +} + +func (w *repairWriter) trackOutputConversation(record repairRecord) { + if conversationRecordKind(record.data) == "" { + return + } + if record.sourceValid { + w.preservedConversationChain = w.preservedConversationChain.append(record.data) + w.preservedConversationRecords++ + return + } + w.reconstructedConversationRecords++ +} + +func (w *repairWriter) verifyConversationIntegrity() error { + if w.sourceConversationRecords != w.preservedConversationRecords || w.sourceConversationChain != w.preservedConversationChain { + return fmt.Errorf("conversation integrity verification failed: source=%d preserved=%d", w.sourceConversationRecords, w.preservedConversationRecords) + } + return nil +} + +func (w *repairWriter) addBuffered(bytes int64) error { + w.bufferedBytes += bytes + if w.bufferedBytes > w.maximumBuffered { + w.maximumBuffered = w.bufferedBytes + } + if w.bufferedBytes > maxRepairBufferedBytes { + return fmt.Errorf("interrupted record buffer exceeded %d bytes", maxRepairBufferedBytes) + } + return nil +} diff --git a/internal/reconcile/repair_test.go b/internal/reconcile/repair_test.go new file mode 100644 index 0000000..816934b --- /dev/null +++ b/internal/reconcile/repair_test.go @@ -0,0 +1,197 @@ +package reconcile + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/samekind/codexfold/internal/storage" +) + +func TestRepairRestoresInterruptedRecordBeforeInsertedRecord(t *testing.T) { + dir := t.TempDir() + outer := recordWithText("2026-07-13T01:00:00Z", "abcdef") + inner := recordWithText("2026-07-13T01:00:01Z", "inner") + prefix, suffix := splitAt(t, outer, "abc") + input := filepath.Join(dir, "broken.jsonl") + if err := os.WriteFile(input, []byte(prefix+inner+"\n"+suffix+"\n"), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "repaired.jsonl") + + result, err := Repair(input, output) + if err != nil { + t.Fatal(err) + } + if result.InvalidPhysicalLines != 2 || result.ReconstructedRecords != 2 || result.OutputRecords != 2 { + t.Fatalf("unexpected result: %#v", result) + } + if result.Storage == nil || result.Storage.Budget.ProjectedPeakBytes <= 0 || result.Storage.ActualReclaimedBytes != 0 { + t.Fatalf("repair storage accounting is incomplete: %#v", result.Storage) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if string(data) != outer+"\n"+inner+"\n" { + t.Fatalf("repaired bytes:\n%s", data) + } +} + +func TestRepairBudgetRejectsBeforeCreatingOutput(t *testing.T) { + dir := t.TempDir() + input := filepath.Join(dir, "broken.jsonl") + if err := os.WriteFile(input, []byte(recordWithText("2026-07-13T01:00:00Z", "value")+"\n"), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "output", "repaired.jsonl") + checker := &reconcileRejectingChecker{} + if _, err := RepairWithOptions(input, output, RepairOptions{Budget: checker}); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("RepairWithOptions error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 || checker.Projection.Operation != "repair-rollout" { + t.Fatalf("unexpected repair budget projection: %#v", checker) + } + if _, err := os.Stat(filepath.Dir(output)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("repair output directory exists after preflight rejection: %v", err) + } +} + +func TestRepairBuffersValidPhysicalRecordsWhileOuterRecordIsOpen(t *testing.T) { + dir := t.TempDir() + outer := recordWithText("2026-07-13T01:00:00Z", "abcdef") + inner := recordWithText("2026-07-13T01:00:01Z", "inner") + prefix, suffix := splitAt(t, outer, "abc") + input := filepath.Join(dir, "broken.jsonl") + if err := os.WriteFile(input, []byte(prefix+"\n"+inner+"\n"+suffix+"\n"), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "repaired.jsonl") + + if _, err := Repair(input, output); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if string(data) != outer+"\n"+inner+"\n" { + t.Fatalf("repaired bytes:\n%s", data) + } +} + +func TestRepairRestoresNestedInterruptions(t *testing.T) { + dir := t.TempDir() + outer := recordWithText("2026-07-13T01:00:00Z", "abcdef") + middle := recordWithText("2026-07-13T01:00:01Z", "ghijkl") + inner := recordWithText("2026-07-13T01:00:02Z", "inner") + outerPrefix, outerSuffix := splitAt(t, outer, "abc") + middlePrefix, middleSuffix := splitAt(t, middle, "ghi") + input := filepath.Join(dir, "broken.jsonl") + physical := outerPrefix + middlePrefix + inner + "\n" + middleSuffix + "\n" + outerSuffix + "\n" + if err := os.WriteFile(input, []byte(physical), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "repaired.jsonl") + + if _, err := Repair(input, output); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if string(data) != outer+"\n"+middle+"\n"+inner+"\n" { + t.Fatalf("repaired bytes:\n%s", data) + } +} + +func TestRepairSalvageKeepsValidRecordsAfterUnfinishedFragment(t *testing.T) { + dir := t.TempDir() + before := recordWithText("2026-07-13T01:00:00Z", "before") + unfinished := `{"timestamp":"2026-07-13T01:00:01Z","type":"event_msg","payload":{"text":"unfinished` + afterOne := recordWithText("2026-07-13T01:00:02Z", "after-one") + afterTwo := recordWithText("2026-07-13T01:00:03Z", "after-two") + input := filepath.Join(dir, "broken.jsonl") + physical := before + "\n" + unfinished + "\n" + afterOne + "\n" + afterTwo + "\n" + if err := os.WriteFile(input, []byte(physical), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "repaired.jsonl") + orphans := filepath.Join(dir, "orphans.bin") + + result, err := RepairWithOptions(input, output, RepairOptions{AllowOrphans: true, OrphanPath: orphans}) + if err != nil { + t.Fatal(err) + } + if result.OutputRecords != 3 || result.OrphanLines != 1 || result.OrphanBytes != int64(len(unfinished)) { + t.Fatalf("unexpected salvage result: %#v", result) + } + repaired, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if string(repaired) != before+"\n"+afterOne+"\n"+afterTwo+"\n" { + t.Fatalf("salvaged bytes:\n%s", repaired) + } + orphaned, err := os.ReadFile(orphans) + if err != nil { + t.Fatal(err) + } + if string(orphaned) != unfinished+"\n" { + t.Fatalf("orphan bytes: %q", orphaned) + } +} + +func TestRepairSalvageKeepsValidRecordsAcrossNestedUnfinishedFragments(t *testing.T) { + dir := t.TempDir() + outer := `{"timestamp":"2026-07-13T01:00:00Z","type":"event_msg","payload":{"text":"outer` + between := recordWithText("2026-07-13T01:00:01Z", "between") + inner := `{"timestamp":"2026-07-13T01:00:02Z","type":"event_msg","payload":{"text":"inner` + after := recordWithText("2026-07-13T01:00:03Z", "after") + input := filepath.Join(dir, "broken.jsonl") + physical := outer + "\n" + between + "\n" + inner + "\n" + after + "\n" + if err := os.WriteFile(input, []byte(physical), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(dir, "repaired.jsonl") + orphans := filepath.Join(dir, "orphans.bin") + + result, err := RepairWithOptions(input, output, RepairOptions{AllowOrphans: true, OrphanPath: orphans}) + if err != nil { + t.Fatal(err) + } + if result.OutputRecords != 2 || result.OrphanLines != 2 { + t.Fatalf("unexpected nested salvage result: %#v", result) + } + repaired, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if string(repaired) != between+"\n"+after+"\n" { + t.Fatalf("nested salvaged bytes:\n%s", repaired) + } + orphaned, err := os.ReadFile(orphans) + if err != nil { + t.Fatal(err) + } + if string(orphaned) != outer+"\n"+inner+"\n" { + t.Fatalf("nested orphan bytes: %q", orphaned) + } +} + +func recordWithText(timestamp, text string) string { + return `{"timestamp":"` + timestamp + `","type":"event_msg","payload":{"text":"` + text + `"}}` +} + +func splitAt(t *testing.T, value, marker string) (string, string) { + t.Helper() + index := strings.Index(value, marker) + if index < 0 { + t.Fatalf("marker %q not found", marker) + } + index += len(marker) + return value[:index], value[index:] +} diff --git a/internal/reconcile/semantic.go b/internal/reconcile/semantic.go new file mode 100644 index 0000000..3a28c4f --- /dev/null +++ b/internal/reconcile/semantic.go @@ -0,0 +1,63 @@ +package reconcile + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "regexp" +) + +var conversationRecordStartPattern = regexp.MustCompile(`\{\s*"timestamp"\s*:`) + +type conversationChain [sha256.Size]byte + +func (chain conversationChain) append(record []byte) conversationChain { + recordDigest := sha256.Sum256(record) + var input [sha256.Size * 2]byte + copy(input[:sha256.Size], chain[:]) + copy(input[sha256.Size:], recordDigest[:]) + return sha256.Sum256(input[:]) +} + +func conversationRecordKind(record []byte) string { + if !bytes.Contains(record, []byte(`"type"`)) || !bytes.Contains(record, []byte(`"payload"`)) { + return "" + } + var envelope struct { + Timestamp json.RawMessage `json:"timestamp"` + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` + } + if err := json.Unmarshal(record, &envelope); err != nil || len(envelope.Timestamp) == 0 { + return "" + } + var payload struct { + Type string `json:"type"` + Role string `json:"role"` + } + if err := json.Unmarshal(envelope.Payload, &payload); err != nil { + return "" + } + switch envelope.Type { + case "response_item": + if payload.Type == "message" && (payload.Role == "user" || payload.Role == "assistant") { + return envelope.Type + ":" + payload.Role + } + case "event_msg": + if payload.Type == "user_message" || payload.Type == "agent_message" { + return envelope.Type + ":" + payload.Type + } + } + return "" +} + +func containsCompleteConversationRecord(fragment []byte) bool { + for _, match := range conversationRecordStartPattern.FindAllIndex(fragment, -1) { + decoder := json.NewDecoder(bytes.NewReader(fragment[match[0]:])) + var record json.RawMessage + if err := decoder.Decode(&record); err == nil && conversationRecordKind(record) != "" { + return true + } + } + return false +} diff --git a/internal/reconcile/semantic_test.go b/internal/reconcile/semantic_test.go new file mode 100644 index 0000000..ddeb150 --- /dev/null +++ b/internal/reconcile/semantic_test.go @@ -0,0 +1,71 @@ +package reconcile + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRepairVerifiesConversationRecordsAcrossSalvage(t *testing.T) { + dir := t.TempDir() + user := conversationRecord("2026-07-16T00:00:00Z", "response_item", "message", "user") + unfinished := `{"timestamp":"2026-07-16T00:00:01Z","type":"event_msg","payload":{"text":"unfinished` + agent := conversationRecord("2026-07-16T00:00:02Z", "event_msg", "agent_message", "") + source := filepath.Join(dir, "source.jsonl") + if err := os.WriteFile(source, []byte(user+"\n"+unfinished+"\n"+agent+"\n"), 0o600); err != nil { + t.Fatal(err) + } + result, err := RepairWithOptions(source, filepath.Join(dir, "repaired.jsonl"), RepairOptions{ + AllowOrphans: true, + OrphanPath: filepath.Join(dir, "orphans.bin"), + }) + if err != nil { + t.Fatal(err) + } + if !result.ConversationIntegrityVerified || result.SourceConversationRecords != 2 || result.PreservedConversationRecords != 2 || result.ReconstructedConversationRecords != 0 { + t.Fatalf("unexpected conversation verification: %#v", result) + } +} + +func TestRepairCountsReconstructedConversationRecord(t *testing.T) { + dir := t.TempDir() + record := conversationRecord("2026-07-16T00:00:00Z", "response_item", "message", "assistant") + prefix, suffix := splitAt(t, record, `"role"`) + source := filepath.Join(dir, "source.jsonl") + if err := os.WriteFile(source, []byte(prefix+"\n"+suffix+"\n"), 0o600); err != nil { + t.Fatal(err) + } + result, err := Repair(source, filepath.Join(dir, "repaired.jsonl")) + if err != nil { + t.Fatal(err) + } + if !result.ConversationIntegrityVerified || result.SourceConversationRecords != 0 || result.PreservedConversationRecords != 0 || result.ReconstructedConversationRecords != 1 { + t.Fatalf("unexpected reconstructed conversation verification: %#v", result) + } +} + +func TestRepairRefusesCompleteConversationRecordInOrphan(t *testing.T) { + dir := t.TempDir() + record := conversationRecord("2026-07-16T00:00:00Z", "event_msg", "user_message", "") + source := filepath.Join(dir, "source.jsonl") + if err := os.WriteFile(source, []byte("garbage"+record+"trailing\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err := RepairWithOptions(source, filepath.Join(dir, "repaired.jsonl"), RepairOptions{ + AllowOrphans: true, + OrphanPath: filepath.Join(dir, "orphans.bin"), + }) + if err == nil || !strings.Contains(err.Error(), "refusing to orphan") { + t.Fatalf("RepairWithOptions error = %v, want complete conversation refusal", err) + } +} + +func conversationRecord(timestamp, entryType, payloadType, role string) string { + payload := `{"type":"` + payloadType + `"` + if role != "" { + payload += `,"role":"` + role + `"` + } + payload += `}` + return `{"timestamp":"` + timestamp + `","type":"` + entryType + `","payload":` + payload + `}` +} diff --git a/internal/scan/evaluate.go b/internal/scan/evaluate.go index 1c99917..a65db72 100644 --- a/internal/scan/evaluate.go +++ b/internal/scan/evaluate.go @@ -14,7 +14,7 @@ import ( "strings" "time" - "github.com/jstar0/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/codex" ) const ( diff --git a/internal/scan/evaluate_test.go b/internal/scan/evaluate_test.go index b4572b6..b3eb8f1 100644 --- a/internal/scan/evaluate_test.go +++ b/internal/scan/evaluate_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/jstar0/codexfold/internal/codex" + "github.com/samekind/codexfold/internal/codex" ) func TestIncrementalEvaluateSkipsUnchangedFileWithoutDoubleCounting(t *testing.T) { diff --git a/internal/scan/jsonl.go b/internal/scan/jsonl.go index e5e2c81..08e4911 100644 --- a/internal/scan/jsonl.go +++ b/internal/scan/jsonl.go @@ -9,8 +9,8 @@ import ( "hash" "io" - "github.com/jstar0/codexfold/internal/cdc" - "github.com/jstar0/codexfold/internal/jsonraw" + "github.com/samekind/codexfold/internal/cdc" + "github.com/samekind/codexfold/internal/jsonraw" ) const ( diff --git a/internal/service/binary_update.go b/internal/service/binary_update.go new file mode 100644 index 0000000..9de9cfe --- /dev/null +++ b/internal/service/binary_update.go @@ -0,0 +1,168 @@ +package service + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/samekind/codexfold/internal/buildid" +) + +type BinaryUpdate struct { + Target string `json:"target"` + Candidate string `json:"candidate"` + CurrentSHA256 string `json:"current_sha256"` + CandidateSHA256 string `json:"candidate_sha256"` + stagedPath string + backupPath string +} + +func StageBinaryUpdate(candidate string, target string) (*BinaryUpdate, error) { + if !filepath.IsAbs(candidate) || !filepath.IsAbs(target) { + return nil, errors.New("absolute candidate and target binary paths are required") + } + candidate = filepath.Clean(candidate) + target = filepath.Clean(target) + if candidate == target { + return nil, errors.New("candidate binary must be separate from the installed target") + } + targetInfo, err := os.Stat(target) + if err != nil { + return nil, err + } + if !targetInfo.Mode().IsRegular() { + return nil, errors.New("installed service binary is not a regular file") + } + candidateInfo, err := os.Stat(candidate) + if err != nil { + return nil, err + } + if !candidateInfo.Mode().IsRegular() || candidateInfo.Mode().Perm()&0o111 == 0 { + return nil, errors.New("candidate service binary must be a regular executable file") + } + currentSHA256, err := buildid.FileSHA256(target) + if err != nil { + return nil, err + } + candidateSHA256, err := buildid.FileSHA256(candidate) + if err != nil { + return nil, err + } + root := filepath.Dir(target) + stagedPath, err := copyBinaryTemporary(candidate, root, ".codexfold-candidate-*", targetInfo.Mode().Perm()) + if err != nil { + return nil, err + } + backupPath, err := copyBinaryTemporary(target, root, ".codexfold-backup-*", targetInfo.Mode().Perm()) + if err != nil { + _ = os.Remove(stagedPath) + return nil, err + } + if err := syncServiceDirectory(root); err != nil { + _ = os.Remove(stagedPath) + _ = os.Remove(backupPath) + return nil, err + } + return &BinaryUpdate{ + Target: target, Candidate: candidate, CurrentSHA256: currentSHA256, CandidateSHA256: candidateSHA256, + stagedPath: stagedPath, backupPath: backupPath, + }, nil +} + +func (u *BinaryUpdate) Promote() error { + if u == nil || u.stagedPath == "" || u.Target == "" { + return errors.New("staged binary update is required") + } + if err := replaceServiceBinary(u.stagedPath, u.Target); err != nil { + return err + } + u.stagedPath = "" + if err := syncServiceDirectory(filepath.Dir(u.Target)); err != nil { + return err + } + digest, err := buildid.FileSHA256(u.Target) + if err != nil { + return err + } + if digest != u.CandidateSHA256 { + return fmt.Errorf("promoted binary digest=%s expected=%s", digest, u.CandidateSHA256) + } + return nil +} + +func (u *BinaryUpdate) Rollback() error { + if u == nil || u.backupPath == "" || u.Target == "" { + return errors.New("binary update backup is unavailable") + } + if err := replaceServiceBinary(u.backupPath, u.Target); err != nil { + return err + } + u.backupPath = "" + if err := syncServiceDirectory(filepath.Dir(u.Target)); err != nil { + return err + } + digest, err := buildid.FileSHA256(u.Target) + if err != nil { + return err + } + if digest != u.CurrentSHA256 { + return fmt.Errorf("rolled back binary digest=%s expected=%s", digest, u.CurrentSHA256) + } + return nil +} + +func (u *BinaryUpdate) Commit() error { + if u == nil { + return nil + } + var result error + for _, path := range []string{u.stagedPath, u.backupPath} { + if path == "" { + continue + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + result = errors.Join(result, err) + } + } + u.stagedPath = "" + u.backupPath = "" + return errors.Join(result, syncServiceDirectory(filepath.Dir(u.Target))) +} + +func copyBinaryTemporary(source string, directory string, pattern string, mode os.FileMode) (string, error) { + input, err := os.Open(source) + if err != nil { + return "", err + } + temporary, err := os.CreateTemp(directory, pattern) + if err != nil { + _ = input.Close() + return "", err + } + path := temporary.Name() + cleanup := func(operationErr error) (string, error) { + _ = input.Close() + _ = temporary.Close() + _ = os.Remove(path) + return "", operationErr + } + if err := temporary.Chmod(mode); err != nil { + return cleanup(err) + } + if _, err := io.Copy(temporary, input); err != nil { + return cleanup(err) + } + if err := input.Close(); err != nil { + return cleanup(err) + } + if err := temporary.Sync(); err != nil { + return cleanup(err) + } + if err := temporary.Close(); err != nil { + _ = os.Remove(path) + return "", err + } + return path, nil +} diff --git a/internal/service/binary_update_test.go b/internal/service/binary_update_test.go new file mode 100644 index 0000000..a2fd778 --- /dev/null +++ b/internal/service/binary_update_test.go @@ -0,0 +1,77 @@ +package service + +import ( + "os" + "path/filepath" + "testing" + + "github.com/samekind/codexfold/internal/buildid" +) + +func TestBinaryUpdatePromotesAndCommitsAtomically(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "codexfold") + candidate := filepath.Join(root, "candidate") + if err := os.WriteFile(target, []byte("old-build"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(candidate, []byte("new-build"), 0o700); err != nil { + t.Fatal(err) + } + update, err := StageBinaryUpdate(candidate, target) + if err != nil { + t.Fatal(err) + } + if err := update.Promote(); err != nil { + t.Fatal(err) + } + if digest, err := buildid.FileSHA256(target); err != nil || digest != update.CandidateSHA256 { + t.Fatalf("promoted digest=%s err=%v", digest, err) + } + if err := update.Commit(); err != nil { + t.Fatal(err) + } + assertNoBinaryUpdateArtifacts(t, root) +} + +func TestBinaryUpdateRollsBackPromotedCandidate(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "codexfold") + candidate := filepath.Join(root, "candidate") + if err := os.WriteFile(target, []byte("old-build"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(candidate, []byte("new-build"), 0o700); err != nil { + t.Fatal(err) + } + update, err := StageBinaryUpdate(candidate, target) + if err != nil { + t.Fatal(err) + } + if err := update.Promote(); err != nil { + t.Fatal(err) + } + if err := update.Rollback(); err != nil { + t.Fatal(err) + } + if digest, err := buildid.FileSHA256(target); err != nil || digest != update.CurrentSHA256 { + t.Fatalf("rolled back digest=%s err=%v", digest, err) + } + if err := update.Commit(); err != nil { + t.Fatal(err) + } + assertNoBinaryUpdateArtifacts(t, root) +} + +func assertNoBinaryUpdateArtifacts(t *testing.T, root string) { + t.Helper() + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if len(entry.Name()) >= len(".codexfold-") && entry.Name()[:len(".codexfold-")] == ".codexfold-" { + t.Fatalf("binary update artifact remained: %s", entry.Name()) + } + } +} diff --git a/internal/service/binary_update_unix.go b/internal/service/binary_update_unix.go new file mode 100644 index 0000000..a8a05bb --- /dev/null +++ b/internal/service/binary_update_unix.go @@ -0,0 +1,22 @@ +//go:build !windows + +package service + +import ( + "errors" + "os" +) + +func replaceServiceBinary(source string, target string) error { + return os.Rename(source, target) +} + +func syncServiceDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + syncErr := directory.Sync() + closeErr := directory.Close() + return errors.Join(syncErr, closeErr) +} diff --git a/internal/service/binary_update_windows.go b/internal/service/binary_update_windows.go new file mode 100644 index 0000000..0f91149 --- /dev/null +++ b/internal/service/binary_update_windows.go @@ -0,0 +1,21 @@ +//go:build windows + +package service + +import ( + "golang.org/x/sys/windows" +) + +func replaceServiceBinary(source string, target string) error { + sourcePath, err := windows.UTF16PtrFromString(source) + if err != nil { + return err + } + targetPath, err := windows.UTF16PtrFromString(target) + if err != nil { + return err + } + return windows.MoveFileEx(sourcePath, targetPath, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) +} + +func syncServiceDirectory(string) error { return nil } diff --git a/internal/service/build_status.go b/internal/service/build_status.go new file mode 100644 index 0000000..52c75bc --- /dev/null +++ b/internal/service/build_status.go @@ -0,0 +1,351 @@ +package service + +import ( + "encoding/xml" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/samekind/codexfold/internal/buildid" + "github.com/samekind/codexfold/internal/mountid" +) + +type BuildStatus struct { + Healthy bool `json:"healthy"` + RunningBuildSHA256 string `json:"running_build_sha256,omitempty"` + ConfiguredBinaryPath string `json:"configured_binary_path,omitempty"` + ConfiguredBuildSHA256 string `json:"configured_build_sha256,omitempty"` + Error string `json:"error,omitempty"` +} + +func InspectBuild(platform Platform, definitionPath string, mountPoint string) BuildStatus { + status := BuildStatus{} + binaryPath, err := DefinitionBinary(platform, definitionPath) + if err != nil { + status.Error = err.Error() + return status + } + status.ConfiguredBinaryPath = binaryPath + status.ConfiguredBuildSHA256, err = buildid.FileSHA256(binaryPath) + if err != nil { + status.Error = err.Error() + return status + } + identityBytes, err := os.ReadFile(filepath.Join(mountPoint, mountid.Path)) + if err != nil { + status.Error = fmt.Sprintf("read running mount build identity: %v", err) + return status + } + identity, err := mountid.Parse(identityBytes) + if err != nil { + status.Error = err.Error() + return status + } + status.RunningBuildSHA256 = identity.BuildSHA256 + if status.RunningBuildSHA256 == "" { + status.Error = "running mount identity does not include a build SHA-256" + return status + } + if status.RunningBuildSHA256 != status.ConfiguredBuildSHA256 { + status.Error = "running daemon build does not match the configured binary on disk" + return status + } + status.Healthy = true + return status +} + +func DefinitionBinary(platform Platform, definitionPath string) (string, error) { + if !filepath.IsAbs(definitionPath) { + return "", errors.New("absolute service definition path is required") + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return "", err + } + var binary string + switch platform { + case PlatformLaunchd: + binary, err = launchdDefinitionBinary(definition) + case PlatformSystemd: + binary, err = systemdDefinitionBinary(definition) + case PlatformWindows: + var config WindowsConfig + config, err = ParseWindowsConfig(definition) + binary = config.BinaryPath + default: + err = errors.New("unknown service platform") + } + if err != nil { + return "", err + } + if !filepath.IsAbs(binary) && !absoluteWindowsServicePath(binary) { + return "", errors.New("configured service binary path is not absolute") + } + return filepath.Clean(binary), nil +} + +func DefinitionLauncher(platform Platform, definitionPath string) (string, error) { + if !filepath.IsAbs(definitionPath) { + return "", errors.New("absolute service definition path is required") + } + if platform != PlatformLaunchd { + return "", nil + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return "", err + } + arguments, err := launchdDefinitionArguments(definition) + if err != nil { + return "", err + } + if len(arguments) < 3 || arguments[1] != "--run-helper" { + return "", nil + } + if !filepath.IsAbs(arguments[0]) { + return "", errors.New("configured service launcher path is not absolute") + } + return filepath.Clean(arguments[0]), nil +} + +func DefinitionFrontend(platform Platform, definitionPath string) (string, error) { + if !filepath.IsAbs(definitionPath) { + return "", errors.New("absolute service definition path is required") + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return "", err + } + if platform != PlatformLaunchd { + return "fuse", nil + } + arguments, err := launchdDefinitionArguments(definition) + if err != nil { + return "", err + } + for index := 0; index < len(arguments); index++ { + if arguments[index] != "--frontend" { + continue + } + if index+1 >= len(arguments) { + return "", errors.New("launchd definition has an incomplete --frontend argument") + } + if arguments[index+1] != "fuse" && arguments[index+1] != "native-fskit" { + return "", fmt.Errorf("launchd definition has unsupported frontend %q", arguments[index+1]) + } + return arguments[index+1], nil + } + return "fuse", nil +} + +func DefinitionFSKitResource(platform Platform, definitionPath string) (string, error) { + frontend, err := DefinitionFrontend(platform, definitionPath) + if err != nil { + return "", err + } + if frontend != "native-fskit" { + return "", nil + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return "", err + } + arguments, err := launchdDefinitionArguments(definition) + if err != nil { + return "", err + } + for index := 0; index < len(arguments); index++ { + if arguments[index] != "--fskit-resource" { + continue + } + if index+1 >= len(arguments) || !filepath.IsAbs(arguments[index+1]) { + return "", errors.New("launchd definition has an invalid --fskit-resource argument") + } + return filepath.Clean(arguments[index+1]), nil + } + return "", errors.New("native-fskit launchd definition has no --fskit-resource argument") +} + +func DefinitionStore(platform Platform, definitionPath string) (string, error) { + if !filepath.IsAbs(definitionPath) { + return "", errors.New("absolute service definition path is required") + } + if platform != PlatformLaunchd { + return "", errors.New("service store inspection is currently available only for launchd definitions") + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return "", err + } + arguments, err := launchdDefinitionArguments(definition) + if err != nil { + return "", err + } + for index := 0; index < len(arguments); index++ { + if arguments[index] != "--store" { + continue + } + if index+1 >= len(arguments) || !filepath.IsAbs(arguments[index+1]) { + return "", errors.New("launchd definition has an invalid --store argument") + } + return filepath.Clean(arguments[index+1]), nil + } + return "", errors.New("launchd definition has no --store argument") +} + +func DefinitionLabel(platform Platform, definitionPath string) (string, error) { + if !filepath.IsAbs(definitionPath) { + return "", errors.New("absolute service definition path is required") + } + definition, err := os.ReadFile(filepath.Clean(definitionPath)) + if err != nil { + return "", err + } + var label string + switch platform { + case PlatformLaunchd: + label, err = launchdDefinitionLabel(definition) + case PlatformSystemd: + label = strings.TrimSuffix(filepath.Base(definitionPath), ".service") + case PlatformWindows: + var config WindowsConfig + config, err = ParseWindowsConfig(definition) + label = config.ServiceName + default: + err = errors.New("unknown service platform") + } + if err != nil { + return "", err + } + if !safeLabel(label) { + return "", errors.New("configured service label is invalid") + } + return label, nil +} + +func launchdDefinitionBinary(definition []byte) (string, error) { + arguments, err := launchdDefinitionArguments(definition) + if err != nil { + return "", err + } + if len(arguments) == 0 { + return "", errors.New("launchd definition has no ProgramArguments binary") + } + if len(arguments) >= 3 && arguments[1] == "--run-helper" { + return arguments[2], nil + } + return arguments[0], nil +} + +func launchdDefinitionArguments(definition []byte) ([]string, error) { + decoder := xml.NewDecoder(strings.NewReader(string(definition))) + wantArguments := false + inArguments := false + arguments := make([]string, 0, 16) + for { + token, err := decoder.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, err + } + switch element := token.(type) { + case xml.StartElement: + switch element.Name.Local { + case "key": + var key string + if err := decoder.DecodeElement(&key, &element); err != nil { + return nil, err + } + wantArguments = key == "ProgramArguments" + case "array": + if wantArguments { + inArguments = true + wantArguments = false + } + case "string": + if inArguments { + var argument string + if err := decoder.DecodeElement(&argument, &element); err != nil { + return nil, err + } + arguments = append(arguments, argument) + } + } + case xml.EndElement: + if element.Name.Local == "array" && inArguments { + return arguments, nil + } + } + } + return nil, errors.New("launchd definition has no ProgramArguments array") +} + +func launchdDefinitionLabel(definition []byte) (string, error) { + decoder := xml.NewDecoder(strings.NewReader(string(definition))) + wantLabel := false + for { + token, err := decoder.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return "", err + } + start, ok := token.(xml.StartElement) + if !ok { + continue + } + switch start.Name.Local { + case "key": + var key string + if err := decoder.DecodeElement(&key, &start); err != nil { + return "", err + } + wantLabel = key == "Label" + case "string": + if wantLabel { + var label string + if err := decoder.DecodeElement(&label, &start); err != nil { + return "", err + } + return label, nil + } + } + } + return "", errors.New("launchd definition has no Label") +} + +func systemdDefinitionBinary(definition []byte) (string, error) { + for _, line := range strings.Split(string(definition), "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "ExecStart=:") { + continue + } + value := strings.TrimSpace(strings.TrimPrefix(line, "ExecStart=:")) + if len(value) < 2 || value[0] != '"' { + return "", errors.New("systemd ExecStart binary is not quoted") + } + var binary strings.Builder + for index := 1; index < len(value); index++ { + switch value[index] { + case '"': + return strings.ReplaceAll(binary.String(), "%%", "%"), nil + case '\\': + index++ + if index >= len(value) { + return "", errors.New("systemd ExecStart binary has an incomplete escape") + } + binary.WriteByte(value[index]) + default: + binary.WriteByte(value[index]) + } + } + return "", errors.New("systemd ExecStart binary is missing its closing quote") + } + return "", errors.New("systemd definition has no ExecStart binary") +} diff --git a/internal/service/build_status_test.go b/internal/service/build_status_test.go new file mode 100644 index 0000000..f71493a --- /dev/null +++ b/internal/service/build_status_test.go @@ -0,0 +1,154 @@ +package service + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/samekind/codexfold/internal/buildid" + "github.com/samekind/codexfold/internal/mountid" +) + +func TestInspectBuildMatchesRunningMountAndConfiguredBinary(t *testing.T) { + root := t.TempDir() + binary := filepath.Join(root, "codexfold") + if err := os.WriteFile(binary, []byte("candidate-binary"), 0o700); err != nil { + t.Fatal(err) + } + definition := filepath.Join(root, "com.codexfold.fs.plist") + plist, err := RenderLaunchd(Options{ + Label: "com.codexfold.fs", BinaryPath: binary, CodexHome: filepath.Join(root, "home"), + StoreDir: filepath.Join(root, "store"), MountPoint: filepath.Join(root, "mount"), + StdoutPath: filepath.Join(root, "stdout.log"), StderrPath: filepath.Join(root, "stderr.log"), + }) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(definition, plist, 0o600); err != nil { + t.Fatal(err) + } + mount := filepath.Join(root, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + digest, err := buildid.FileSHA256(binary) + if err != nil { + t.Fatal(err) + } + identity, err := mountid.New(digest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(mount, mountid.Path), []byte(identity), 0o600); err != nil { + t.Fatal(err) + } + status := InspectBuild(PlatformLaunchd, definition, mount) + if !status.Healthy || status.RunningBuildSHA256 != digest || status.ConfiguredBuildSHA256 != digest || status.ConfiguredBinaryPath != binary { + t.Fatalf("build status = %#v", status) + } +} + +func TestInspectBuildRejectsStaleRunningDaemon(t *testing.T) { + root := t.TempDir() + binary := filepath.Join(root, "codexfold") + if err := os.WriteFile(binary, []byte("new-binary"), 0o700); err != nil { + t.Fatal(err) + } + definition := filepath.Join(root, "com.codexfold.fs.plist") + plist, err := RenderLaunchd(Options{ + Label: "com.codexfold.fs", BinaryPath: binary, CodexHome: filepath.Join(root, "home"), + StoreDir: filepath.Join(root, "store"), MountPoint: filepath.Join(root, "mount"), + StdoutPath: filepath.Join(root, "stdout.log"), StderrPath: filepath.Join(root, "stderr.log"), + }) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(definition, plist, 0o600); err != nil { + t.Fatal(err) + } + mount := filepath.Join(root, "mount") + if err := os.MkdirAll(mount, 0o700); err != nil { + t.Fatal(err) + } + identity, err := mountid.New(strings.Repeat("a", 64)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(mount, mountid.Path), []byte(identity), 0o600); err != nil { + t.Fatal(err) + } + status := InspectBuild(PlatformLaunchd, definition, mount) + if status.Healthy || !strings.Contains(status.Error, "does not match") { + t.Fatalf("stale build status = %#v", status) + } +} + +func TestDefinitionBinaryParsesEveryRenderedPlatform(t *testing.T) { + root := t.TempDir() + options := Options{ + Label: "com.codexfold.fs", BinaryPath: filepath.Join(root, "Codex Fold"), + CodexHome: filepath.Join(root, "home"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "stdout.log"), + StderrPath: filepath.Join(root, "stderr.log"), + } + for _, platform := range []Platform{PlatformLaunchd, PlatformSystemd, PlatformWindows} { + definition, err := RenderDefinition(platform, options) + if err != nil { + t.Fatalf("render %s: %v", platform, err) + } + path := filepath.Join(root, string(platform)+".definition") + if err := os.WriteFile(path, definition, 0o600); err != nil { + t.Fatal(err) + } + binary, err := DefinitionBinary(platform, path) + if err != nil || binary != options.BinaryPath { + t.Fatalf("definition binary %s = %q err=%v", platform, binary, err) + } + } +} + +func TestDefinitionFrontendParsesNativeFSKitLaunchdArguments(t *testing.T) { + root := t.TempDir() + resource := filepath.Join(root, "store", "fs", "native-fskit.resource") + launcher := filepath.Join(root, "CodexFoldFSKit.app", "Contents", "MacOS", "CodexFoldFSKit") + binary := filepath.Join(root, "codexfold") + definition, err := RenderLaunchd(Options{ + Label: "com.codexfold.fs", BinaryPath: binary, LauncherPath: launcher, + CodexHome: filepath.Join(root, "home"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "stdout.log"), + StderrPath: filepath.Join(root, "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), Frontend: "native-fskit", FSKitResource: resource, + }) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(root, "com.codexfold.fs.plist") + if err := os.WriteFile(path, definition, 0o600); err != nil { + t.Fatal(err) + } + frontend, err := DefinitionFrontend(PlatformLaunchd, path) + if err != nil || frontend != "native-fskit" { + t.Fatalf("frontend = %q err=%v", frontend, err) + } + gotResource, err := DefinitionFSKitResource(PlatformLaunchd, path) + if err != nil || gotResource != resource { + t.Fatalf("resource = %q err=%v", gotResource, err) + } + gotStore, err := DefinitionStore(PlatformLaunchd, path) + if err != nil || gotStore != filepath.Join(root, "store") { + t.Fatalf("store = %q err=%v", gotStore, err) + } + label, err := DefinitionLabel(PlatformLaunchd, path) + if err != nil || label != "com.codexfold.fs" { + t.Fatalf("label = %q err=%v", label, err) + } + configuredBinary, err := DefinitionBinary(PlatformLaunchd, path) + if err != nil || configuredBinary != binary { + t.Fatalf("wrapped definition binary = %q err=%v", configuredBinary, err) + } + configuredLauncher, err := DefinitionLauncher(PlatformLaunchd, path) + if err != nil || configuredLauncher != launcher { + t.Fatalf("wrapped definition launcher = %q err=%v", configuredLauncher, err) + } +} diff --git a/internal/service/definition_update.go b/internal/service/definition_update.go new file mode 100644 index 0000000..c580f6d --- /dev/null +++ b/internal/service/definition_update.go @@ -0,0 +1,150 @@ +package service + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/samekind/codexfold/internal/buildid" +) + +type DefinitionUpdate struct { + Target string `json:"target"` + CurrentSHA256 string `json:"current_sha256,omitempty"` + CandidateSHA256 string `json:"candidate_sha256"` + HadTarget bool `json:"had_target"` + stagedPath string + backupPath string +} + +func StageDefinitionUpdate(target string, definition []byte) (*DefinitionUpdate, error) { + if !filepath.IsAbs(target) || len(definition) == 0 { + return nil, errors.New("absolute definition target and non-empty content are required") + } + target = filepath.Clean(target) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return nil, err + } + temporary, err := os.CreateTemp(filepath.Dir(target), ".codexfold-definition-candidate-*") + if err != nil { + return nil, err + } + stagedPath := temporary.Name() + cleanup := func(operationErr error) (*DefinitionUpdate, error) { + _ = temporary.Close() + _ = os.Remove(stagedPath) + return nil, operationErr + } + if err := temporary.Chmod(0o600); err != nil { + return cleanup(err) + } + if _, err := temporary.Write(definition); err != nil { + return cleanup(err) + } + if err := temporary.Sync(); err != nil { + return cleanup(err) + } + if err := temporary.Close(); err != nil { + _ = os.Remove(stagedPath) + return nil, err + } + digest := sha256.Sum256(definition) + update := &DefinitionUpdate{Target: target, CandidateSHA256: hex.EncodeToString(digest[:]), stagedPath: stagedPath} + if info, err := os.Stat(target); err == nil { + if !info.Mode().IsRegular() { + _ = update.Commit() + return nil, errors.New("installed service definition is not a regular file") + } + update.HadTarget = true + update.CurrentSHA256, err = buildid.FileSHA256(target) + if err != nil { + _ = update.Commit() + return nil, err + } + update.backupPath, err = copyBinaryTemporary(target, filepath.Dir(target), ".codexfold-definition-backup-*", info.Mode().Perm()) + if err != nil { + _ = update.Commit() + return nil, err + } + } else if !errors.Is(err, os.ErrNotExist) { + _ = update.Commit() + return nil, err + } + if err := syncServiceDirectory(filepath.Dir(target)); err != nil { + _ = update.Commit() + return nil, err + } + return update, nil +} + +func (u *DefinitionUpdate) Promote() error { + if u == nil || u.Target == "" || u.stagedPath == "" { + return errors.New("staged definition update is required") + } + if err := replaceServiceBinary(u.stagedPath, u.Target); err != nil { + return err + } + u.stagedPath = "" + if err := syncServiceDirectory(filepath.Dir(u.Target)); err != nil { + return err + } + digest, err := buildid.FileSHA256(u.Target) + if err != nil { + return err + } + if digest != u.CandidateSHA256 { + return fmt.Errorf("promoted definition digest=%s expected=%s", digest, u.CandidateSHA256) + } + return nil +} + +func (u *DefinitionUpdate) Rollback() error { + if u == nil || u.Target == "" { + return nil + } + if u.HadTarget { + if u.backupPath == "" { + return errors.New("definition rollback backup is unavailable") + } + if err := replaceServiceBinary(u.backupPath, u.Target); err != nil { + return err + } + u.backupPath = "" + } else if err := os.Remove(u.Target); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := syncServiceDirectory(filepath.Dir(u.Target)); err != nil { + return err + } + if u.HadTarget { + digest, err := buildid.FileSHA256(u.Target) + if err != nil { + return err + } + if digest != u.CurrentSHA256 { + return fmt.Errorf("rolled back definition digest=%s expected=%s", digest, u.CurrentSHA256) + } + } + return nil +} + +func (u *DefinitionUpdate) Commit() error { + if u == nil { + return nil + } + var result error + for _, path := range []string{u.stagedPath, u.backupPath} { + if path == "" { + continue + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + result = errors.Join(result, err) + } + } + u.stagedPath = "" + u.backupPath = "" + return errors.Join(result, syncServiceDirectory(filepath.Dir(u.Target))) +} diff --git a/internal/service/definition_update_test.go b/internal/service/definition_update_test.go new file mode 100644 index 0000000..e32acd7 --- /dev/null +++ b/internal/service/definition_update_test.go @@ -0,0 +1,88 @@ +package service + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestDefinitionUpdatePromotesAndCommitsExistingDefinition(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "service.plist") + if err := os.WriteFile(target, []byte("old-definition"), 0o600); err != nil { + t.Fatal(err) + } + update, err := StageDefinitionUpdate(target, []byte("new-definition")) + if err != nil { + t.Fatal(err) + } + if err := update.Promote(); err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(target); err != nil || string(data) != "new-definition" { + t.Fatalf("promoted definition=%q err=%v", data, err) + } + if err := update.Commit(); err != nil { + t.Fatal(err) + } + assertNoDefinitionUpdateArtifacts(t, root) +} + +func TestDefinitionUpdateRollsBackExistingAndNewDefinitions(t *testing.T) { + root := t.TempDir() + for _, test := range []struct { + name string + old []byte + }{ + {name: "existing", old: []byte("old-definition")}, + {name: "new"}, + } { + t.Run(test.name, func(t *testing.T) { + dir := filepath.Join(root, test.name) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + target := filepath.Join(dir, "service.plist") + if test.old != nil { + if err := os.WriteFile(target, test.old, 0o600); err != nil { + t.Fatal(err) + } + } + update, err := StageDefinitionUpdate(target, []byte("new-definition")) + if err != nil { + t.Fatal(err) + } + if err := update.Promote(); err != nil { + t.Fatal(err) + } + if err := update.Rollback(); err != nil { + t.Fatal(err) + } + if test.old == nil { + if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("new definition remained after rollback: %v", err) + } + } else if data, err := os.ReadFile(target); err != nil || string(data) != string(test.old) { + t.Fatalf("rolled back definition=%q err=%v", data, err) + } + if err := update.Commit(); err != nil { + t.Fatal(err) + } + assertNoDefinitionUpdateArtifacts(t, dir) + }) + } +} + +func assertNoDefinitionUpdateArtifacts(t *testing.T, root string) { + t.Helper() + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if len(entry.Name()) >= len(".codexfold-definition-") && entry.Name()[:len(".codexfold-definition-")] == ".codexfold-definition-" { + t.Fatalf("definition update artifact remained: %s", entry.Name()) + } + } +} diff --git a/internal/service/fskit_app.go b/internal/service/fskit_app.go new file mode 100644 index 0000000..2bac3f6 --- /dev/null +++ b/internal/service/fskit_app.go @@ -0,0 +1,44 @@ +package service + +import ( + "errors" + "path/filepath" + "strings" +) + +const ( + FSKitAppBundleName = "CodexFoldFSKit.app" + FSKitHostExecutableName = "CodexFoldFSKit" + FSKitHostBundleIdentifier = "vip.jstar.codexfold.fskitprofileprobe" + FSKitModuleBundleName = "CodexFoldFSKitModule.appex" + FSKitModuleIdentifier = "vip.jstar.codexfold.fskitprofileprobe.module" + FSKitAppGroupIdentifier = "group.vip.jstar.codexfold" + FSKitResourceDirectoryName = "native-fskit" +) + +func DefaultFSKitAppPath(userHome string) string { + return filepath.Join(filepath.Clean(userHome), "Applications", FSKitAppBundleName) +} + +func FSKitHostLauncherPath(appPath string) (string, error) { + if !filepath.IsAbs(appPath) { + return "", errors.New("FSKit app path must be absolute") + } + appPath = filepath.Clean(appPath) + if !strings.HasSuffix(filepath.Base(appPath), ".app") { + return "", errors.New("FSKit app path must identify an app bundle") + } + return filepath.Join(appPath, "Contents", "MacOS", FSKitHostExecutableName), nil +} + +func FSKitModulePath(appPath string) (string, error) { + launcher, err := FSKitHostLauncherPath(appPath) + if err != nil { + return "", err + } + return filepath.Join(filepath.Dir(filepath.Dir(launcher)), "Extensions", FSKitModuleBundleName), nil +} + +func DefaultFSKitResourcePath(userHome string) string { + return filepath.Join(filepath.Clean(userHome), "Library", "Group Containers", FSKitAppGroupIdentifier, FSKitResourceDirectoryName) +} diff --git a/internal/service/fskit_app_test.go b/internal/service/fskit_app_test.go new file mode 100644 index 0000000..ad28e21 --- /dev/null +++ b/internal/service/fskit_app_test.go @@ -0,0 +1,33 @@ +package service + +import ( + "path/filepath" + "testing" +) + +func TestFSKitManagedPathsUseStableAppAndAppGroupLocations(t *testing.T) { + home := filepath.Join(t.TempDir(), "user") + app := DefaultFSKitAppPath(home) + if app != filepath.Join(home, "Applications", FSKitAppBundleName) { + t.Fatalf("default app path = %q", app) + } + launcher, err := FSKitHostLauncherPath(app) + if err != nil { + t.Fatal(err) + } + if launcher != filepath.Join(app, "Contents", "MacOS", FSKitHostExecutableName) { + t.Fatalf("launcher path = %q", launcher) + } + resource := DefaultFSKitResourcePath(home) + if resource != filepath.Join(home, "Library", "Group Containers", FSKitAppGroupIdentifier, FSKitResourceDirectoryName) { + t.Fatalf("resource path = %q", resource) + } +} + +func TestFSKitHostLauncherRejectsNonAppAndRelativePaths(t *testing.T) { + for _, path := range []string{"CodexFoldFSKit.app", filepath.Join(t.TempDir(), "CodexFoldFSKit")} { + if _, err := FSKitHostLauncherPath(path); err == nil { + t.Fatalf("FSKitHostLauncherPath(%q) succeeded", path) + } + } +} diff --git a/internal/service/mount_probe_darwin.go b/internal/service/mount_probe_darwin.go new file mode 100644 index 0000000..19e28a0 --- /dev/null +++ b/internal/service/mount_probe_darwin.go @@ -0,0 +1,69 @@ +//go:build darwin + +package service + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/samekind/codexfold/internal/mountid" + "golang.org/x/sys/unix" +) + +func defaultMountProbe(path string) error { + var stat unix.Statfs_t + if err := unix.Statfs(path, &stat); err != nil { + return err + } + mountedAt := unix.ByteSliceToString(stat.Mntonname[:]) + filesystem := strings.ToLower(unix.ByteSliceToString(stat.Fstypename[:])) + mountedFrom := strings.ToLower(unix.ByteSliceToString(stat.Mntfromname[:])) + if canonicalMountPath(mountedAt) != canonicalMountPath(path) { + return errors.New("path is not a mount root") + } + if !validDarwinMountProvider(filesystem, mountedFrom) { + return errors.New("mount root is not backed by CodexFold native FSKit or the supported FUSE-T fallback") + } + value, err := os.ReadFile(filepath.Join(path, mountid.Path)) + if err != nil { + return fmt.Errorf("read CodexFold mount identity: %w", err) + } + if len(value) == 0 || len(value) > 256 { + return errors.New("CodexFold mount identity size is invalid") + } + if err := mountid.Validate(value); err != nil { + return err + } + return nil +} + +// MountPresent reports whether path is currently the root of any mounted +// filesystem. It deliberately does not require a healthy CodexFold identity: +// update code must not treat a damaged but still-mounted filesystem as absent. +func MountPresent(path string) (bool, error) { + var stat unix.Statfs_t + if err := unix.Statfs(path, &stat); err != nil { + if errors.Is(err, unix.ENOENT) { + return false, nil + } + return false, err + } + return canonicalMountPath(unix.ByteSliceToString(stat.Mntonname[:])) == canonicalMountPath(path), nil +} + +func validDarwinMountProvider(filesystem string, mountedFrom string) bool { + filesystem = strings.ToLower(strings.TrimSpace(filesystem)) + mountedFrom = strings.ToLower(strings.TrimSpace(mountedFrom)) + return filesystem == "codexfold" || filesystem == "nfs" && strings.HasPrefix(mountedFrom, "fuse-t:") +} + +func canonicalMountPath(path string) string { + resolved, err := filepath.EvalSymlinks(path) + if err == nil { + return filepath.Clean(resolved) + } + return filepath.Clean(path) +} diff --git a/internal/service/mount_probe_darwin_test.go b/internal/service/mount_probe_darwin_test.go new file mode 100644 index 0000000..8740ba2 --- /dev/null +++ b/internal/service/mount_probe_darwin_test.go @@ -0,0 +1,46 @@ +//go:build darwin + +package service + +import ( + "os" + "testing" +) + +func TestValidDarwinMountProviderAcceptsNativeFSKitAndFallbackOnly(t *testing.T) { + tests := []struct { + filesystem string + mountedFrom string + want bool + }{ + {filesystem: "codexfold", mountedFrom: "file:///private/tmp/resource.bin", want: true}, + {filesystem: "CODEXFOLD", mountedFrom: "FILE:///private/tmp/resource.bin", want: true}, + {filesystem: "nfs", mountedFrom: "fuse-t:/private/tmp/resource", want: true}, + {filesystem: "nfs", mountedFrom: "server:/export", want: false}, + {filesystem: "apfs", mountedFrom: "/dev/disk1s1", want: false}, + } + for _, test := range tests { + if got := validDarwinMountProvider(test.filesystem, test.mountedFrom); got != test.want { + t.Errorf("provider filesystem=%q mountedFrom=%q = %t, want %t", test.filesystem, test.mountedFrom, got, test.want) + } + } +} + +func TestMountPresentRejectsOrdinaryDirectory(t *testing.T) { + path := t.TempDir() + present, err := MountPresent(path) + if err != nil { + t.Fatal(err) + } + if present { + t.Fatalf("ordinary directory %s reported as a mount root", path) + } + missing := path + ".missing" + if _, err := os.Stat(missing); !os.IsNotExist(err) { + t.Fatalf("test path unexpectedly exists: %v", err) + } + present, err = MountPresent(missing) + if err != nil || present { + t.Fatalf("missing path mount presence = %t, %v", present, err) + } +} diff --git a/internal/service/mount_probe_linux.go b/internal/service/mount_probe_linux.go new file mode 100644 index 0000000..9162019 --- /dev/null +++ b/internal/service/mount_probe_linux.go @@ -0,0 +1,71 @@ +//go:build linux + +package service + +import ( + "errors" + "os" + "path/filepath" + "strings" + + "github.com/samekind/codexfold/internal/mountid" +) + +func defaultMountProbe(path string) error { + data, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + return err + } + want := filepath.Clean(path) + fuseMount := false + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) < 7 || filepath.Clean(unescapeLinuxMountField(fields[4])) != want { + continue + } + separator := -1 + for index := 6; index < len(fields); index++ { + if fields[index] == "-" { + separator = index + break + } + } + if separator >= 0 && separator+2 < len(fields) && strings.HasPrefix(fields[separator+1], "fuse") && strings.Contains(strings.ToLower(unescapeLinuxMountField(fields[separator+2])), "codexfold") { + fuseMount = true + } + break + } + if !fuseMount { + return errors.New("path is not a CodexFold FUSE mount root") + } + return validateMountIdentity(path) +} + +// MountPresent reports whether path is currently a mount root, independent of +// whether it is backed by CodexFold. +func MountPresent(path string) (bool, error) { + data, err := os.ReadFile("/proc/self/mountinfo") + if err != nil { + return false, err + } + want := filepath.Clean(path) + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) >= 5 && filepath.Clean(unescapeLinuxMountField(fields[4])) == want { + return true, nil + } + } + return false, nil +} + +func unescapeLinuxMountField(value string) string { + return strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`).Replace(value) +} + +func validateMountIdentity(path string) error { + value, err := os.ReadFile(filepath.Join(path, mountid.Path)) + if err != nil { + return err + } + return mountid.Validate(value) +} diff --git a/internal/service/mount_probe_other.go b/internal/service/mount_probe_other.go new file mode 100644 index 0000000..320ad8d --- /dev/null +++ b/internal/service/mount_probe_other.go @@ -0,0 +1,21 @@ +//go:build !darwin && !linux && !windows + +package service + +import ( + "errors" + "os" +) + +func defaultMountProbe(path string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + if !info.IsDir() { + return errors.New("mount path is not a directory") + } + return nil +} + +func MountPresent(string) (bool, error) { return false, nil } diff --git a/internal/service/mount_probe_windows.go b/internal/service/mount_probe_windows.go new file mode 100644 index 0000000..6b9597c --- /dev/null +++ b/internal/service/mount_probe_windows.go @@ -0,0 +1,20 @@ +//go:build windows + +package service + +import ( + "os" + "path/filepath" + + "github.com/samekind/codexfold/internal/mountid" +) + +func defaultMountProbe(path string) error { + value, err := os.ReadFile(filepath.Join(path, mountid.Path)) + if err != nil { + return err + } + return mountid.Validate(value) +} + +func MountPresent(string) (bool, error) { return false, nil } diff --git a/internal/service/native_fskit_operations_darwin.go b/internal/service/native_fskit_operations_darwin.go new file mode 100644 index 0000000..eed4e76 --- /dev/null +++ b/internal/service/native_fskit_operations_darwin.go @@ -0,0 +1,104 @@ +//go:build darwin + +package service + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/samekind/codexfold/internal/fskitproto" + "github.com/samekind/codexfold/internal/mountid" + "golang.org/x/sys/unix" +) + +type nativeFSKitOperations struct{} + +func defaultNativeFSKitOperations() (NativeFSKitOperations, error) { + return nativeFSKitOperations{}, nil +} + +func (nativeFSKitOperations) DaemonHealthy(ctx context.Context, resourcePath string) error { + if err := ctx.Err(); err != nil { + return err + } + client, err := fskitproto.DialResource(resourcePath, 2*time.Second) + if err != nil { + return err + } + defer client.Close() + _, err = client.Call(fskitproto.OpPing, nil) + return err +} + +func (nativeFSKitOperations) MountState(ctx context.Context, mountPoint string, timeout time.Duration) (NativeFSKitMountState, error) { + var stat unix.Statfs_t + if err := unix.Statfs(mountPoint, &stat); err != nil { + return NativeFSKitMountState{}, err + } + requested := canonicalMountPath(mountPoint) + actual := canonicalMountPath(unix.ByteSliceToString(stat.Mntonname[:])) + if requested != actual { + return NativeFSKitMountState{}, nil + } + filesystem := strings.ToLower(unix.ByteSliceToString(stat.Fstypename[:])) + state := NativeFSKitMountState{Mounted: true, Owned: filesystem == "codexfold"} + if !state.Owned { + return state, nil + } + if timeout <= 0 { + timeout = 2 * time.Second + } + probeCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + for _, directory := range []string{"sessions", "archived_sessions"} { + command := exec.CommandContext(probeCtx, "/usr/bin/stat", "-f", "%HT", filepath.Join(mountPoint, directory)) + if output, err := command.CombinedOutput(); err != nil { + return state, fmt.Errorf("probe native FSKit directory %s: %w: %s", directory, err, strings.TrimSpace(string(output))) + } + } + identity, err := os.ReadFile(filepath.Join(mountPoint, mountid.Path)) + if err != nil { + return state, fmt.Errorf("read native FSKit mount identity: %w", err) + } + if err := mountid.Validate(identity); err != nil { + return state, fmt.Errorf("validate native FSKit mount identity: %w", err) + } + state.Healthy = true + return state, nil +} + +func (nativeFSKitOperations) Mount(ctx context.Context, resourcePath string, mountPoint string) error { + if err := os.MkdirAll(mountPoint, 0o700); err != nil { + return err + } + output, err := exec.CommandContext(ctx, "/sbin/mount", nativeFSKitMountArguments(resourcePath, mountPoint)...).CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(output))) + } + return nil +} + +func nativeFSKitMountArguments(resourcePath string, mountPoint string) []string { + return []string{"-F", "-t", "codexfoldnative", resourcePath, mountPoint} +} + +func (nativeFSKitOperations) Unmount(ctx context.Context, mountPoint string, force bool) error { + arguments := []string{mountPoint} + if force { + arguments = []string{"-f", mountPoint} + } + output, err := exec.CommandContext(ctx, "/sbin/umount", arguments...).CombinedOutput() + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(output))) + } + return nil +} diff --git a/internal/service/native_fskit_operations_darwin_test.go b/internal/service/native_fskit_operations_darwin_test.go new file mode 100644 index 0000000..c83a579 --- /dev/null +++ b/internal/service/native_fskit_operations_darwin_test.go @@ -0,0 +1,15 @@ +//go:build darwin + +package service + +import ( + "reflect" + "testing" +) + +func TestNativeFSKitMountArgumentsForceFSKitModule(t *testing.T) { + want := []string{"-F", "-t", "codexfoldnative", "/tmp/resource", "/tmp/mount"} + if got := nativeFSKitMountArguments("/tmp/resource", "/tmp/mount"); !reflect.DeepEqual(got, want) { + t.Fatalf("mount arguments = %v, want %v", got, want) + } +} diff --git a/internal/service/native_fskit_operations_other.go b/internal/service/native_fskit_operations_other.go new file mode 100644 index 0000000..bdf1f8b --- /dev/null +++ b/internal/service/native_fskit_operations_other.go @@ -0,0 +1,11 @@ +//go:build !darwin + +package service + +import ( + "errors" +) + +func defaultNativeFSKitOperations() (NativeFSKitOperations, error) { + return nil, errors.New("native FSKit supervision is available only on macOS") +} diff --git a/internal/service/native_fskit_supervisor.go b/internal/service/native_fskit_supervisor.go new file mode 100644 index 0000000..4827258 --- /dev/null +++ b/internal/service/native_fskit_supervisor.go @@ -0,0 +1,160 @@ +package service + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "time" +) + +var ErrForeignMount = errors.New("mount point is occupied by a foreign filesystem") + +const NativeFSKitSupervisorLockName = "supervisor.lock" + +type NativeFSKitMountState struct { + Mounted bool + Owned bool + Healthy bool +} + +type NativeFSKitOperations interface { + DaemonHealthy(context.Context, string) error + MountState(context.Context, string, time.Duration) (NativeFSKitMountState, error) + Mount(context.Context, string, string) error + Unmount(context.Context, string, bool) error +} + +type NativeFSKitSupervisorOptions struct { + ResourcePath string + MountPoint string + Interval time.Duration + ProbeTimeout time.Duration + RecoveryTimeout time.Duration + Operations NativeFSKitOperations + Event func(string) +} + +func RunNativeFSKitSupervisor(ctx context.Context, options NativeFSKitSupervisorOptions) error { + if !filepath.IsAbs(options.ResourcePath) || !filepath.IsAbs(options.MountPoint) { + return errors.New("absolute FSKit resource and mount paths are required") + } + if options.Interval <= 0 { + options.Interval = time.Second + } + if options.ProbeTimeout <= 0 { + options.ProbeTimeout = 2 * time.Second + } + if options.RecoveryTimeout <= 0 { + options.RecoveryTimeout = 15 * time.Second + } + operations := options.Operations + if operations == nil { + var err error + operations, err = defaultNativeFSKitOperations() + if err != nil { + return err + } + } + state := nativeFSKitSupervisorState{} + ticker := time.NewTicker(options.Interval) + defer ticker.Stop() + for { + err := reconcileNativeFSKit(ctx, options, operations, &state) + if errors.Is(err, ErrForeignMount) { + return err + } + if err != nil && options.Event != nil { + options.Event(err.Error()) + } + select { + case <-ctx.Done(): + return shutdownNativeFSKit(options, operations) + case <-ticker.C: + } + } +} + +type nativeFSKitSupervisorState struct { + unhealthyOwnedMounts int +} + +func reconcileNativeFSKit( + ctx context.Context, + options NativeFSKitSupervisorOptions, + operations NativeFSKitOperations, + state *nativeFSKitSupervisorState, +) error { + mountState, mountErr := operations.MountState(ctx, options.MountPoint, options.ProbeTimeout) + if mountState.Mounted && !mountState.Owned { + return fmt.Errorf("%w: %s", ErrForeignMount, options.MountPoint) + } + daemonErr := operations.DaemonHealthy(ctx, options.ResourcePath) + if mountState.Owned && mountState.Healthy && daemonErr == nil { + state.unhealthyOwnedMounts = 0 + return nil + } + if mountState.Owned && !mountState.Healthy { + state.unhealthyOwnedMounts++ + if state.unhealthyOwnedMounts < 2 { + return errors.Join(mountErr, daemonErr, errors.New("owned FSKit mount failed its first health probe")) + } + if err := operations.Unmount(ctx, options.MountPoint, true); err != nil { + return errors.Join(mountErr, daemonErr, fmt.Errorf("force-unmount stale FSKit mount: %w", err)) + } + mountState = NativeFSKitMountState{} + state.unhealthyOwnedMounts = 0 + } + if daemonErr != nil { + return errors.Join(mountErr, fmt.Errorf("FSKit daemon unavailable: %w", daemonErr)) + } + if mountErr != nil && mountState.Mounted { + return mountErr + } + if mountState.Owned && mountState.Healthy { + return nil + } + if err := operations.Mount(ctx, options.ResourcePath, options.MountPoint); err != nil { + return fmt.Errorf("mount native FSKit volume: %w", err) + } + return waitForNativeFSKitMount(ctx, options, operations) +} + +func waitForNativeFSKitMount(ctx context.Context, options NativeFSKitSupervisorOptions, operations NativeFSKitOperations) error { + deadline := time.NewTimer(options.RecoveryTimeout) + defer deadline.Stop() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var lastErr error + for { + mountState, mountErr := operations.MountState(ctx, options.MountPoint, options.ProbeTimeout) + daemonErr := operations.DaemonHealthy(ctx, options.ResourcePath) + if mountState.Mounted && !mountState.Owned { + return fmt.Errorf("%w: %s", ErrForeignMount, options.MountPoint) + } + if mountState.Owned && mountState.Healthy && daemonErr == nil { + return nil + } + lastErr = errors.Join(mountErr, daemonErr) + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline.C: + return fmt.Errorf("native FSKit mount did not become healthy: %w", lastErr) + case <-ticker.C: + } + } +} + +func shutdownNativeFSKit(options NativeFSKitSupervisorOptions, operations NativeFSKitOperations) error { + ctx, cancel := context.WithTimeout(context.Background(), options.RecoveryTimeout) + defer cancel() + mountState, err := operations.MountState(ctx, options.MountPoint, options.ProbeTimeout) + if err != nil || !mountState.Owned { + return nil + } + if unmountErr := operations.Unmount(ctx, options.MountPoint, false); unmountErr == nil { + return nil + } + return operations.Unmount(ctx, options.MountPoint, true) +} diff --git a/internal/service/native_fskit_supervisor_test.go b/internal/service/native_fskit_supervisor_test.go new file mode 100644 index 0000000..62cec5d --- /dev/null +++ b/internal/service/native_fskit_supervisor_test.go @@ -0,0 +1,165 @@ +package service + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +func TestNativeFSKitSupervisorMountsAndUnmountsOnShutdown(t *testing.T) { + operations := &fakeNativeFSKitOperations{ + daemonHealthy: true, + mounted: make(chan struct{}), forceUnmounted: make(chan struct{}), + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- RunNativeFSKitSupervisor(ctx, NativeFSKitSupervisorOptions{ + ResourcePath: "/tmp/resource", MountPoint: "/tmp/mount", + Interval: time.Millisecond, RecoveryTimeout: 100 * time.Millisecond, + Operations: operations, + }) + }() + + select { + case <-operations.mounted: + case <-time.After(time.Second): + t.Fatal("supervisor did not mount") + } + cancel() + if err := <-done; err != nil { + t.Fatalf("supervisor shutdown: %v", err) + } + operations.mu.Lock() + defer operations.mu.Unlock() + if operations.mountCalls != 1 || operations.unmountCalls != 1 || operations.forceUnmountCalls != 0 { + t.Fatalf("mount calls=%d unmount=%d force=%d", operations.mountCalls, operations.unmountCalls, operations.forceUnmountCalls) + } +} + +func TestNativeFSKitSupervisorForceUnmountsStaleOwnedMountAfterTwoFailures(t *testing.T) { + operations := &fakeNativeFSKitOperations{ + daemonErr: errors.New("daemon unavailable"), + state: NativeFSKitMountState{Mounted: true, Owned: true, Healthy: false}, + mounted: make(chan struct{}), forceUnmounted: make(chan struct{}), + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + done <- RunNativeFSKitSupervisor(ctx, NativeFSKitSupervisorOptions{ + ResourcePath: "/tmp/resource", MountPoint: "/tmp/mount", + Interval: time.Millisecond, RecoveryTimeout: 100 * time.Millisecond, + Operations: operations, + }) + }() + + select { + case <-operations.forceUnmounted: + cancel() + case <-time.After(time.Second): + t.Fatal("supervisor did not force-unmount stale owned mount") + } + if err := <-done; err != nil { + t.Fatalf("supervisor shutdown: %v", err) + } + operations.mu.Lock() + defer operations.mu.Unlock() + if operations.forceUnmountCalls != 1 || operations.probeCalls < 2 { + t.Fatalf("force unmount=%d probes=%d", operations.forceUnmountCalls, operations.probeCalls) + } +} + +func TestNativeFSKitSupervisorRefusesForeignMount(t *testing.T) { + operations := &fakeNativeFSKitOperations{ + daemonHealthy: true, + state: NativeFSKitMountState{Mounted: true, Owned: false, Healthy: false}, + mounted: make(chan struct{}), forceUnmounted: make(chan struct{}), + } + err := RunNativeFSKitSupervisor(context.Background(), NativeFSKitSupervisorOptions{ + ResourcePath: "/tmp/resource", MountPoint: "/tmp/mount", + Interval: time.Millisecond, RecoveryTimeout: 100 * time.Millisecond, + Operations: operations, + }) + if !errors.Is(err, ErrForeignMount) { + t.Fatalf("foreign mount error = %v", err) + } + operations.mu.Lock() + defer operations.mu.Unlock() + if operations.mountCalls != 0 || operations.unmountCalls != 0 || operations.forceUnmountCalls != 0 { + t.Fatalf("foreign mount was mutated: %#v", operations) + } +} + +type fakeNativeFSKitOperations struct { + mu sync.Mutex + + daemonHealthy bool + daemonErr error + state NativeFSKitMountState + + probeCalls int + mountCalls int + unmountCalls int + forceUnmountCalls int + + mounted chan struct{} + forceUnmounted chan struct{} +} + +func (f *fakeNativeFSKitOperations) DaemonHealthy(context.Context, string) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.daemonErr != nil { + return f.daemonErr + } + if !f.daemonHealthy { + return errors.New("daemon unavailable") + } + return nil +} + +func (f *fakeNativeFSKitOperations) MountState(context.Context, string, time.Duration) (NativeFSKitMountState, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.probeCalls++ + return f.state, nil +} + +func (f *fakeNativeFSKitOperations) Mount(context.Context, string, string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.mountCalls++ + f.state = NativeFSKitMountState{Mounted: true, Owned: true, Healthy: true} + select { + case <-f.mounted: + default: + close(f.mounted) + } + return nil +} + +func (f *fakeNativeFSKitOperations) Unmount(_ context.Context, _ string, force bool) error { + f.mu.Lock() + defer f.mu.Unlock() + f.recordUnmount(force) + if f.state.Owned { + f.state = NativeFSKitMountState{} + } + return nil +} + +func (f *fakeNativeFSKitOperations) recordUnmount(force bool) { + if force { + f.forceUnmountCalls++ + select { + case <-f.forceUnmounted: + default: + close(f.forceUnmounted) + } + return + } + f.unmountCalls++ +} diff --git a/internal/service/platform.go b/internal/service/platform.go new file mode 100644 index 0000000..e43828e --- /dev/null +++ b/internal/service/platform.go @@ -0,0 +1,40 @@ +package service + +import ( + "errors" + "runtime" +) + +type Platform string + +const ( + PlatformLaunchd Platform = "launchd" + PlatformSystemd Platform = "systemd-user" + PlatformWindows Platform = "windows-service" +) + +func CurrentPlatform() (Platform, error) { + switch runtime.GOOS { + case "darwin": + return PlatformLaunchd, nil + case "linux": + return PlatformSystemd, nil + case "windows": + return PlatformWindows, nil + default: + return "", errors.New("transparent filesystem services are supported only on macOS, Linux, and Windows") + } +} + +func RenderDefinition(platform Platform, options Options) ([]byte, error) { + switch platform { + case PlatformLaunchd: + return RenderLaunchd(options) + case PlatformSystemd: + return RenderSystemd(options) + case PlatformWindows: + return RenderWindowsConfig(options) + default: + return nil, errors.New("unknown service platform") + } +} diff --git a/internal/service/process_lock.go b/internal/service/process_lock.go new file mode 100644 index 0000000..4bc2ed8 --- /dev/null +++ b/internal/service/process_lock.go @@ -0,0 +1,105 @@ +package service + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" +) + +type ProcessLock struct { + file *os.File +} + +type ProcessLockStatus struct { + Held bool + PID int +} + +func AcquireProcessLock(path string) (*ProcessLock, error) { + if !filepath.IsAbs(path) { + return nil, errors.New("absolute process lock path is required") + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, err + } + file, err := os.OpenFile(filepath.Clean(path), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, err + } + locked, err := tryLockProcessFile(file) + if err != nil { + _ = file.Close() + return nil, err + } + if !locked { + _ = file.Close() + return nil, errors.New("filesystem service process lock is already held") + } + if err := file.Truncate(0); err != nil { + _ = unlockProcessFile(file) + _ = file.Close() + return nil, err + } + if _, err := fmt.Fprintf(file, "%d\n", os.Getpid()); err != nil { + _ = unlockProcessFile(file) + _ = file.Close() + return nil, err + } + if err := file.Sync(); err != nil { + _ = unlockProcessFile(file) + _ = file.Close() + return nil, err + } + return &ProcessLock{file: file}, nil +} + +func (l *ProcessLock) Close() error { + if l == nil || l.file == nil { + return nil + } + file := l.file + l.file = nil + unlockErr := unlockProcessFile(file) + closeErr := file.Close() + if unlockErr != nil { + return unlockErr + } + return closeErr +} + +func InspectProcessLock(path string) (ProcessLockStatus, error) { + if !filepath.IsAbs(path) { + return ProcessLockStatus{}, errors.New("absolute process lock path is required") + } + file, err := os.OpenFile(filepath.Clean(path), os.O_RDWR, 0) + if errors.Is(err, os.ErrNotExist) { + return ProcessLockStatus{}, nil + } + if err != nil { + return ProcessLockStatus{}, err + } + defer file.Close() + locked, err := tryLockProcessFile(file) + if err != nil { + return ProcessLockStatus{}, err + } + if locked { + return ProcessLockStatus{}, unlockProcessFile(file) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return ProcessLockStatus{Held: true}, err + } + value, err := io.ReadAll(io.LimitReader(file, 64)) + if err != nil { + return ProcessLockStatus{Held: true}, err + } + pid, err := strconv.Atoi(strings.TrimSpace(string(value))) + if err != nil || pid <= 1 { + return ProcessLockStatus{Held: true}, errors.New("held process lock has an invalid owner PID") + } + return ProcessLockStatus{Held: true, PID: pid}, nil +} diff --git a/internal/service/process_lock_unix.go b/internal/service/process_lock_unix.go new file mode 100644 index 0000000..3eb3013 --- /dev/null +++ b/internal/service/process_lock_unix.go @@ -0,0 +1,22 @@ +//go:build !windows + +package service + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func tryLockProcessFile(file *os.File) (bool, error) { + err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if errors.Is(err, unix.EWOULDBLOCK) { + return false, nil + } + return err == nil, err +} + +func unlockProcessFile(file *os.File) error { + return unix.Flock(int(file.Fd()), unix.LOCK_UN) +} diff --git a/internal/service/process_lock_windows.go b/internal/service/process_lock_windows.go new file mode 100644 index 0000000..7ed4855 --- /dev/null +++ b/internal/service/process_lock_windows.go @@ -0,0 +1,23 @@ +//go:build windows + +package service + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +func tryLockProcessFile(file *os.File) (bool, error) { + overlapped := new(windows.Overlapped) + err := windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, overlapped) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return false, nil + } + return err == nil, err +} + +func unlockProcessFile(file *os.File) error { + return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, new(windows.Overlapped)) +} diff --git a/internal/service/process_parent_darwin.go b/internal/service/process_parent_darwin.go new file mode 100644 index 0000000..a87d9da --- /dev/null +++ b/internal/service/process_parent_darwin.go @@ -0,0 +1,24 @@ +//go:build darwin + +package service + +import ( + "errors" + + "golang.org/x/sys/unix" +) + +func ProcessParentPID(pid int) (int, error) { + if pid <= 1 { + return 0, errors.New("valid process PID is required") + } + process, err := unix.SysctlKinfoProc("kern.proc.pid", pid) + if err != nil { + return 0, err + } + parent := int(process.Eproc.Ppid) + if parent <= 0 { + return 0, errors.New("process parent PID is unavailable") + } + return parent, nil +} diff --git a/internal/service/process_parent_darwin_test.go b/internal/service/process_parent_darwin_test.go new file mode 100644 index 0000000..3bbc3c2 --- /dev/null +++ b/internal/service/process_parent_darwin_test.go @@ -0,0 +1,18 @@ +//go:build darwin + +package service + +import ( + "os" + "testing" +) + +func TestProcessParentPIDReportsCurrentParent(t *testing.T) { + parent, err := ProcessParentPID(os.Getpid()) + if err != nil { + t.Fatal(err) + } + if parent != os.Getppid() { + t.Fatalf("parent PID = %d, want %d", parent, os.Getppid()) + } +} diff --git a/internal/service/process_parent_other.go b/internal/service/process_parent_other.go new file mode 100644 index 0000000..9c52456 --- /dev/null +++ b/internal/service/process_parent_other.go @@ -0,0 +1,9 @@ +//go:build !darwin + +package service + +import "errors" + +func ProcessParentPID(int) (int, error) { + return 0, errors.New("process parent inspection is available only on macOS") +} diff --git a/internal/service/service.go b/internal/service/service.go new file mode 100644 index 0000000..f5b2f8b --- /dev/null +++ b/internal/service/service.go @@ -0,0 +1,450 @@ +package service + +import ( + "bytes" + "context" + "encoding/xml" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/samekind/codexfold/internal/compat" + "github.com/samekind/codexfold/internal/fsctl" +) + +type Options struct { + Label string + BinaryPath string + LauncherPath string + CodexHome string + StoreDir string + MountPoint string + StdoutPath string + StderrPath string + CanonicalNamespace bool + NativeRoot string + OperationTrace string + EnrollmentInterval time.Duration + EnrollmentStableFor time.Duration + EnrollmentBatchSize int + EnrollmentCanary bool + Frontend string + FSKitResource string +} + +type InstallResult struct { + Path string `json:"path"` + DryRun bool `json:"dry_run"` + Bytes int `json:"bytes"` +} + +type Runner interface { + Run(context.Context, string, ...string) ([]byte, error) +} + +type ExecRunner struct{} + +func (ExecRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) { + return exec.CommandContext(ctx, name, args...).CombinedOutput() +} + +type Manager struct { + UID int + Runner Runner + MountProbe func(string) error +} + +type Status struct { + DaemonRunning bool `json:"daemon_running"` + DaemonPID int `json:"daemon_pid,omitempty"` + SupervisorRunning bool `json:"supervisor_running,omitempty"` + SupervisorPID int `json:"supervisor_pid,omitempty"` + MountHealthy bool `json:"mount_healthy"` + DaemonError string `json:"daemon_error,omitempty"` + SupervisorError string `json:"supervisor_error,omitempty"` + MountError string `json:"mount_error,omitempty"` + Build BuildStatus `json:"build"` +} + +type UpdateInput struct { + Capability fsctl.Capability + DoctorHealthy bool + Compatibility compat.Evaluation + NativeFallbackReady bool + Automatic bool + ExplicitPromotion bool +} + +type UpdateDecision struct { + Allowed bool `json:"allowed"` + Quarantine bool `json:"quarantine"` + RequiresNativeFallback bool `json:"requires_native_fallback"` + Reason string `json:"reason,omitempty"` +} + +func RenderLaunchd(options Options) ([]byte, error) { + serveArguments, err := ServeArguments(options) + if err != nil { + return nil, err + } + return renderLaunchdJob(options.Label, launchdProgramArguments(options, serveArguments), options.StdoutPath, options.StderrPath, "Interactive"), nil +} + +func RenderLaunchdSupervisor(options Options) ([]byte, error) { + if err := validateOptions(options); err != nil { + return nil, err + } + if options.Frontend != "native-fskit" { + return nil, errors.New("native FSKit supervisor requires the native-fskit frontend") + } + arguments := []string{ + "fs", "supervise", "--apply", + "--resource", options.FSKitResource, "--mount", options.MountPoint, + } + return renderLaunchdJob(options.Label+".supervisor", launchdProgramArguments(options, arguments), options.StdoutPath, options.StderrPath, "Background"), nil +} + +func launchdProgramArguments(options Options, childArguments []string) []string { + if options.Frontend == "native-fskit" { + arguments := []string{options.LauncherPath, "--run-helper", options.BinaryPath} + return append(arguments, childArguments...) + } + return append([]string{options.BinaryPath}, childArguments...) +} + +func renderLaunchdJob(label string, arguments []string, stdoutPath string, stderrPath string, processType string) []byte { + var output bytes.Buffer + output.WriteString("\n") + output.WriteString("\n") + output.WriteString("\n\n") + writePlistString(&output, "Label", label) + output.WriteString(" ProgramArguments\n \n") + for _, argument := range arguments { + output.WriteString(" ") + _ = xml.EscapeText(&output, []byte(argument)) + output.WriteString("\n") + } + output.WriteString(" \n") + writePlistString(&output, "StandardOutPath", stdoutPath) + writePlistString(&output, "StandardErrorPath", stderrPath) + output.WriteString(" RunAtLoad\n \n") + output.WriteString(" KeepAlive\n \n") + writePlistString(&output, "ProcessType", processType) + output.WriteString(" ThrottleInterval\n 2\n") + output.WriteString("\n\n") + return output.Bytes() +} + +func ServeArguments(options Options) ([]string, error) { + if err := validateOptions(options); err != nil { + return nil, err + } + arguments := []string{ + "fs", "serve", "--apply", "--foreground=true", + "--codex-home", options.CodexHome, "--store", options.StoreDir, "--mount", options.MountPoint, + } + frontend := options.Frontend + if frontend == "" { + frontend = "fuse" + } + arguments = append(arguments, "--frontend", frontend) + if frontend == "native-fskit" { + arguments = append(arguments, "--fskit-resource", options.FSKitResource) + } + if options.CanonicalNamespace { + arguments = append(arguments, "--canonical-namespace", "--native-root", options.NativeRoot) + } + if options.OperationTrace != "" { + arguments = append(arguments, "--operation-trace", options.OperationTrace) + } + if options.EnrollmentInterval > 0 { + arguments = append(arguments, + "--enrollment-interval", options.EnrollmentInterval.String(), + "--enrollment-stable-for", options.EnrollmentStableFor.String(), + "--enrollment-batch-size", strconv.Itoa(options.EnrollmentBatchSize), + ) + if options.EnrollmentCanary { + arguments = append(arguments, "--enrollment-canary") + } + } + return arguments, nil +} + +func WriteDefinition(path string, definition []byte, apply bool) (InstallResult, error) { + if !filepath.IsAbs(path) || len(definition) == 0 { + return InstallResult{}, errors.New("absolute definition path and non-empty definition are required") + } + result := InstallResult{Path: filepath.Clean(path), DryRun: !apply, Bytes: len(definition)} + if !apply { + return result, nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return InstallResult{}, err + } + temporary, err := os.CreateTemp(filepath.Dir(path), ".service-definition-*.tmp") + if err != nil { + return InstallResult{}, err + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return InstallResult{}, err + } + if _, err := temporary.Write(definition); err != nil { + _ = temporary.Close() + return InstallResult{}, err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return InstallResult{}, err + } + if err := temporary.Close(); err != nil { + return InstallResult{}, err + } + if err := os.Rename(temporaryPath, path); err != nil { + return InstallResult{}, err + } + return result, nil +} + +func (m Manager) Bootstrap(ctx context.Context, plistPath string) error { + if !filepath.IsAbs(plistPath) { + return errors.New("absolute launchd plist path is required") + } + output, err := m.runner().Run(ctx, "launchctl", "bootstrap", m.domain(), plistPath) + if err != nil { + return commandFailure("launchctl bootstrap", output, err) + } + return nil +} + +func (m Manager) Bootout(ctx context.Context, plistPath string) error { + if !filepath.IsAbs(plistPath) { + return errors.New("absolute launchd plist path is required") + } + runner := m.runner() + output, err := runner.Run(ctx, "launchctl", "bootout", m.domain(), plistPath) + if err != nil { + if launchdJobMissing(output) { + return nil + } + label, labelErr := DefinitionLabel(PlatformLaunchd, plistPath) + if labelErr == nil { + statusOutput, statusErr := runner.Run(ctx, "launchctl", "print", m.domain()+"/"+label) + if statusErr != nil && launchdJobMissing(statusOutput) { + return nil + } + } + return commandFailure("launchctl bootout", output, err) + } + return nil +} + +func launchdJobMissing(output []byte) bool { + message := strings.ToLower(string(output)) + return strings.Contains(message, "could not find") && strings.Contains(message, "service") +} + +func (m Manager) Kickstart(ctx context.Context, label string) error { + if !safeLabel(label) { + return errors.New("safe launchd label is required") + } + output, err := m.runner().Run(ctx, "launchctl", "kickstart", m.domain()+"/"+label) + if err != nil { + return commandFailure("launchctl kickstart", output, err) + } + return nil +} + +func (m Manager) Enable(ctx context.Context, label string) error { + if !safeLabel(label) { + return errors.New("safe launchd label is required") + } + output, err := m.runner().Run(ctx, "launchctl", "enable", m.domain()+"/"+label) + if err != nil { + return commandFailure("launchctl enable", output, err) + } + return nil +} + +func (m Manager) Disable(ctx context.Context, label string) error { + if !safeLabel(label) { + return errors.New("safe launchd label is required") + } + output, err := m.runner().Run(ctx, "launchctl", "disable", m.domain()+"/"+label) + if err != nil { + return commandFailure("launchctl disable", output, err) + } + return nil +} + +func (m Manager) Status(ctx context.Context, label string, mountPoint string) Status { + result := Status{} + output, err := m.runner().Run(ctx, "launchctl", "print", m.domain()+"/"+label) + if err != nil { + result.DaemonError = err.Error() + } else if strings.Contains(string(output), "state = running") { + result.DaemonRunning = true + for _, line := range strings.Split(string(output), "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "pid = ") { + continue + } + result.DaemonPID, _ = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "pid = "))) + break + } + if result.DaemonPID <= 1 { + result.DaemonRunning = false + result.DaemonError = "launchd job is running without a valid PID" + } + } else { + result.DaemonError = "launchd job is loaded but not running" + } + probe := m.MountProbe + if probe == nil { + probe = ProbeMount + } + if err := probe(mountPoint); err != nil { + result.MountError = err.Error() + } else { + result.MountHealthy = true + } + return result +} + +func (m Manager) WaitHealthy(ctx context.Context, label string, mountPoint string, timeout time.Duration) (Status, error) { + if timeout <= 0 { + timeout = 15 * time.Second + } + deadline := time.NewTimer(timeout) + defer deadline.Stop() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var last Status + for { + last = m.Status(ctx, label, mountPoint) + if last.DaemonRunning && last.MountHealthy { + return last, nil + } + select { + case <-ctx.Done(): + return last, ctx.Err() + case <-deadline.C: + return last, fmt.Errorf("filesystem service did not become healthy: daemon=%t mount=%t daemon_error=%q mount_error=%q", last.DaemonRunning, last.MountHealthy, last.DaemonError, last.MountError) + case <-ticker.C: + } + } +} + +func ProbeMount(path string) error { return defaultMountProbe(path) } + +func EvaluateUpdate(input UpdateInput) UpdateDecision { + if !input.DoctorHealthy { + return UpdateDecision{Reason: "filesystem doctor is not healthy"} + } + if input.Compatibility.Quarantine || !input.Compatibility.Approved { + return UpdateDecision{Quarantine: true, RequiresNativeFallback: !input.NativeFallbackReady, Reason: "installed client version is not approved"} + } + if input.Automatic && (input.Capability == fsctl.FSEnginePreview || input.Capability == fsctl.PlatformCanary) { + return UpdateDecision{Reason: "automatic updates are disabled before platform production readiness"} + } + if (input.Capability == fsctl.FSEnginePreview || input.Capability == fsctl.PlatformCanary) && !input.ExplicitPromotion { + return UpdateDecision{Reason: "preview and canary updates require explicit promotion"} + } + return UpdateDecision{Allowed: true} +} + +func (m Manager) runner() Runner { + if m.Runner != nil { + return m.Runner + } + return ExecRunner{} +} + +func (m Manager) domain() string { + uid := m.UID + if uid <= 0 { + uid = os.Getuid() + } + return fmt.Sprintf("gui/%d", uid) +} + +func validateOptions(options Options) error { + if !safeLabel(options.Label) { + return errors.New("safe service label is required") + } + for name, path := range map[string]string{ + "binary": options.BinaryPath, "Codex home": options.CodexHome, "store": options.StoreDir, + "mount": options.MountPoint, "stdout": options.StdoutPath, "stderr": options.StderrPath, + } { + if !filepath.IsAbs(path) { + return fmt.Errorf("%s path must be absolute", name) + } + } + if options.CanonicalNamespace && !filepath.IsAbs(options.NativeRoot) { + return errors.New("canonical namespace requires an absolute native root") + } + if options.OperationTrace != "" && !filepath.IsAbs(options.OperationTrace) { + return errors.New("operation trace path must be absolute") + } + if options.EnrollmentInterval < 0 || options.EnrollmentStableFor < 0 || options.EnrollmentBatchSize < 0 { + return errors.New("enrollment timing and batch values cannot be negative") + } + if options.EnrollmentInterval > 0 { + if !options.CanonicalNamespace { + return errors.New("periodic enrollment requires the canonical namespace") + } + if options.EnrollmentStableFor <= 0 || options.EnrollmentBatchSize <= 0 { + return errors.New("periodic enrollment requires a positive stable window and batch size") + } + } + if options.EnrollmentCanary && options.EnrollmentInterval <= 0 { + return errors.New("enrollment canary requires periodic enrollment") + } + frontend := options.Frontend + if frontend == "" { + frontend = "fuse" + } + if frontend != "fuse" && frontend != "native-fskit" { + return errors.New("filesystem frontend must be fuse or native-fskit") + } + if frontend == "native-fskit" { + if !options.CanonicalNamespace { + return errors.New("native-fskit frontend requires the canonical namespace") + } + if !filepath.IsAbs(options.LauncherPath) { + return errors.New("native-fskit frontend requires an absolute host launcher path") + } + if !filepath.IsAbs(options.FSKitResource) { + return errors.New("native-fskit frontend requires an absolute resource path") + } + } + return nil +} + +func safeLabel(label string) bool { + if label == "" || strings.HasPrefix(label, ".") || strings.HasSuffix(label, ".") { + return false + } + for _, character := range label { + if character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' || character >= '0' && character <= '9' || character == '.' || character == '-' { + continue + } + return false + } + return true +} + +func writePlistString(output *bytes.Buffer, key string, value string) { + output.WriteString(" ") + _ = xml.EscapeText(output, []byte(key)) + output.WriteString("\n ") + _ = xml.EscapeText(output, []byte(value)) + output.WriteString("\n") +} diff --git a/internal/service/service_test.go b/internal/service/service_test.go new file mode 100644 index 0000000..6ae7657 --- /dev/null +++ b/internal/service/service_test.go @@ -0,0 +1,414 @@ +package service + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/samekind/codexfold/internal/compat" + "github.com/samekind/codexfold/internal/fsctl" +) + +func TestRenderLaunchdUsesAbsoluteArgumentsAndContainsNoSessionContent(t *testing.T) { + root := t.TempDir() + definition, err := RenderLaunchd(Options{ + Label: "com.codexfold.fs", BinaryPath: filepath.Join(root, "bin", "codexfold"), + CodexHome: filepath.Join(root, "codex"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "logs", "stdout.log"), + StderrPath: filepath.Join(root, "logs", "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), OperationTrace: filepath.Join(root, "logs", "operations.log"), + EnrollmentInterval: 5 * time.Minute, EnrollmentStableFor: time.Hour, EnrollmentBatchSize: 2, + EnrollmentCanary: true, + }) + if err != nil { + t.Fatalf("RenderLaunchd: %v", err) + } + text := string(definition) + for _, required := range []string{ + "fs", "serve", "--apply", + "--canonical-namespace", "--native-root", + "--operation-trace", filepath.Join(root, "logs", "operations.log"), + "--enrollment-interval", "5m0s", + "--enrollment-stable-for", "1h0m0s", + "--enrollment-batch-size", "2", + "--enrollment-canary", + filepath.Join(root, "store"), filepath.Join(root, "mount"), filepath.Join(root, "native"), + } { + if !strings.Contains(text, required) { + t.Fatalf("definition missing %q:\n%s", required, text) + } + } + if strings.Contains(text, "session_meta") || strings.Contains(text, "rollout") { + t.Fatalf("definition contains session content: %s", text) + } + if runtime.GOOS == "darwin" { + path := filepath.Join(root, "service.plist") + if err := os.WriteFile(path, definition, 0o600); err != nil { + t.Fatal(err) + } + if output, err := exec.Command("/usr/bin/plutil", "-lint", path).CombinedOutput(); err != nil { + t.Fatalf("plutil rejected definition: %v\n%s", err, output) + } + } + if _, err := RenderLaunchd(Options{Label: "com.codexfold.fs", BinaryPath: "codexfold", CodexHome: root, StoreDir: root, MountPoint: root, StdoutPath: filepath.Join(root, "out"), StderrPath: filepath.Join(root, "err")}); err == nil { + t.Fatal("relative binary path should be rejected") + } +} + +func TestRenderLaunchdNativeFSKitSeparatesDaemonAndSupervisor(t *testing.T) { + root := t.TempDir() + launcher := filepath.Join(root, "CodexFoldFSKit.app", "Contents", "MacOS", "CodexFoldFSKit") + binary := filepath.Join(root, "bin", "codexfold") + options := Options{ + Label: "com.codexfold.fs", BinaryPath: binary, LauncherPath: launcher, + CodexHome: filepath.Join(root, "codex"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "logs", "stdout.log"), + StderrPath: filepath.Join(root, "logs", "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), Frontend: "native-fskit", + FSKitResource: filepath.Join(root, "store", "fs", "native-fskit.resource"), + } + daemon, err := RenderLaunchd(options) + if err != nil { + t.Fatalf("RenderLaunchd: %v", err) + } + for _, required := range []string{ + "" + launcher + "", "--run-helper", "" + binary + "", + "--frontend", "native-fskit", + "--fskit-resource", options.FSKitResource, + "ProcessType\n Interactive", + } { + if !strings.Contains(string(daemon), required) { + t.Fatalf("native daemon definition missing %q:\n%s", required, daemon) + } + } + + supervisor, err := RenderLaunchdSupervisor(options) + if err != nil { + t.Fatalf("RenderLaunchdSupervisor: %v", err) + } + if !strings.Contains(string(supervisor), "ProcessType\n Background") { + t.Fatalf("native supervisor is not background:\n%s", supervisor) + } + text := string(supervisor) + for _, required := range []string{ + "com.codexfold.fs.supervisor", + "" + launcher + "", "--run-helper", "" + binary + "", + "fs", "supervise", "--apply", + "--resource", options.FSKitResource, + "--mount", options.MountPoint, + } { + if !strings.Contains(text, required) { + t.Fatalf("native supervisor definition missing %q:\n%s", required, text) + } + } +} + +func TestRenderLaunchdNativeFSKitRequiresHostLauncher(t *testing.T) { + root := t.TempDir() + _, err := RenderLaunchd(Options{ + Label: "com.codexfold.fs", BinaryPath: filepath.Join(root, "codexfold"), + CodexHome: filepath.Join(root, "home"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "stdout.log"), + StderrPath: filepath.Join(root, "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), Frontend: "native-fskit", + FSKitResource: filepath.Join(root, "resource"), + }) + if err == nil || !strings.Contains(err.Error(), "launcher") { + t.Fatalf("native FSKit without launcher error = %v", err) + } +} + +func TestRenderSystemdUsesTheSameServeArgumentsAndRestartPolicy(t *testing.T) { + root := filepath.Join(t.TempDir(), "path with space % and $") + options := Options{ + Label: "com.codexfold.fs", BinaryPath: filepath.Join(root, "bin", "codexfold"), + CodexHome: filepath.Join(root, "codex"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "logs", "stdout.log"), + StderrPath: filepath.Join(root, "logs", "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), OperationTrace: filepath.Join(root, "logs", "operations.log"), + EnrollmentInterval: 5 * time.Minute, EnrollmentStableFor: time.Hour, EnrollmentBatchSize: 2, + EnrollmentCanary: true, + } + definition, err := RenderSystemd(options) + if err != nil { + t.Fatalf("RenderSystemd: %v", err) + } + text := string(definition) + for _, required := range []string{ + "[Service]", "Type=simple", "Restart=on-failure", "RestartSec=2s", "TimeoutStopSec=30s", + "--canonical-namespace", "--native-root", "--operation-trace", + "--enrollment-interval", "5m0s", "--enrollment-canary", + "ExecStart=:\"", "StandardOutput=append:", "StandardError=append:", "\\x20", "%%", "$", + } { + if !strings.Contains(text, required) { + t.Fatalf("systemd definition missing %q:\n%s", required, text) + } + } + if strings.Contains(text, "session_meta") || strings.Contains(text, "rollout") { + t.Fatalf("definition contains session content: %s", text) + } +} + +func TestRenderWindowsConfigUsesTheSameServeArguments(t *testing.T) { + root := t.TempDir() + definition, err := RenderWindowsConfig(Options{ + Label: "com.codexfold.fs", BinaryPath: filepath.Join(root, "codexfold.exe"), + CodexHome: filepath.Join(root, "codex"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "logs", "stdout.log"), + StderrPath: filepath.Join(root, "logs", "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), + }) + if err != nil { + t.Fatalf("RenderWindowsConfig: %v", err) + } + config, err := ParseWindowsConfig(definition) + if err != nil { + t.Fatalf("ParseWindowsConfig: %v", err) + } + if config.Version != 1 || config.ServiceName != "com.codexfold.fs" { + t.Fatalf("unexpected Windows config: %#v", config) + } + joined := strings.Join(config.Arguments, " ") + for _, required := range []string{"fs serve --apply --foreground=true", "--canonical-namespace", "--native-root"} { + if !strings.Contains(joined, required) { + t.Fatalf("Windows config missing %q: %s", required, joined) + } + } +} + +func TestManagerUsesOnlyPerUserLaunchctlAndSeparatesDaemonFromMount(t *testing.T) { + root := t.TempDir() + runner := &recordingRunner{outputs: map[string][]byte{"launchctl print gui/501/com.codexfold.fs": []byte("state = running\npid = 123\n")}} + manager := Manager{UID: 501, Runner: runner, MountProbe: func(string) error { return errors.New("mount unavailable") }} + plist := filepath.Join(root, "com.codexfold.fs.plist") + if err := os.WriteFile(plist, []byte("plist"), 0o600); err != nil { + t.Fatal(err) + } + if err := manager.Bootstrap(context.Background(), plist); err != nil { + t.Fatalf("Bootstrap: %v", err) + } + if err := manager.Kickstart(context.Background(), "com.codexfold.fs"); err != nil { + t.Fatalf("Kickstart: %v", err) + } + status := manager.Status(context.Background(), "com.codexfold.fs", filepath.Join(root, "mount")) + if !status.DaemonRunning || status.DaemonPID != 123 || status.MountHealthy { + t.Fatalf("status did not separate daemon and mount: %#v", status) + } + joined := strings.Join(runner.calls, "\n") + if strings.Contains(joined, "sudo") || !strings.Contains(joined, "launchctl bootstrap gui/501") || !strings.Contains(joined, "launchctl kickstart gui/501/com.codexfold.fs") { + t.Fatalf("unexpected lifecycle commands:\n%s", joined) + } +} + +func TestManagerBootoutTreatsExplicitlyMissingJobAsAlreadyStopped(t *testing.T) { + root := t.TempDir() + plist := filepath.Join(root, "com.codexfold.test.plist") + if err := os.WriteFile(plist, renderLaunchdJob("com.codexfold.test", []string{"/tmp/codexfold"}, filepath.Join(root, "out.log"), filepath.Join(root, "err.log"), "Background"), 0o600); err != nil { + t.Fatal(err) + } + bootout := "launchctl bootout gui/501 " + plist + printJob := "launchctl print gui/501/com.codexfold.test" + runner := &recordingRunner{ + outputs: map[string][]byte{printJob: []byte("Bad request.\nCould not find service \\\"com.codexfold.test\\\" in domain for user gui: 501\n")}, + errors: map[string]error{bootout: errors.New("exit status 5"), printJob: errors.New("exit status 113")}, + } + if err := (Manager{UID: 501, Runner: runner}).Bootout(context.Background(), plist); err != nil { + t.Fatalf("Bootout missing job: %v", err) + } + if want := []string{bootout, printJob}; strings.Join(runner.calls, "\n") != strings.Join(want, "\n") { + t.Fatalf("calls = %v, want %v", runner.calls, want) + } +} + +func TestManagerBootoutDoesNotHideUnclassifiedLaunchdFailure(t *testing.T) { + root := t.TempDir() + plist := filepath.Join(root, "com.codexfold.test.plist") + if err := os.WriteFile(plist, renderLaunchdJob("com.codexfold.test", []string{"/tmp/codexfold"}, filepath.Join(root, "out.log"), filepath.Join(root, "err.log"), "Background"), 0o600); err != nil { + t.Fatal(err) + } + bootout := "launchctl bootout gui/501 " + plist + printJob := "launchctl print gui/501/com.codexfold.test" + runner := &recordingRunner{ + outputs: map[string][]byte{printJob: []byte("Operation not permitted\n")}, + errors: map[string]error{bootout: errors.New("exit status 5"), printJob: errors.New("exit status 1")}, + } + if err := (Manager{UID: 501, Runner: runner}).Bootout(context.Background(), plist); err == nil || !strings.Contains(err.Error(), "launchctl bootout") { + t.Fatalf("Bootout error = %v", err) + } +} + +func TestSystemdManagerUsesOnlyTheUserManagerAndSeparatesMountHealth(t *testing.T) { + runner := &recordingRunner{outputs: map[string][]byte{ + "systemctl --user show com.codexfold.fs.service --property=ActiveState --property=SubState --no-pager": []byte("ActiveState=active\nSubState=running\n"), + }} + manager := SystemdManager{Runner: runner, MountProbe: func(string) error { return errors.New("mount unavailable") }} + if err := manager.Start(context.Background(), "com.codexfold.fs.service"); err != nil { + t.Fatalf("Start: %v", err) + } + if err := manager.Stop(context.Background(), "com.codexfold.fs.service"); err != nil { + t.Fatalf("Stop: %v", err) + } + status := manager.Status(context.Background(), "com.codexfold.fs.service", filepath.Join(t.TempDir(), "mount")) + if !status.DaemonRunning || status.MountHealthy { + t.Fatalf("status did not separate daemon and mount: %#v", status) + } + joined := strings.Join(runner.calls, "\n") + for _, required := range []string{ + "systemctl --user daemon-reload", + "systemctl --user enable --now com.codexfold.fs.service", + "systemctl --user stop com.codexfold.fs.service", + } { + if !strings.Contains(joined, required) { + t.Fatalf("missing systemd user command %q:\n%s", required, joined) + } + } + if strings.Contains(joined, "sudo") || strings.Contains(joined, "systemctl enable") { + t.Fatalf("system service command leaked into user manager:\n%s", joined) + } +} + +func TestWindowsManagerInstallsStartsStopsAndReportsSCMState(t *testing.T) { + installRunner := &recordingRunner{errors: map[string]error{ + "sc.exe query com.codexfold.fs": errors.New("service does not exist"), + }} + manager := WindowsManager{Runner: installRunner} + binary := `C:\Program Files\CodexFold\codexfold.exe` + definition := `C:\ProgramData\CodexFold\service.json` + if err := manager.Install(context.Background(), "com.codexfold.fs", binary, definition); err != nil { + t.Fatalf("Install: %v", err) + } + joined := strings.Join(installRunner.calls, "\n") + for _, required := range []string{ + "sc.exe create com.codexfold.fs", + `"C:\Program Files\CodexFold\codexfold.exe" fs service run --definition C:\ProgramData\CodexFold\service.json`, + "start= auto", + "sc.exe failure com.codexfold.fs", + } { + if !strings.Contains(joined, required) { + t.Fatalf("missing Windows service command %q:\n%s", required, joined) + } + } + + statusRunner := &recordingRunner{outputs: map[string][]byte{ + "sc.exe queryex com.codexfold.fs": []byte("STATE : 4 RUNNING\n"), + }} + manager = WindowsManager{Runner: statusRunner, MountProbe: func(string) error { return nil }} + if err := manager.Start(context.Background(), "com.codexfold.fs"); err != nil { + t.Fatalf("Start: %v", err) + } + if err := manager.Stop(context.Background(), "com.codexfold.fs"); err != nil { + t.Fatalf("Stop: %v", err) + } + status := manager.Status(context.Background(), "com.codexfold.fs", filepath.Join(t.TempDir(), "mount")) + if !status.DaemonRunning || !status.MountHealthy { + t.Fatalf("Windows service status = %#v", status) + } +} + +func TestStatusDoesNotTreatLoadedExitedJobAsRunning(t *testing.T) { + runner := &recordingRunner{outputs: map[string][]byte{ + "launchctl print gui/501/com.codexfold.fs": []byte("state = exited\nlast exit code = 1\n"), + }} + status := (Manager{UID: 501, Runner: runner, MountProbe: func(string) error { return errors.New("not mounted") }}).Status( + context.Background(), "com.codexfold.fs", filepath.Join(t.TempDir(), "mount"), + ) + if status.DaemonRunning || status.DaemonError == "" { + t.Fatalf("loaded exited job was reported as running: %#v", status) + } +} + +func TestWaitHealthyRequiresRunningDaemonAndLiveMount(t *testing.T) { + runner := &recordingRunner{outputs: map[string][]byte{ + "launchctl print gui/501/com.codexfold.fs": []byte("state = running\npid = 123\n"), + }} + probes := 0 + manager := Manager{UID: 501, Runner: runner, MountProbe: func(string) error { + probes++ + if probes < 3 { + return errors.New("mount starting") + } + return nil + }} + status, err := manager.WaitHealthy(context.Background(), "com.codexfold.fs", filepath.Join(t.TempDir(), "mount"), time.Second) + if err != nil || !status.DaemonRunning || !status.MountHealthy || probes != 3 { + t.Fatalf("WaitHealthy status=%#v probes=%d err=%v", status, probes, err) + } +} + +func TestEvaluateUpdateQuarantinesUnknownVersionsAndRejectsPreviewAutomation(t *testing.T) { + unknown := EvaluateUpdate(UpdateInput{Capability: fsctl.StorageEngine, DoctorHealthy: true, Compatibility: compat.Evaluation{Quarantine: true}, NativeFallbackReady: false}) + if unknown.Allowed || !unknown.Quarantine || !unknown.RequiresNativeFallback { + t.Fatalf("unknown version was not quarantined: %#v", unknown) + } + ready := EvaluateUpdate(UpdateInput{Capability: fsctl.StorageEngine, DoctorHealthy: true, Compatibility: compat.Evaluation{Quarantine: true}, NativeFallbackReady: true}) + if ready.Allowed || !ready.Quarantine || ready.RequiresNativeFallback { + t.Fatalf("quarantine should remain blocked after fallback: %#v", ready) + } + automatic := EvaluateUpdate(UpdateInput{Capability: fsctl.FSEnginePreview, DoctorHealthy: true, Compatibility: compat.Evaluation{Approved: true}, Automatic: true}) + if automatic.Allowed { + t.Fatalf("preview automatic update should be rejected: %#v", automatic) + } + manual := EvaluateUpdate(UpdateInput{Capability: fsctl.FSEnginePreview, DoctorHealthy: true, Compatibility: compat.Evaluation{Approved: true}, ExplicitPromotion: true}) + if !manual.Allowed { + t.Fatalf("explicit preview promotion should pass: %#v", manual) + } +} + +func TestProcessLockAllowsOnlyOneFilesystemHost(t *testing.T) { + path := filepath.Join(t.TempDir(), "service.lock") + status, err := InspectProcessLock(path) + if err != nil || status.Held { + t.Fatalf("missing process lock status = %#v err=%v", status, err) + } + first, err := AcquireProcessLock(path) + if err != nil { + t.Fatal(err) + } + defer first.Close() + status, err = InspectProcessLock(path) + if err != nil || !status.Held || status.PID != os.Getpid() { + t.Fatalf("held process lock status = %#v err=%v", status, err) + } + + if _, err := AcquireProcessLock(path); err == nil { + t.Fatal("a second filesystem host acquired the same process lock") + } + if err := first.Close(); err != nil { + t.Fatal(err) + } + status, err = InspectProcessLock(path) + if err != nil || status.Held { + t.Fatalf("released process lock status = %#v err=%v", status, err) + } + second, err := AcquireProcessLock(path) + if err != nil { + t.Fatalf("lock was not released after the first host exited: %v", err) + } + if err := second.Close(); err != nil { + t.Fatal(err) + } +} + +type recordingRunner struct { + calls []string + outputs map[string][]byte + errors map[string]error +} + +func (r *recordingRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) { + call := strings.Join(append([]string{name}, args...), " ") + r.calls = append(r.calls, call) + if err, ok := r.errors[call]; ok { + return r.outputs[call], err + } + if output, ok := r.outputs[call]; ok { + return output, nil + } + return nil, nil +} diff --git a/internal/service/systemd.go b/internal/service/systemd.go new file mode 100644 index 0000000..e9c0d8f --- /dev/null +++ b/internal/service/systemd.go @@ -0,0 +1,183 @@ +package service + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "time" +) + +type SystemdManager struct { + Runner Runner + MountProbe func(string) error +} + +func RenderSystemd(options Options) ([]byte, error) { + arguments, err := ServeArguments(options) + if err != nil { + return nil, err + } + execStart := make([]string, 0, len(arguments)+1) + for _, argument := range append([]string{options.BinaryPath}, arguments...) { + quoted, err := quoteSystemdArgument(argument) + if err != nil { + return nil, err + } + execStart = append(execStart, quoted) + } + stdout, err := escapeSystemdSettingValue("append:" + options.StdoutPath) + if err != nil { + return nil, err + } + stderr, err := escapeSystemdSettingValue("append:" + options.StderrPath) + if err != nil { + return nil, err + } + + var output bytes.Buffer + output.WriteString("[Unit]\n") + output.WriteString("Description=CodexFold transparent session filesystem\n") + output.WriteString("After=default.target\n\n") + output.WriteString("[Service]\n") + output.WriteString("Type=simple\n") + output.WriteString("ExecStart=:") + output.WriteString(strings.Join(execStart, " ")) + output.WriteByte('\n') + output.WriteString("Restart=on-failure\n") + output.WriteString("RestartSec=2s\n") + output.WriteString("TimeoutStopSec=30s\n") + output.WriteString("KillMode=mixed\n") + output.WriteString("StandardOutput=") + output.WriteString(stdout) + output.WriteByte('\n') + output.WriteString("StandardError=") + output.WriteString(stderr) + output.WriteString("\n\n[Install]\n") + output.WriteString("WantedBy=default.target\n") + return output.Bytes(), nil +} + +func SystemdUnitName(label string) (string, error) { + if !safeLabel(label) { + return "", errors.New("safe service label is required") + } + return label + ".service", nil +} + +func (m SystemdManager) Start(ctx context.Context, unit string) error { + if !safeSystemdUnit(unit) { + return errors.New("safe systemd service unit is required") + } + if output, err := m.runner().Run(ctx, "systemctl", "--user", "daemon-reload"); err != nil { + return commandFailure("systemctl --user daemon-reload", output, err) + } + if output, err := m.runner().Run(ctx, "systemctl", "--user", "enable", "--now", unit); err != nil { + return commandFailure("systemctl --user enable --now", output, err) + } + return nil +} + +func (m SystemdManager) Stop(ctx context.Context, unit string) error { + if !safeSystemdUnit(unit) { + return errors.New("safe systemd service unit is required") + } + output, err := m.runner().Run(ctx, "systemctl", "--user", "stop", unit) + if err != nil { + return commandFailure("systemctl --user stop", output, err) + } + return nil +} + +func (m SystemdManager) Status(ctx context.Context, unit string, mountPoint string) Status { + result := Status{} + if !safeSystemdUnit(unit) { + result.DaemonError = "safe systemd service unit is required" + } else { + output, err := m.runner().Run(ctx, "systemctl", "--user", "show", unit, "--property=ActiveState", "--property=SubState", "--no-pager") + if err != nil { + result.DaemonError = commandFailure("systemctl --user show", output, err).Error() + } else if systemdStateRunning(output) { + result.DaemonRunning = true + } else { + result.DaemonError = "systemd user service is loaded but not running" + } + } + probe := m.MountProbe + if probe == nil { + probe = ProbeMount + } + if err := probe(mountPoint); err != nil { + result.MountError = err.Error() + } else { + result.MountHealthy = true + } + return result +} + +func (m SystemdManager) WaitHealthy(ctx context.Context, unit string, mountPoint string, timeout time.Duration) (Status, error) { + return waitHealthy(ctx, timeout, func() Status { return m.Status(ctx, unit, mountPoint) }) +} + +func (m SystemdManager) runner() Runner { + if m.Runner != nil { + return m.Runner + } + return ExecRunner{} +} + +func quoteSystemdArgument(value string) (string, error) { + if strings.ContainsAny(value, "\x00\r\n") { + return "", errors.New("systemd arguments cannot contain NUL or newlines") + } + value = strings.ReplaceAll(value, "\\", "\\\\") + value = strings.ReplaceAll(value, "\"", "\\\"") + value = strings.ReplaceAll(value, "%", "%%") + return "\"" + value + "\"", nil +} + +func escapeSystemdSettingValue(value string) (string, error) { + if strings.ContainsAny(value, "\x00\r\n") { + return "", errors.New("systemd setting values cannot contain NUL or newlines") + } + var output strings.Builder + for index := 0; index < len(value); index++ { + character := value[index] + switch { + case character == '%': + output.WriteString("%%") + case character <= 0x20 || character == '\\' || character == '"': + _, _ = fmt.Fprintf(&output, "\\x%02x", character) + default: + output.WriteByte(character) + } + } + return output.String(), nil +} + +func safeSystemdUnit(unit string) bool { + return strings.HasSuffix(unit, ".service") && safeLabel(strings.TrimSuffix(unit, ".service")) +} + +func systemdStateRunning(output []byte) bool { + active := false + running := false + for _, line := range strings.Split(string(output), "\n") { + switch strings.TrimSpace(line) { + case "ActiveState=active": + active = true + case "SubState=running": + running = true + } + } + return active && running +} + +func commandFailure(action string, output []byte, err error) error { + message := strings.TrimSpace(string(output)) + if message == "" { + return fmt.Errorf("%s: %w", action, err) + } + return fmt.Errorf("%s: %w: %s", action, err, message) +} diff --git a/internal/service/systemd_linux_test.go b/internal/service/systemd_linux_test.go new file mode 100644 index 0000000..deb010b --- /dev/null +++ b/internal/service/systemd_linux_test.go @@ -0,0 +1,102 @@ +//go:build linux + +package service + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestRenderSystemdPassesSystemdAnalyzeVerify(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_SYSTEMD_USER_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_SYSTEMD_USER_TEST=1 to run systemd verification") + } + root := filepath.Join(t.TempDir(), "path with space % and $") + binary := filepath.Join(root, "bin", "codexfold") + if err := os.MkdirAll(filepath.Dir(binary), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(binary, []byte("#!/bin/sh\nexit 0\n"), 0o700); err != nil { + t.Fatal(err) + } + definition, err := RenderSystemd(Options{ + Label: "com.codexfold.fs", BinaryPath: binary, + CodexHome: filepath.Join(root, "codex"), StoreDir: filepath.Join(root, "store"), + MountPoint: filepath.Join(root, "mount"), StdoutPath: filepath.Join(root, "logs", "stdout.log"), + StderrPath: filepath.Join(root, "logs", "stderr.log"), CanonicalNamespace: true, + NativeRoot: filepath.Join(root, "native"), + }) + if err != nil { + t.Fatal(err) + } + unitPath := filepath.Join(root, "com.codexfold.fs.service") + if err := os.WriteFile(unitPath, definition, 0o600); err != nil { + t.Fatal(err) + } + if output, err := exec.Command("systemd-analyze", "--user", "verify", unitPath).CombinedOutput(); err != nil { + t.Fatalf("systemd-analyze rejected generated unit: %v\n%s", err, output) + } +} + +func TestRealSystemdUserManagerLifecycle(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_SYSTEMD_USER_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_SYSTEMD_USER_TEST=1 to run systemd lifecycle") + } + home, err := os.UserHomeDir() + if err != nil { + t.Fatal(err) + } + unit := fmt.Sprintf("codexfold-validation-%d.service", os.Getpid()) + unitPath := filepath.Join(home, ".config", "systemd", "user", unit) + if err := os.MkdirAll(filepath.Dir(unitPath), 0o700); err != nil { + t.Fatal(err) + } + definition := []byte("[Unit]\nDescription=CodexFold systemd user validation\n\n[Service]\nType=simple\nExecStart=/usr/bin/sleep infinity\n\n[Install]\nWantedBy=default.target\n") + if err := os.WriteFile(unitPath, definition, 0o600); err != nil { + t.Fatal(err) + } + cleanup := func() { + _, _ = exec.Command("systemctl", "--user", "disable", "--now", unit).CombinedOutput() + _ = os.Remove(unitPath) + _, _ = exec.Command("systemctl", "--user", "daemon-reload").CombinedOutput() + } + t.Cleanup(cleanup) + + manager := SystemdManager{MountProbe: func(string) error { return nil }} + mountPoint := filepath.Join(t.TempDir(), "mount") + if err := manager.Start(context.Background(), unit); err != nil { + t.Fatal(err) + } + status, err := manager.WaitHealthy(context.Background(), unit, mountPoint, 10*time.Second) + if err != nil { + t.Fatal(err) + } + if !status.DaemonRunning || !status.MountHealthy { + t.Fatalf("unexpected running status: %#v", status) + } + if err := manager.Stop(context.Background(), unit); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + status = manager.Status(context.Background(), unit, mountPoint) + if !status.DaemonRunning { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("systemd user unit remained running: %#v", status) +} + +func TestLinuxMountProbeRejectsAnOrdinaryDirectory(t *testing.T) { + err := ProbeMount(t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "not a CodexFold FUSE mount root") { + t.Fatalf("ordinary directory probe error = %v", err) + } +} diff --git a/internal/service/wait.go b/internal/service/wait.go new file mode 100644 index 0000000..fdbfb86 --- /dev/null +++ b/internal/service/wait.go @@ -0,0 +1,31 @@ +package service + +import ( + "context" + "fmt" + "time" +) + +func waitHealthy(ctx context.Context, timeout time.Duration, status func() Status) (Status, error) { + if timeout <= 0 { + timeout = 15 * time.Second + } + deadline := time.NewTimer(timeout) + defer deadline.Stop() + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + var last Status + for { + last = status() + if last.DaemonRunning && last.MountHealthy { + return last, nil + } + select { + case <-ctx.Done(): + return last, ctx.Err() + case <-deadline.C: + return last, fmt.Errorf("filesystem service did not become healthy: daemon=%t mount=%t daemon_error=%q mount_error=%q", last.DaemonRunning, last.MountHealthy, last.DaemonError, last.MountError) + case <-ticker.C: + } + } +} diff --git a/internal/service/windows.go b/internal/service/windows.go new file mode 100644 index 0000000..3d67769 --- /dev/null +++ b/internal/service/windows.go @@ -0,0 +1,197 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "regexp" + "strings" + "time" +) + +const windowsConfigVersion = 1 + +var windowsRunningState = regexp.MustCompile(`(?m)STATE\s*:\s*4\s+RUNNING\b`) + +type WindowsConfig struct { + Version int `json:"version"` + ServiceName string `json:"service_name"` + BinaryPath string `json:"binary_path,omitempty"` + Arguments []string `json:"arguments"` + StdoutPath string `json:"stdout_path"` + StderrPath string `json:"stderr_path"` +} + +type WindowsManager struct { + Runner Runner + MountProbe func(string) error +} + +func RenderWindowsConfig(options Options) ([]byte, error) { + arguments, err := ServeArguments(options) + if err != nil { + return nil, err + } + return json.MarshalIndent(WindowsConfig{ + Version: windowsConfigVersion, ServiceName: options.Label, BinaryPath: options.BinaryPath, Arguments: arguments, + StdoutPath: options.StdoutPath, StderrPath: options.StderrPath, + }, "", " ") +} + +func ParseWindowsConfig(definition []byte) (WindowsConfig, error) { + var config WindowsConfig + if err := json.Unmarshal(definition, &config); err != nil { + return WindowsConfig{}, err + } + if config.Version != windowsConfigVersion { + return WindowsConfig{}, fmt.Errorf("unsupported Windows service config version %d", config.Version) + } + if !safeLabel(config.ServiceName) { + return WindowsConfig{}, errors.New("safe Windows service name is required") + } + if config.BinaryPath == "" || (!filepath.IsAbs(config.BinaryPath) && !absoluteWindowsServicePath(config.BinaryPath)) { + return WindowsConfig{}, errors.New("Windows service config binary path must be absolute") + } + if len(config.Arguments) < 2 || config.Arguments[0] != "fs" || config.Arguments[1] != "serve" { + return WindowsConfig{}, errors.New("Windows service config must run fs serve") + } + for _, path := range []string{config.StdoutPath, config.StderrPath} { + if !filepath.IsAbs(path) { + return WindowsConfig{}, errors.New("Windows service log paths must be absolute") + } + } + return config, nil +} + +func (m WindowsManager) Install(ctx context.Context, name string, binaryPath string, definitionPath string) error { + if !safeLabel(name) { + return errors.New("safe Windows service name is required") + } + if !absoluteWindowsServicePath(binaryPath) || !absoluteWindowsServicePath(definitionPath) { + return errors.New("Windows service binary and definition paths must be absolute") + } + commandLine := windowsServiceCommand(binaryPath, definitionPath) + _, queryErr := m.runner().Run(ctx, "sc.exe", "query", name) + if queryErr == nil { + output, err := m.runner().Run(ctx, "sc.exe", "config", name, "binPath=", commandLine, "start=", "auto") + if err != nil { + return commandFailure("sc.exe config", output, err) + } + } else { + output, err := m.runner().Run(ctx, "sc.exe", "create", name, "binPath=", commandLine, "start=", "auto", "DisplayName=", "CodexFold Transparent Session Filesystem") + if err != nil { + return commandFailure("sc.exe create", output, err) + } + } + if output, err := m.runner().Run(ctx, "sc.exe", "description", name, "CodexFold transparent Codex session filesystem"); err != nil { + return commandFailure("sc.exe description", output, err) + } + if output, err := m.runner().Run(ctx, "sc.exe", "failure", name, "reset=", "86400", "actions=", "restart/5000/restart/15000/\"\"/0"); err != nil { + return commandFailure("sc.exe failure", output, err) + } + return nil +} + +func (m WindowsManager) Start(ctx context.Context, name string) error { + if !safeLabel(name) { + return errors.New("safe Windows service name is required") + } + output, err := m.runner().Run(ctx, "sc.exe", "start", name) + if err != nil { + return commandFailure("sc.exe start", output, err) + } + return nil +} + +func (m WindowsManager) Stop(ctx context.Context, name string) error { + if !safeLabel(name) { + return errors.New("safe Windows service name is required") + } + output, err := m.runner().Run(ctx, "sc.exe", "stop", name) + if err != nil { + return commandFailure("sc.exe stop", output, err) + } + return nil +} + +func (m WindowsManager) Status(ctx context.Context, name string, mountPoint string) Status { + result := Status{} + if !safeLabel(name) { + result.DaemonError = "safe Windows service name is required" + } else { + output, err := m.runner().Run(ctx, "sc.exe", "queryex", name) + if err != nil { + result.DaemonError = commandFailure("sc.exe queryex", output, err).Error() + } else if windowsRunningState.Match(output) { + result.DaemonRunning = true + } else { + result.DaemonError = "Windows service is installed but not running" + } + } + probe := m.MountProbe + if probe == nil { + probe = ProbeMount + } + if err := probe(mountPoint); err != nil { + result.MountError = err.Error() + } else { + result.MountHealthy = true + } + return result +} + +func (m WindowsManager) WaitHealthy(ctx context.Context, name string, mountPoint string, timeout time.Duration) (Status, error) { + return waitHealthy(ctx, timeout, func() Status { return m.Status(ctx, name, mountPoint) }) +} + +func (m WindowsManager) runner() Runner { + if m.Runner != nil { + return m.Runner + } + return ExecRunner{} +} + +func windowsServiceCommand(binaryPath string, definitionPath string) string { + return strings.Join([]string{ + quoteWindowsCommandLineArgument(binaryPath), "fs", "service", "run", "--definition", + quoteWindowsCommandLineArgument(definitionPath), + }, " ") +} + +func quoteWindowsCommandLineArgument(value string) string { + if value != "" && !strings.ContainsAny(value, " \t\n\v\"") { + return value + } + var output strings.Builder + output.WriteByte('"') + backslashes := 0 + for _, character := range value { + switch character { + case '\\': + backslashes++ + case '"': + output.WriteString(strings.Repeat("\\", backslashes*2+1)) + output.WriteRune(character) + backslashes = 0 + default: + output.WriteString(strings.Repeat("\\", backslashes)) + output.WriteRune(character) + backslashes = 0 + } + } + output.WriteString(strings.Repeat("\\", backslashes*2)) + output.WriteByte('"') + return output.String() +} + +func absoluteWindowsServicePath(path string) bool { + if filepath.IsAbs(path) { + return true + } + if len(path) >= 3 && ((path[0] >= 'a' && path[0] <= 'z') || (path[0] >= 'A' && path[0] <= 'Z')) && path[1] == ':' && (path[2] == '\\' || path[2] == '/') { + return true + } + return strings.HasPrefix(path, `\\`) +} diff --git a/internal/sessionns/activation.go b/internal/sessionns/activation.go new file mode 100644 index 0000000..a2bedb1 --- /dev/null +++ b/internal/sessionns/activation.go @@ -0,0 +1,331 @@ +package sessionns + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" +) + +const ( + actionActivate = "activate" + actionDeactivate = "deactivate" +) + +var sessionDirectories = []string{"sessions", "archived_sessions"} + +type Options struct { + Home string + Mount string + NativeRoot string + MountProbe func(string) error +} + +type Result struct { + Active bool `json:"active"` + Recovered bool `json:"recovered"` + Home string `json:"home"` + Mount string `json:"mount"` + NativeRoot string `json:"native_root"` + Journal string `json:"journal"` +} + +type journal struct { + Version int `json:"version"` + Action string `json:"action"` +} + +func Inspect(options Options) (Result, error) { + options, err := validate(options) + if err != nil { + return Result{}, err + } + result := resultFor(options) + activeLinks := 0 + nativeDirectories := 0 + nativeEntries := 0 + for _, name := range sessionDirectories { + homePath := filepath.Join(options.Home, name) + info, err := os.Lstat(homePath) + if err != nil { + return Result{}, fmt.Errorf("inspect %s: %w", homePath, err) + } + if info.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(homePath) + if err != nil || filepath.Clean(target) != filepath.Join(options.Mount, name) { + return Result{}, fmt.Errorf("unexpected namespace link %s", homePath) + } + activeLinks++ + } else if !info.IsDir() { + return Result{}, fmt.Errorf("namespace source is not a directory: %s", homePath) + } + nativePath := filepath.Join(options.NativeRoot, name) + if info, err := os.Stat(nativePath); err == nil && info.IsDir() { + nativeDirectories++ + entries, err := os.ReadDir(nativePath) + if err != nil { + return Result{}, err + } + nativeEntries += len(entries) + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return Result{}, err + } + } + if activeLinks == len(sessionDirectories) && nativeDirectories == len(sessionDirectories) { + result.Active = true + return result, nil + } + if activeLinks == 0 && nativeEntries == 0 { + return result, nil + } + return Result{}, errors.New("session namespace is partially activated") +} + +func Activate(options Options) (Result, error) { + options, err := validate(options) + if err != nil { + return Result{}, err + } + if _, err := os.Stat(journalPath(options)); err == nil { + if _, recoverErr := Recover(options); recoverErr != nil { + return Result{}, recoverErr + } + } else if !errors.Is(err, os.ErrNotExist) { + return Result{}, err + } + if options.MountProbe == nil { + return Result{}, errors.New("mount identity probe is required for namespace activation") + } + if err := options.MountProbe(options.Mount); err != nil { + return Result{}, fmt.Errorf("canonical mount identity is not healthy: %w", err) + } + status, err := Inspect(options) + if err == nil && status.Active { + if err := installRouteGuard(options); err != nil { + return Result{}, err + } + return status, nil + } + if err != nil { + return Result{}, err + } + for _, name := range sessionDirectories { + if info, err := os.Stat(filepath.Join(options.Mount, name)); err != nil || !info.IsDir() { + return Result{}, fmt.Errorf("canonical mount directory is unavailable: %s", filepath.Join(options.Mount, name)) + } + } + if err := os.MkdirAll(options.NativeRoot, 0o700); err != nil { + return Result{}, err + } + for _, name := range sessionDirectories { + if err := removeEmptyDirectory(filepath.Join(options.NativeRoot, name)); err != nil { + return Result{}, err + } + } + if err := writeJournal(options, journal{Version: 1, Action: actionActivate}); err != nil { + return Result{}, err + } + for _, name := range sessionDirectories { + homePath := filepath.Join(options.Home, name) + nativePath := filepath.Join(options.NativeRoot, name) + if err := os.Rename(homePath, nativePath); err != nil { + return rollbackAfterError(options, err) + } + if err := os.Symlink(filepath.Join(options.Mount, name), homePath); err != nil { + return rollbackAfterError(options, err) + } + } + if err := installRouteGuard(options); err != nil { + return rollbackAfterError(options, err) + } + if err := removeJournal(options); err != nil { + return Result{}, err + } + return Inspect(options) +} + +func Deactivate(options Options) (Result, error) { + options, err := validate(options) + if err != nil { + return Result{}, err + } + status, err := Recover(options) + if err != nil { + return Result{}, err + } + if !status.Active { + return status, nil + } + if err := writeJournal(options, journal{Version: 1, Action: actionDeactivate}); err != nil { + return Result{}, err + } + for _, name := range sessionDirectories { + homePath := filepath.Join(options.Home, name) + if err := os.Remove(homePath); err != nil { + return finishAfterError(options, err) + } + if err := os.Rename(filepath.Join(options.NativeRoot, name), homePath); err != nil { + return finishAfterError(options, err) + } + } + if err := removeRouteGuard(options); err != nil { + return finishAfterError(options, err) + } + if err := removeJournal(options); err != nil { + return Result{}, err + } + return Inspect(options) +} + +func Recover(options Options) (Result, error) { + options, err := validate(options) + if err != nil { + return Result{}, err + } + data, err := os.ReadFile(journalPath(options)) + if errors.Is(err, os.ErrNotExist) { + return Inspect(options) + } + if err != nil { + return Result{}, err + } + var transaction journal + if err := json.Unmarshal(data, &transaction); err != nil || transaction.Version != 1 { + return Result{}, errors.New("invalid session namespace journal") + } + if transaction.Action != actionActivate && transaction.Action != actionDeactivate { + return Result{}, errors.New("unknown session namespace journal action") + } + for _, name := range sessionDirectories { + homePath := filepath.Join(options.Home, name) + nativePath := filepath.Join(options.NativeRoot, name) + if info, err := os.Lstat(homePath); err == nil && info.Mode()&os.ModeSymlink != 0 { + if err := os.Remove(homePath); err != nil { + return Result{}, err + } + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return Result{}, err + } + if _, err := os.Lstat(homePath); errors.Is(err, os.ErrNotExist) { + if info, nativeErr := os.Stat(nativePath); nativeErr == nil && info.IsDir() { + if err := os.Rename(nativePath, homePath); err != nil { + return Result{}, err + } + } else if nativeErr != nil && !errors.Is(nativeErr, os.ErrNotExist) { + return Result{}, nativeErr + } + } + } + if err := removeRouteGuard(options); err != nil { + return Result{}, err + } + if err := removeJournal(options); err != nil { + return Result{}, err + } + result, err := Inspect(options) + if err != nil { + return Result{}, err + } + result.Recovered = true + return result, nil +} + +func rollbackAfterError(options Options, cause error) (Result, error) { + _, recoverErr := Recover(options) + if recoverErr != nil { + return Result{}, errors.Join(cause, recoverErr) + } + return Result{}, cause +} + +func finishAfterError(options Options, cause error) (Result, error) { + _, recoverErr := Recover(options) + if recoverErr != nil { + return Result{}, errors.Join(cause, recoverErr) + } + return Result{}, cause +} + +func validate(options Options) (Options, error) { + if !filepath.IsAbs(options.Home) || !filepath.IsAbs(options.Mount) || !filepath.IsAbs(options.NativeRoot) { + return Options{}, errors.New("absolute home, mount, and native root paths are required") + } + options.Home = filepath.Clean(options.Home) + options.Mount = filepath.Clean(options.Mount) + options.NativeRoot = filepath.Clean(options.NativeRoot) + if options.Home == options.Mount || options.Home == options.NativeRoot || options.Mount == options.NativeRoot { + return Options{}, errors.New("home, mount, and native root paths must be distinct") + } + return options, nil +} + +func resultFor(options Options) Result { + return Result{Home: options.Home, Mount: options.Mount, NativeRoot: options.NativeRoot, Journal: journalPath(options)} +} + +func journalPath(options Options) string { + return filepath.Join(options.Home, ".codexfold-namespace.json") +} + +func writeJournal(options Options, transaction journal) error { + data, err := json.Marshal(transaction) + if err != nil { + return err + } + temporary, err := os.CreateTemp(options.Home, ".codexfold-namespace-*.tmp") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return err + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := os.Rename(temporaryPath, journalPath(options)); err != nil { + return err + } + return syncDirectory(options.Home) +} + +func removeJournal(options Options) error { + if err := os.Remove(journalPath(options)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return syncDirectory(options.Home) +} + +func removeEmptyDirectory(path string) error { + entries, err := os.ReadDir(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if len(entries) != 0 { + return fmt.Errorf("native namespace destination is not empty: %s", path) + } + return os.Remove(path) +} + +func syncDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} diff --git a/internal/sessionns/activation_test.go b/internal/sessionns/activation_test.go new file mode 100644 index 0000000..970a051 --- /dev/null +++ b/internal/sessionns/activation_test.go @@ -0,0 +1,259 @@ +package sessionns + +import ( + "database/sql" + "os" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func TestActivateAndDeactivatePreserveSessionTrees(t *testing.T) { + root := t.TempDir() + home := filepath.Join(root, "home") + mount := filepath.Join(root, "mount") + nativeRoot := filepath.Join(home, "fold-native") + writeFixture(t, filepath.Join(home, "sessions", "2026", "07", "12", "active.jsonl"), "active\n") + writeFixture(t, filepath.Join(home, "archived_sessions", "archived.jsonl"), "archived\n") + createStateDatabase(t, home) + for _, directory := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(mount, directory), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(nativeRoot, directory), 0o700); err != nil { + t.Fatal(err) + } + } + + result, err := Activate(Options{Home: home, Mount: mount, NativeRoot: nativeRoot, MountProbe: healthyMountProbe}) + if err != nil { + t.Fatal(err) + } + if !result.Active || result.Recovered { + t.Fatalf("activation result = %#v", result) + } + assertLink(t, filepath.Join(home, "sessions"), filepath.Join(mount, "sessions")) + assertLink(t, filepath.Join(home, "archived_sessions"), filepath.Join(mount, "archived_sessions")) + assertFile(t, filepath.Join(nativeRoot, "sessions", "2026", "07", "12", "active.jsonl"), "active\n") + assertFile(t, filepath.Join(nativeRoot, "archived_sessions", "archived.jsonl"), "archived\n") + + status, err := Inspect(Options{Home: home, Mount: mount, NativeRoot: nativeRoot}) + if err != nil || !status.Active { + t.Fatalf("active status = %#v err=%v", status, err) + } + result, err = Deactivate(Options{Home: home, Mount: mount, NativeRoot: nativeRoot}) + if err != nil { + t.Fatal(err) + } + if result.Active { + t.Fatalf("deactivation result = %#v", result) + } + assertFile(t, filepath.Join(home, "sessions", "2026", "07", "12", "active.jsonl"), "active\n") + assertFile(t, filepath.Join(home, "archived_sessions", "archived.jsonl"), "archived\n") + for _, directory := range []string{"sessions", "archived_sessions"} { + info, err := os.Lstat(filepath.Join(home, directory)) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + t.Fatalf("restored %s = %#v err=%v", directory, info, err) + } + } +} + +func TestActivateRejectsOrdinaryDirectoryThatLooksLikeMount(t *testing.T) { + root := t.TempDir() + home := filepath.Join(root, "home") + mount := filepath.Join(root, "mount") + nativeRoot := filepath.Join(home, "fold-native") + writeFixture(t, filepath.Join(home, "sessions", "active.jsonl"), "active\n") + writeFixture(t, filepath.Join(home, "archived_sessions", "archived.jsonl"), "archived\n") + for _, directory := range []string{"sessions", "archived_sessions"} { + if err := os.MkdirAll(filepath.Join(mount, directory), 0o700); err != nil { + t.Fatal(err) + } + } + + _, err := Activate(Options{ + Home: home, Mount: mount, NativeRoot: nativeRoot, + MountProbe: func(string) error { return os.ErrInvalid }, + }) + if err == nil { + t.Fatal("activation must reject an ordinary directory even when it has canonical subdirectories") + } + assertFile(t, filepath.Join(home, "sessions", "active.jsonl"), "active\n") + assertFile(t, filepath.Join(home, "archived_sessions", "archived.jsonl"), "archived\n") +} + +func TestActiveNamespaceNormalizesDesktopMountAliasesInStateDatabase(t *testing.T) { + root := t.TempDir() + home := filepath.Join(root, "home") + mount := filepath.Join(home, "fold-fs") + nativeRoot := filepath.Join(home, "fold-native") + activeRoute := filepath.Join(home, "sessions", "2026", "07", "13", "rollout-session.jsonl") + writeFixture(t, activeRoute, "active\n") + if err := os.MkdirAll(filepath.Join(home, "archived_sessions"), 0o700); err != nil { + t.Fatal(err) + } + for _, directory := range sessionDirectories { + if err := os.MkdirAll(filepath.Join(mount, directory), 0o700); err != nil { + t.Fatal(err) + } + } + db, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`create table threads (id text primary key, rollout_path text not null); insert into threads values ('session', ?)`, activeRoute); err != nil { + _ = db.Close() + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + + options := Options{Home: home, Mount: mount, NativeRoot: nativeRoot, MountProbe: healthyMountProbe} + if _, err := Activate(options); err != nil { + t.Fatal(err) + } + db, err = sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + mountAlias := filepath.Join(mount, "sessions", "2026", "07", "13", "rollout-session.jsonl") + if _, err := db.Exec(`update threads set rollout_path = ? where id = 'session'`, mountAlias); err != nil { + _ = db.Close() + t.Fatal(err) + } + var normalized string + if err := db.QueryRow(`select rollout_path from threads where id = 'session'`).Scan(&normalized); err != nil { + _ = db.Close() + t.Fatal(err) + } + if filepath.Clean(normalized) != filepath.Clean(activeRoute) { + _ = db.Close() + t.Fatalf("normalized route = %q, want %q", normalized, activeRoute) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + + if _, err := Deactivate(options); err != nil { + t.Fatal(err) + } + db, err = sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var triggers int + if err := db.QueryRow(`select count(*) from sqlite_master where type = 'trigger' and name like 'codexfold_normalize_rollout_path_%'`).Scan(&triggers); err != nil { + t.Fatal(err) + } + if triggers != 0 { + t.Fatalf("route normalization triggers remained after deactivation: %d", triggers) + } +} + +func TestRouteGuardNormalizesMountAliasesWithUnicodePaths(t *testing.T) { + home := filepath.Join(t.TempDir(), "用户", ".codex") + mount := filepath.Join(home, "fold-fs") + if err := os.MkdirAll(home, 0o700); err != nil { + t.Fatal(err) + } + createStateDatabase(t, home) + options := Options{Home: home, Mount: mount, NativeRoot: filepath.Join(home, "fold-native")} + if err := installRouteGuard(options); err != nil { + t.Fatal(err) + } + database, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + defer database.Close() + mountAlias := filepath.Join(mount, "archived_sessions", "rollout-session.jsonl") + if _, err := database.Exec(`insert into threads values ('session', ?)`, mountAlias); err != nil { + t.Fatal(err) + } + var normalized string + if err := database.QueryRow(`select rollout_path from threads where id = 'session'`).Scan(&normalized); err != nil { + t.Fatal(err) + } + want := filepath.Join(home, "archived_sessions", "rollout-session.jsonl") + if filepath.Clean(normalized) != filepath.Clean(want) { + t.Fatalf("normalized Unicode route = %q, want %q", normalized, want) + } +} + +func TestRecoverRollsBackInterruptedActivation(t *testing.T) { + root := t.TempDir() + home := filepath.Join(root, "home") + mount := filepath.Join(root, "mount") + nativeRoot := filepath.Join(home, "fold-native") + writeFixture(t, filepath.Join(home, "sessions", "active.jsonl"), "active\n") + writeFixture(t, filepath.Join(home, "archived_sessions", "archived.jsonl"), "archived\n") + if err := os.MkdirAll(nativeRoot, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(filepath.Join(home, "sessions"), filepath.Join(nativeRoot, "sessions")); err != nil { + t.Fatal(err) + } + options := Options{Home: home, Mount: mount, NativeRoot: nativeRoot} + if err := writeJournal(options, journal{Version: 1, Action: actionActivate}); err != nil { + t.Fatal(err) + } + + result, err := Recover(options) + if err != nil { + t.Fatal(err) + } + if result.Active || !result.Recovered { + t.Fatalf("recovery result = %#v", result) + } + assertFile(t, filepath.Join(home, "sessions", "active.jsonl"), "active\n") + assertFile(t, filepath.Join(home, "archived_sessions", "archived.jsonl"), "archived\n") + if _, err := os.Stat(journalPath(options)); !os.IsNotExist(err) { + t.Fatalf("journal remained after recovery: %v", err) + } +} + +func writeFixture(t *testing.T, path string, contents string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } +} + +func assertFile(t *testing.T, path string, want string) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil || string(got) != want { + t.Fatalf("file %s = %q err=%v", path, got, err) + } +} + +func assertLink(t *testing.T, path string, want string) { + t.Helper() + got, err := os.Readlink(path) + if err != nil || filepath.Clean(got) != filepath.Clean(want) { + t.Fatalf("link %s = %q err=%v", path, got, err) + } +} + +func healthyMountProbe(string) error { return nil } + +func createStateDatabase(t *testing.T, home string) { + t.Helper() + database, err := sql.Open("sqlite", filepath.Join(home, "state_5.sqlite")) + if err != nil { + t.Fatal(err) + } + if _, err := database.Exec(`create table threads (id text primary key, rollout_path text not null)`); err != nil { + _ = database.Close() + t.Fatal(err) + } + if err := database.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/sessionns/routing_guard.go b/internal/sessionns/routing_guard.go new file mode 100644 index 0000000..6a94797 --- /dev/null +++ b/internal/sessionns/routing_guard.go @@ -0,0 +1,125 @@ +package sessionns + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + _ "modernc.org/sqlite" +) + +const ( + routeInsertTrigger = "codexfold_normalize_rollout_path_insert" + routeUpdateTrigger = "codexfold_normalize_rollout_path_update" +) + +func installRouteGuard(options Options) error { + return updateRouteGuard(options, true) +} + +func removeRouteGuard(options Options) error { + return updateRouteGuard(options, false) +} + +func updateRouteGuard(options Options, install bool) error { + databasePath := filepath.Join(options.Home, "state_5.sqlite") + if _, err := os.Stat(databasePath); err != nil { + if !install && errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("locate Codex state database: %w", err) + } + database, err := sql.Open("sqlite", databasePath) + if err != nil { + return fmt.Errorf("open Codex state database: %w", err) + } + defer database.Close() + connection, err := database.Conn(context.Background()) + if err != nil { + return err + } + defer connection.Close() + if _, err := connection.ExecContext(context.Background(), `pragma busy_timeout = 10000`); err != nil { + return err + } + if _, err := connection.ExecContext(context.Background(), `begin immediate`); err != nil { + return fmt.Errorf("begin route guard transaction: %w", err) + } + committed := false + defer func() { + if !committed { + _, _ = connection.ExecContext(context.Background(), `rollback`) + } + }() + for _, name := range []string{routeInsertTrigger, routeUpdateTrigger} { + if _, err := connection.ExecContext(context.Background(), `drop trigger if exists `+name); err != nil { + return err + } + } + if install { + for _, statement := range routeGuardTriggerStatements(options) { + if _, err := connection.ExecContext(context.Background(), statement); err != nil { + return fmt.Errorf("install Codex route guard: %w", err) + } + } + } + if _, err := connection.ExecContext(context.Background(), normalizeExistingRoutesStatement(options)); err != nil { + return fmt.Errorf("normalize existing Codex routes: %w", err) + } + if _, err := connection.ExecContext(context.Background(), `commit`); err != nil { + return fmt.Errorf("commit route guard transaction: %w", err) + } + committed = true + return nil +} + +func routeGuardTriggerStatements(options Options) []string { + body := routeGuardBody(options) + condition := routeGuardCondition(options, "NEW.rollout_path") + return []string{ + fmt.Sprintf(`create trigger %s after insert on threads when %s begin %s end`, routeInsertTrigger, condition, body), + fmt.Sprintf(`create trigger %s after update of rollout_path on threads when %s begin %s end`, routeUpdateTrigger, condition, body), + } +} + +func routeGuardBody(options Options) string { + return fmt.Sprintf(`update threads set rollout_path = %s where id = NEW.id;`, routeGuardCase(options, "NEW.rollout_path")) +} + +func normalizeExistingRoutesStatement(options Options) string { + return fmt.Sprintf(`update threads set rollout_path = %s where %s`, routeGuardCase(options, "rollout_path"), routeGuardCondition(options, "rollout_path")) +} + +func routeGuardCase(options Options, value string) string { + activeMount := filepath.Join(options.Mount, "sessions") + string(filepath.Separator) + archiveMount := filepath.Join(options.Mount, "archived_sessions") + string(filepath.Separator) + activeHome := filepath.Join(options.Home, "sessions") + string(filepath.Separator) + archiveHome := filepath.Join(options.Home, "archived_sessions") + string(filepath.Separator) + activeMountSQL := quoteSQLString(activeMount) + archiveMountSQL := quoteSQLString(archiveMount) + return fmt.Sprintf( + `case when substr(%s, 1, length(%s)) = %s then %s || substr(%s, length(%s) + 1) else %s || substr(%s, length(%s) + 1) end`, + value, activeMountSQL, activeMountSQL, quoteSQLString(activeHome), value, activeMountSQL, + quoteSQLString(archiveHome), value, archiveMountSQL, + ) +} + +func routeGuardCondition(options Options, value string) string { + activeMount := filepath.Join(options.Mount, "sessions") + string(filepath.Separator) + archiveMount := filepath.Join(options.Mount, "archived_sessions") + string(filepath.Separator) + activeMountSQL := quoteSQLString(activeMount) + archiveMountSQL := quoteSQLString(archiveMount) + return fmt.Sprintf( + `substr(%s, 1, length(%s)) = %s or substr(%s, 1, length(%s)) = %s`, + value, activeMountSQL, activeMountSQL, + value, archiveMountSQL, archiveMountSQL, + ) +} + +func quoteSQLString(value string) string { + return "'" + strings.ReplaceAll(value, "'", "''") + "'" +} diff --git a/internal/storage/accounting.go b/internal/storage/accounting.go new file mode 100644 index 0000000..87da407 --- /dev/null +++ b/internal/storage/accounting.go @@ -0,0 +1,32 @@ +package storage + +import "context" + +type MutationAccounting struct { + Before Inventory `json:"before"` + Budget BudgetReport `json:"budget"` + After Inventory `json:"after"` + ProjectedReclaimableBytes int64 `json:"projected_reclaimable_bytes"` + ActualReclaimedBytes int64 `json:"actual_reclaimed_bytes"` + AfterInventoryError string `json:"after_inventory_error,omitempty"` +} + +func CompleteAccounting(ctx context.Context, assessment Assessment, storeDir string) *MutationAccounting { + accounting := &MutationAccounting{ + Before: assessment.Inventory, Budget: assessment.Budget, + ProjectedReclaimableBytes: assessment.Budget.ProjectedReclaimableBytes, + } + if storeDir == "" { + return accounting + } + after, err := Scan(ctx, Options{StoreDir: storeDir, AllowMetadataIssues: true}) + if err != nil { + accounting.AfterInventoryError = err.Error() + return accounting + } + accounting.After = after + if assessment.Inventory.StoreDir != "" && assessment.Inventory.TotalPhysicalBytes > after.TotalPhysicalBytes { + accounting.ActualReclaimedBytes = assessment.Inventory.TotalPhysicalBytes - after.TotalPhysicalBytes + } + return accounting +} diff --git a/internal/storage/budget.go b/internal/storage/budget.go new file mode 100644 index 0000000..e94e8c9 --- /dev/null +++ b/internal/storage/budget.go @@ -0,0 +1,103 @@ +package storage + +import ( + "errors" + "fmt" + "math" +) + +var ErrBudgetExceeded = errors.New("storage budget exceeded") + +type RejectionReason string + +const ( + RejectionPhysicalBudget RejectionReason = "physical-budget" + RejectionTemporaryBudget RejectionReason = "temporary-budget" + RejectionFreeSpaceReserve RejectionReason = "free-space-reserve" +) + +type Limits struct { + MaxPhysicalBytes int64 `json:"max_physical_bytes"` + MaxTemporaryBytes int64 `json:"max_temporary_bytes"` + FreeSpaceReserveBytes int64 `json:"free_space_reserve_bytes"` +} + +type BudgetRequest struct { + Operation string `json:"operation"` + CurrentPhysicalBytes int64 `json:"current_physical_bytes"` + AdditionalPersistentBytes int64 `json:"additional_persistent_bytes"` + TemporaryBytes int64 `json:"temporary_bytes"` + TemporaryPersistentOverlapBytes int64 `json:"temporary_persistent_overlap_bytes"` + ReclaimableBytes int64 `json:"reclaimable_bytes"` + AvailableBytes int64 `json:"available_bytes"` +} + +type BudgetReport struct { + Operation string `json:"operation"` + Allowed bool `json:"allowed"` + CurrentPhysicalBytes int64 `json:"current_physical_bytes"` + ProjectedPeakBytes int64 `json:"projected_peak_bytes"` + ProjectedFinalBytes int64 `json:"projected_final_bytes"` + ProjectedReclaimableBytes int64 `json:"projected_reclaimable_bytes"` + AvailableBytes int64 `json:"available_bytes"` + FreeSpaceAfterPeak int64 `json:"free_space_after_peak"` + Rejections []RejectionReason `json:"rejections,omitempty"` +} + +func CheckBudget(request BudgetRequest, limits Limits) (BudgetReport, error) { + if request.Operation == "" { + return BudgetReport{}, errors.New("storage budget operation is required") + } + if request.CurrentPhysicalBytes < 0 || request.AdditionalPersistentBytes < 0 || request.TemporaryBytes < 0 || request.TemporaryPersistentOverlapBytes < 0 || request.ReclaimableBytes < 0 || request.AvailableBytes < 0 { + return BudgetReport{}, errors.New("storage budget byte counts cannot be negative") + } + if request.TemporaryPersistentOverlapBytes > request.TemporaryBytes || request.TemporaryPersistentOverlapBytes > request.AdditionalPersistentBytes { + return BudgetReport{}, errors.New("temporary and persistent overlap exceeds the projected bytes") + } + if limits.MaxPhysicalBytes < 0 || limits.MaxTemporaryBytes < 0 || limits.FreeSpaceReserveBytes < 0 { + return BudgetReport{}, errors.New("storage limits cannot be negative") + } + additional, overflow := addBudgetBytes(request.AdditionalPersistentBytes, request.TemporaryBytes) + if overflow { + return BudgetReport{}, errors.New("storage budget additional byte projection overflow") + } + additional -= request.TemporaryPersistentOverlapBytes + peak, overflow := addBudgetBytes(request.CurrentPhysicalBytes, additional) + if overflow { + return BudgetReport{}, errors.New("storage budget peak byte projection overflow") + } + beforeReclaim, overflow := addBudgetBytes(request.CurrentPhysicalBytes, request.AdditionalPersistentBytes) + if overflow { + return BudgetReport{}, errors.New("storage budget final byte projection overflow") + } + final := beforeReclaim - min(request.ReclaimableBytes, beforeReclaim) + freeAfterPeak := request.AvailableBytes - additional + report := BudgetReport{ + Operation: request.Operation, Allowed: true, + CurrentPhysicalBytes: request.CurrentPhysicalBytes, + ProjectedPeakBytes: peak, ProjectedFinalBytes: final, + ProjectedReclaimableBytes: request.ReclaimableBytes, + AvailableBytes: request.AvailableBytes, FreeSpaceAfterPeak: freeAfterPeak, + } + if limits.MaxPhysicalBytes > 0 && peak > limits.MaxPhysicalBytes { + report.Rejections = append(report.Rejections, RejectionPhysicalBudget) + } + if limits.MaxTemporaryBytes > 0 && request.TemporaryBytes > limits.MaxTemporaryBytes { + report.Rejections = append(report.Rejections, RejectionTemporaryBudget) + } + if freeAfterPeak < limits.FreeSpaceReserveBytes { + report.Rejections = append(report.Rejections, RejectionFreeSpaceReserve) + } + if len(report.Rejections) != 0 { + report.Allowed = false + return report, fmt.Errorf("%w: %s rejected by %v", ErrBudgetExceeded, request.Operation, report.Rejections) + } + return report, nil +} + +func addBudgetBytes(left int64, right int64) (int64, bool) { + if left > math.MaxInt64-right { + return 0, true + } + return left + right, false +} diff --git a/internal/storage/budget_test.go b/internal/storage/budget_test.go new file mode 100644 index 0000000..81902d7 --- /dev/null +++ b/internal/storage/budget_test.go @@ -0,0 +1,105 @@ +package storage + +import ( + "errors" + "testing" +) + +func TestCheckBudgetCalculatesPeakWithoutSubtractingFutureReclamation(t *testing.T) { + report, err := CheckBudget(BudgetRequest{ + Operation: "compact", + CurrentPhysicalBytes: 100, + AdditionalPersistentBytes: 30, + TemporaryBytes: 80, + ReclaimableBytes: 70, + AvailableBytes: 1_000, + }, Limits{ + MaxPhysicalBytes: 500, + MaxTemporaryBytes: 100, + FreeSpaceReserveBytes: 200, + }) + if err != nil { + t.Fatalf("CheckBudget: %v", err) + } + if !report.Allowed || report.ProjectedPeakBytes != 210 || report.ProjectedFinalBytes != 60 || report.ProjectedReclaimableBytes != 70 { + t.Fatalf("unexpected report: %#v", report) + } + if report.FreeSpaceAfterPeak != 890 { + t.Fatalf("free space after peak = %d, want 890", report.FreeSpaceAfterPeak) + } +} + +func TestCheckBudgetDoesNotDoubleCountTemporaryBytesThatBecomePersistent(t *testing.T) { + report, err := CheckBudget(BudgetRequest{ + Operation: "copy-on-write", + CurrentPhysicalBytes: 100, + AdditionalPersistentBytes: 80, + TemporaryBytes: 80, + TemporaryPersistentOverlapBytes: 80, + AvailableBytes: 1_000, + }, Limits{}) + if err != nil { + t.Fatalf("CheckBudget: %v", err) + } + if report.ProjectedPeakBytes != 180 || report.ProjectedFinalBytes != 180 || report.FreeSpaceAfterPeak != 920 { + t.Fatalf("temporary rename projection = %#v", report) + } +} + +func TestCheckBudgetRejectsEveryHardLimit(t *testing.T) { + tests := []struct { + name string + request BudgetRequest + limits Limits + reason RejectionReason + }{ + { + name: "physical footprint", + request: BudgetRequest{ + Operation: "pack", CurrentPhysicalBytes: 90, AdditionalPersistentBytes: 20, + AvailableBytes: 1_000, + }, + limits: Limits{MaxPhysicalBytes: 100}, + reason: RejectionPhysicalBudget, + }, + { + name: "temporary bytes", + request: BudgetRequest{ + Operation: "rollback", CurrentPhysicalBytes: 20, TemporaryBytes: 81, + AvailableBytes: 1_000, + }, + limits: Limits{MaxTemporaryBytes: 80}, + reason: RejectionTemporaryBudget, + }, + { + name: "free space reserve", + request: BudgetRequest{ + Operation: "migrate", CurrentPhysicalBytes: 20, AdditionalPersistentBytes: 30, TemporaryBytes: 40, + AvailableBytes: 100, + }, + limits: Limits{FreeSpaceReserveBytes: 31}, + reason: RejectionFreeSpaceReserve, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + report, err := CheckBudget(test.request, test.limits) + if !errors.Is(err, ErrBudgetExceeded) { + t.Fatalf("error = %v, want ErrBudgetExceeded", err) + } + if report.Allowed || len(report.Rejections) != 1 || report.Rejections[0] != test.reason { + t.Fatalf("unexpected rejection report: %#v", report) + } + }) + } +} + +func TestCheckBudgetRejectsInvalidOrOverflowingProjections(t *testing.T) { + if _, err := CheckBudget(BudgetRequest{Operation: "fold", CurrentPhysicalBytes: -1}, Limits{}); err == nil { + t.Fatal("negative current bytes should fail") + } + if _, err := CheckBudget(BudgetRequest{Operation: "fold", CurrentPhysicalBytes: int64(^uint64(0) >> 1), TemporaryBytes: 1, AvailableBytes: 10}, Limits{}); err == nil { + t.Fatal("overflowing peak projection should fail") + } +} diff --git a/internal/storage/gc.go b/internal/storage/gc.go new file mode 100644 index 0000000..4bf1ec7 --- /dev/null +++ b/internal/storage/gc.go @@ -0,0 +1,699 @@ +package storage + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +type CandidateKind string + +const ( + CandidatePackGeneration CandidateKind = "pack-generation" + CandidateManifestGeneration CandidateKind = "manifest-generation" + CandidateSessionGeneration CandidateKind = "session-generation" + CandidateRetiredState CandidateKind = "retired-state" + CandidateTemporary CandidateKind = "unowned-temporary" +) + +type GCCandidate struct { + Kind CandidateKind `json:"kind"` + Path string `json:"path"` + Files int `json:"files"` + ApparentBytes int64 `json:"apparent_bytes"` + PhysicalBytes int64 `json:"physical_bytes"` + ReclaimableBytes int64 `json:"reclaimable_bytes"` +} + +type GCOptions struct { + StoreDir string + Apply bool + TemporaryGrace time.Duration + KeepPackGenerations int + KeepManifestGenerations int + KeepRetiredPerSession int + Now func() time.Time +} + +type StorageGCResult struct { + StoreDir string `json:"store_dir"` + DryRun bool `json:"dry_run"` + Before Inventory `json:"before"` + After Inventory `json:"after"` + Candidates []GCCandidate `json:"candidates,omitempty"` + CandidateCount int `json:"candidate_count"` + CandidateApparentBytes int64 `json:"candidate_apparent_bytes"` + ProjectedReclaimableBytes int64 `json:"projected_reclaimable_bytes"` + RemovedCount int `json:"removed_count"` + RemovedApparentBytes int64 `json:"removed_apparent_bytes"` + ActualReclaimedBytes int64 `json:"actual_reclaimed_bytes"` +} + +type gcBuilder struct { + ctx context.Context + options GCOptions + scanner *scanner + candidates map[string]CandidateKind +} + +type generationEntry struct { + path string + name string + modTime time.Time + sequence uint64 + sequenced bool +} + +func Collect(ctx context.Context, options GCOptions) (StorageGCResult, error) { + if options.StoreDir == "" { + return StorageGCResult{}, errors.New("storage GC store directory is required") + } + if options.TemporaryGrace < 0 || options.KeepPackGenerations < 0 || options.KeepManifestGenerations < 0 || options.KeepRetiredPerSession < 0 { + return StorageGCResult{}, errors.New("storage GC retention values cannot be negative") + } + if options.TemporaryGrace == 0 { + options.TemporaryGrace = time.Hour + } + if options.KeepPackGenerations == 0 { + options.KeepPackGenerations = 2 + } + if options.KeepManifestGenerations == 0 { + options.KeepManifestGenerations = 2 + } + if options.KeepRetiredPerSession == 0 { + options.KeepRetiredPerSession = 1 + } + if options.Now == nil { + options.Now = time.Now + } + store := cleanAbsolutePath(options.StoreDir) + before, err := Scan(ctx, Options{StoreDir: store, AllowMetadataIssues: true}) + if err != nil { + return StorageGCResult{}, err + } + result := StorageGCResult{StoreDir: store, DryRun: !options.Apply, Before: before, After: before} + metadata, exists, err := prepareScanner(ctx, Options{StoreDir: store, AllowMetadataIssues: true}) + if err != nil { + return StorageGCResult{}, err + } + if !exists { + return result, nil + } + builder := &gcBuilder{ctx: ctx, options: options, scanner: metadata, candidates: make(map[string]CandidateKind)} + if err := builder.discoverPackGenerations(); err != nil { + return StorageGCResult{}, err + } + if err := builder.discoverManifestGenerations(); err != nil { + return StorageGCResult{}, err + } + if err := builder.discoverSessionGenerations(); err != nil { + return StorageGCResult{}, err + } + if err := builder.discoverRetiredState(); err != nil { + return StorageGCResult{}, err + } + if err := builder.discoverTemporaryFiles(); err != nil { + return StorageGCResult{}, err + } + candidates, projected, err := describeCandidates(builder.candidates) + if err != nil { + return StorageGCResult{}, err + } + result.Candidates = candidates + result.CandidateCount = len(candidates) + result.ProjectedReclaimableBytes = projected + for _, candidate := range candidates { + result.CandidateApparentBytes += candidate.ApparentBytes + } + if !options.Apply { + return result, nil + } + for _, candidate := range candidates { + if err := ctx.Err(); err != nil { + return result, err + } + allowed, err := builder.revalidate(candidate) + if err != nil { + return result, err + } + if !allowed { + continue + } + if _, err := os.Lstat(candidate.Path); errors.Is(err, os.ErrNotExist) { + continue + } else if err != nil { + return result, err + } + if err := os.RemoveAll(candidate.Path); err != nil { + return result, fmt.Errorf("remove storage GC candidate %s: %w", candidate.Path, err) + } + result.RemovedCount++ + result.RemovedApparentBytes += candidate.ApparentBytes + } + after, err := Scan(ctx, Options{StoreDir: store, AllowMetadataIssues: true}) + if err != nil { + return result, err + } + result.After = after + if before.TotalPhysicalBytes > after.TotalPhysicalBytes { + result.ActualReclaimedBytes = before.TotalPhysicalBytes - after.TotalPhysicalBytes + } + return result, nil +} + +func (b *gcBuilder) add(path string, kind CandidateKind) error { + path = cleanAbsolutePath(path) + if !pathWithin(b.scanner.store, path) || path == b.scanner.store { + return errors.New("storage GC candidate escapes the store") + } + for existing := range b.candidates { + if pathWithin(existing, path) { + return nil + } + if pathWithin(path, existing) { + delete(b.candidates, existing) + } + } + b.candidates[path] = kind + return nil +} + +func (b *gcBuilder) covered(path string) bool { + for candidate := range b.candidates { + if pathWithin(candidate, path) { + return true + } + } + return false +} + +func (b *gcBuilder) discoverPackGenerations() error { + root := filepath.Join(b.scanner.store, "packs") + entries, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if b.scanner.currentPack == "" { + return nil + } + var generations []generationEntry + for _, entry := range entries { + if !entry.IsDir() || strings.HasPrefix(entry.Name(), ".") { + continue + } + info, err := entry.Info() + if err != nil { + return err + } + generations = append(generations, generationEntry{path: filepath.Join(root, entry.Name()), name: entry.Name(), modTime: info.ModTime()}) + } + sortGenerationEntries(generations) + previousToKeep := max(0, b.options.KeepPackGenerations-1) + for _, generation := range generations { + if generation.name == b.scanner.currentPack { + continue + } + active, err := DirectoryHasActiveLease(filepath.Join(generation.path, "leases"), false) + if err != nil { + return err + } + if active { + continue + } + if previousToKeep > 0 { + previousToKeep-- + continue + } + if err := b.add(generation.path, CandidatePackGeneration); err != nil { + return err + } + } + return nil +} + +func (b *gcBuilder) discoverManifestGenerations() error { + root := filepath.Join(b.scanner.store, "manifests", "generations") + entries, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + sessionID := entry.Name() + directory := filepath.Join(root, sessionID) + if _, ok := b.scanner.managedStates[sessionID]; ok { + maintenance, err := b.sessionMaintenanceActive(sessionID) + if err != nil { + return err + } + if maintenance || b.scanner.journalPending[filepath.Join(b.scanner.store, "fs", "sessions", sessionID)] { + continue + } + } + files, err := os.ReadDir(directory) + if err != nil { + return err + } + var generations []generationEntry + for _, file := range files { + if file.IsDir() || filepath.Ext(file.Name()) != ".json" { + continue + } + info, err := file.Info() + if err != nil { + return err + } + sequence, sequenced := manifestGenerationSequence(file.Name()) + generations = append(generations, generationEntry{path: filepath.Join(directory, file.Name()), name: file.Name(), modTime: info.ModTime(), sequence: sequence, sequenced: sequenced}) + } + if len(generations) <= 1 { + continue + } + current := "" + currentSequence := uint64(0) + currentSequenced := false + if state, ok := b.scanner.managedStates[sessionID]; ok { + current = cleanAbsolutePath(state.ManifestPath) + if pathWithin(directory, current) { + currentSequence, currentSequenced = manifestGenerationSequence(filepath.Base(current)) + } + } else if _, ok := b.scanner.primaryManifests[sessionID]; ok { + current = filepath.Join(b.scanner.store, "manifests", sessionID+".json") + } else { + continue + } + sortGenerationEntries(generations) + previousToKeep := max(0, b.options.KeepManifestGenerations-1) + for _, generation := range generations { + if generation.path == current { + continue + } + if currentSequenced { + if !generation.sequenced { + continue + } + if generation.sequence > currentSequence { + if err := b.add(generation.path, CandidateManifestGeneration); err != nil { + return err + } + continue + } + } + if previousToKeep > 0 { + previousToKeep-- + continue + } + if err := b.add(generation.path, CandidateManifestGeneration); err != nil { + return err + } + } + } + return nil +} + +func (b *gcBuilder) sessionMaintenanceActive(sessionID string) (bool, error) { + directory := filepath.Join(b.scanner.store, "fs", "sessions", sessionID) + writerActive, err := FileHasActiveLock(filepath.Join(directory, "writer.lease")) + if err != nil { + return false, err + } + readerActive, err := treeHasActiveLease(filepath.Join(directory, "leases")) + if err != nil { + return false, err + } + return writerActive || readerActive, nil +} + +func manifestGenerationSequence(name string) (uint64, bool) { + if filepath.Ext(name) != ".json" { + return 0, false + } + sequence, err := strconv.ParseUint(strings.TrimSuffix(name, ".json"), 10, 64) + return sequence, err == nil +} + +func (b *gcBuilder) discoverSessionGenerations() error { + for sessionID, state := range b.scanner.managedStates { + if err := b.ctx.Err(); err != nil { + return err + } + directory := filepath.Join(b.scanner.store, "fs", "sessions", sessionID) + writerActive, err := FileHasActiveLock(filepath.Join(directory, "writer.lease")) + if err != nil { + return err + } + readerActive, err := treeHasActiveLease(filepath.Join(directory, "leases")) + if err != nil { + return err + } + if writerActive || readerActive || b.scanner.journalPending[filepath.Clean(directory)] { + continue + } + current := map[string]struct{}{cleanAbsolutePath(state.DeltaPath): {}} + if state.BackingPath != "" { + current[cleanAbsolutePath(state.BackingPath)] = struct{}{} + } + entries, err := os.ReadDir(directory) + if err != nil { + return err + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + path := filepath.Join(directory, entry.Name()) + if _, keep := current[path]; keep { + continue + } + if _, owned := b.scanner.journalOwned[path]; owned { + continue + } + if isSessionGenerationData(b.scanner.store, path) { + if err := b.add(path, CandidateSessionGeneration); err != nil { + return err + } + } + } + } + return nil +} + +func (b *gcBuilder) discoverRetiredState() error { + root := filepath.Join(b.scanner.store, "fs", "retired") + entries, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + groups := make(map[string][]generationEntry) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + directory := filepath.Join(root, entry.Name()) + data, err := os.ReadFile(filepath.Join(directory, "state.json")) + if err != nil { + continue + } + var state struct { + SessionID string `json:"session_id"` + } + if json.Unmarshal(data, &state) != nil || state.SessionID == "" { + continue + } + active, err := treeHasActiveLease(filepath.Join(directory, "leases")) + if err != nil { + return err + } + pending, err := journalPendingAt(directory) + if err != nil { + return err + } + if active || pending { + continue + } + info, err := entry.Info() + if err != nil { + return err + } + groups[state.SessionID] = append(groups[state.SessionID], generationEntry{path: directory, name: entry.Name(), modTime: info.ModTime()}) + } + for _, states := range groups { + sortGenerationEntries(states) + for index := b.options.KeepRetiredPerSession; index < len(states); index++ { + if err := b.add(states[index].path, CandidateRetiredState); err != nil { + return err + } + } + } + return nil +} + +func (b *gcBuilder) discoverTemporaryFiles() error { + cutoff := b.options.Now().Add(-b.options.TemporaryGrace) + retiredRoot := filepath.Join(b.scanner.store, "fs", "retired") + return filepath.WalkDir(b.scanner.store, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := b.ctx.Err(); err != nil { + return err + } + if path == b.scanner.store { + return nil + } + if b.covered(path) { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if pathWithin(retiredRoot, path) { + if entry.IsDir() && path != retiredRoot { + return filepath.SkipDir + } + return nil + } + if _, owned := b.scanner.journalOwned[cleanAbsolutePath(path)]; owned { + return nil + } + if !isUnownedTemporary(path) { + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + if info.ModTime().After(cutoff) { + return nil + } + if err := b.add(path, CandidateTemporary); err != nil { + return err + } + if entry.IsDir() { + return filepath.SkipDir + } + return nil + }) +} + +func (b *gcBuilder) revalidate(candidate GCCandidate) (bool, error) { + switch candidate.Kind { + case CandidatePackGeneration: + data, err := os.ReadFile(filepath.Join(b.scanner.store, "packs", "CURRENT")) + if err != nil { + return false, err + } + if filepath.Base(candidate.Path) == strings.TrimSpace(string(data)) { + return false, nil + } + active, err := DirectoryHasActiveLease(filepath.Join(candidate.Path, "leases"), false) + return !active, err + case CandidateSessionGeneration: + directory := filepath.Dir(candidate.Path) + stateData, err := os.ReadFile(filepath.Join(directory, "state.json")) + if err != nil { + return false, err + } + var state stateRecord + if err := json.Unmarshal(stateData, &state); err != nil { + return false, err + } + if cleanAbsolutePath(state.DeltaPath) == candidate.Path || (state.BackingPath != "" && cleanAbsolutePath(state.BackingPath) == candidate.Path) { + return false, nil + } + writerActive, err := FileHasActiveLock(filepath.Join(directory, "writer.lease")) + if err != nil || writerActive { + return false, err + } + readerActive, err := treeHasActiveLease(filepath.Join(directory, "leases")) + return !readerActive, err + case CandidateTemporary: + info, err := os.Stat(candidate.Path) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + return !info.ModTime().After(b.options.Now().Add(-b.options.TemporaryGrace)), nil + case CandidateRetiredState: + active, err := treeHasActiveLease(filepath.Join(candidate.Path, "leases")) + if err != nil || active { + return false, err + } + pending, err := journalPendingAt(candidate.Path) + return !pending, err + case CandidateManifestGeneration: + for _, state := range b.scanner.managedStates { + if cleanAbsolutePath(state.ManifestPath) == candidate.Path { + return false, nil + } + } + return true, nil + default: + return false, errors.New("unknown storage GC candidate kind") + } +} + +func sortGenerationEntries(entries []generationEntry) { + sort.Slice(entries, func(i, j int) bool { + if entries[i].sequenced && entries[j].sequenced && entries[i].sequence != entries[j].sequence { + return entries[i].sequence > entries[j].sequence + } + if entries[i].sequenced != entries[j].sequenced { + return entries[i].sequenced + } + if entries[i].modTime.Equal(entries[j].modTime) { + return entries[i].name > entries[j].name + } + return entries[i].modTime.After(entries[j].modTime) + }) +} + +func treeHasActiveLease(root string) (bool, error) { + active := false + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if errors.Is(walkErr, os.ErrNotExist) { + return filepath.SkipDir + } + if walkErr != nil { + return walkErr + } + if !entry.IsDir() || path == root { + return nil + } + hasLease, err := DirectoryHasActiveLease(path, false) + if err != nil { + return err + } + if hasLease { + active = true + return filepath.SkipAll + } + return nil + }) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return active, err +} + +func journalPendingAt(directory string) (bool, error) { + path := filepath.Join(directory, "journal.jsonl") + file, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + defer file.Close() + decoder := json.NewDecoder(file) + latest := make(map[string]string) + for { + var record journalRecord + if err := decoder.Decode(&record); errors.Is(err, io.EOF) { + break + } else if err != nil { + return false, err + } + latest[record.OperationID] = record.Phase + } + for _, phase := range latest { + if phase != "complete" && phase != "rolled-back" { + return true, nil + } + } + return false, nil +} + +type candidatePhysical struct { + bytes int64 + links uint64 + candidateLinks uint64 + candidates map[int]struct{} +} + +func describeCandidates(paths map[string]CandidateKind) ([]GCCandidate, int64, error) { + ordered := make([]string, 0, len(paths)) + for path := range paths { + ordered = append(ordered, path) + } + sort.Strings(ordered) + candidates := make([]GCCandidate, len(ordered)) + physical := make(map[string]*candidatePhysical) + for index, path := range ordered { + candidate := GCCandidate{Kind: paths[path], Path: path} + localPhysical := make(map[string]struct{}) + err := filepath.Walk(path, func(filePath string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + if !info.Mode().IsRegular() { + return nil + } + identity, bytes, err := physicalFile(filePath, info) + if err != nil { + return err + } + candidate.Files++ + candidate.ApparentBytes += info.Size() + if _, seen := localPhysical[identity]; !seen { + candidate.PhysicalBytes += bytes + localPhysical[identity] = struct{}{} + } + item := physical[identity] + if item == nil { + links, err := physicalLinkCount(filePath, info) + if err != nil { + return err + } + item = &candidatePhysical{bytes: bytes, links: links, candidates: make(map[int]struct{})} + physical[identity] = item + } + item.candidateLinks++ + item.candidates[index] = struct{}{} + return nil + }) + if err != nil { + return nil, 0, err + } + candidates[index] = candidate + } + var projected int64 + for _, item := range physical { + if item.candidateLinks < item.links { + continue + } + projected += item.bytes + first := len(candidates) + for index := range item.candidates { + if index < first { + first = index + } + } + if first < len(candidates) { + candidates[first].ReclaimableBytes += item.bytes + } + } + return candidates, projected, nil +} diff --git a/internal/storage/gc_test.go b/internal/storage/gc_test.go new file mode 100644 index 0000000..4289aa8 --- /dev/null +++ b/internal/storage/gc_test.go @@ -0,0 +1,211 @@ +package storage + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func TestCollectBoundsGenerationsRetiredStateAndTemporaryFiles(t *testing.T) { + store := filepath.Join(t.TempDir(), "store") + now := time.Unix(2_000_000, 0) + old := now.Add(-2 * time.Hour) + + writeBytesFile(t, filepath.Join(store, "packs", "CURRENT"), []byte("gen-3\n")) + for _, generation := range []string{"gen-0", "gen-1", "gen-2", "gen-3"} { + path := writeSizedFile(t, filepath.Join(store, "packs", generation, "pack-000001.pack"), 16) + setModTime(t, path, old.Add(time.Duration(generation[len(generation)-1]-'0')*time.Minute)) + writeJSONFile(t, filepath.Join(store, "packs", generation, "index.json"), map[string]any{"generation": generation}) + } + leased, err := AcquireLease(filepath.Join(store, "packs", "gen-0", "leases"), "resolver") + if err != nil { + t.Fatal(err) + } + + manifestRoot := filepath.Join(store, "manifests", "generations", "session") + for generation := 1; generation <= 3; generation++ { + writeJSONFile(t, filepath.Join(manifestRoot, string(rune('0'+generation))+".json"), manifestFixture("session", filepath.Join(store, "native.jsonl"), int64(generation*10))) + } + sessionDir := filepath.Join(store, "fs", "sessions", "session") + currentDelta := writeSizedFile(t, filepath.Join(sessionDir, "delta-00000000000000000003.jsonl"), 3) + oldDelta := writeSizedFile(t, filepath.Join(sessionDir, "delta-00000000000000000001.jsonl"), 7) + oldBacking := writeSizedFile(t, filepath.Join(sessionDir, "backing-00000000000000000002.jsonl"), 9) + writeJSONFile(t, filepath.Join(sessionDir, "state.json"), stateFixture( + "session", filepath.Join(manifestRoot, "3.json"), 30, currentDelta, "", "", + )) + + for index := 1; index <= 3; index++ { + directory := filepath.Join(store, "fs", "retired", "retired-"+string(rune('0'+index))) + writeJSONFile(t, filepath.Join(directory, "state.json"), map[string]any{"session_id": "session"}) + setModTime(t, directory, old.Add(time.Duration(index)*time.Minute)) + } + oldTemp := writeSizedFile(t, filepath.Join(store, "fs", "sessions", "session", ".backing-abandoned.tmp"), 11) + recentTemp := writeSizedFile(t, filepath.Join(store, "fs", "sessions", "session", ".state-recent.tmp"), 13) + setModTime(t, oldTemp, old) + setModTime(t, recentTemp, now.Add(-10*time.Minute)) + setModTime(t, oldDelta, old) + setModTime(t, oldBacking, old) + + options := GCOptions{ + StoreDir: store, TemporaryGrace: time.Hour, Now: func() time.Time { return now }, + KeepPackGenerations: 2, KeepManifestGenerations: 2, KeepRetiredPerSession: 1, + } + dry, err := Collect(context.Background(), options) + if err != nil { + t.Fatalf("Collect dry-run: %v", err) + } + if !dry.DryRun || dry.CandidateCount != 7 || dry.RemovedCount != 0 || dry.ProjectedReclaimableBytes <= 0 || dry.ActualReclaimedBytes != 0 { + t.Fatalf("unexpected dry-run result: %#v", dry) + } + for _, path := range []string{filepath.Join(store, "packs", "gen-1"), oldDelta, oldBacking, oldTemp, filepath.Join(manifestRoot, "1.json")} { + if _, err := os.Stat(path); err != nil { + t.Fatalf("dry-run removed %s: %v", path, err) + } + } + + options.Apply = true + applied, err := Collect(context.Background(), options) + if err != nil { + t.Fatalf("Collect apply: %v", err) + } + if applied.RemovedCount != dry.CandidateCount || applied.ActualReclaimedBytes <= 0 { + t.Fatalf("unexpected apply result: %#v", applied) + } + for _, path := range []string{filepath.Join(store, "packs", "gen-3"), filepath.Join(store, "packs", "gen-2"), filepath.Join(store, "packs", "gen-0"), currentDelta, filepath.Join(manifestRoot, "3.json"), filepath.Join(manifestRoot, "2.json"), filepath.Join(store, "fs", "retired", "retired-3"), recentTemp} { + if _, err := os.Stat(path); err != nil { + t.Fatalf("retained path missing %s: %v", path, err) + } + } + + if err := leased.Close(); err != nil { + t.Fatal(err) + } + second, err := Collect(context.Background(), options) + if err != nil { + t.Fatalf("Collect after lease close: %v", err) + } + if second.RemovedCount != 1 { + t.Fatalf("closed leased generation was not collected: %#v", second) + } + third, err := Collect(context.Background(), options) + if err != nil || third.RemovedCount != 0 { + t.Fatalf("repeated Collect is not idempotent: %#v err=%v", third, err) + } +} + +func TestCollectKeepsSoleRecoveryStateAndJournalOwnedFiles(t *testing.T) { + store := filepath.Join(t.TempDir(), "store") + now := time.Unix(3_000_000, 0) + manifest := filepath.Join(store, "manifests", "generations", "session", "1.json") + writeJSONFile(t, manifest, manifestFixture("session", filepath.Join(store, "native.jsonl"), 5)) + sessionDir := filepath.Join(store, "fs", "sessions", "session") + delta := writeSizedFile(t, filepath.Join(sessionDir, "delta.jsonl"), 5) + scratch := writeSizedFile(t, filepath.Join(sessionDir, ".compact-00000000000000000001.jsonl"), 5) + writeJSONFile(t, filepath.Join(sessionDir, "state.json"), stateFixture("session", manifest, 0, delta, "", "")) + writeJSONLine(t, filepath.Join(sessionDir, "journal.jsonl"), map[string]any{ + "operation_id": "compact-1", "phase": "prepared", "native": map[string]any{"path": scratch}, + }) + writeJSONFile(t, filepath.Join(store, "fs", "retired", "only", "state.json"), map[string]any{"session_id": "session"}) + writeSizedFile(t, filepath.Join(store, "packs", "gen-only", "pack-000001.pack"), 5) + setModTime(t, scratch, now.Add(-24*time.Hour)) + + result, err := Collect(context.Background(), GCOptions{StoreDir: store, Apply: true, TemporaryGrace: time.Hour, Now: func() time.Time { return now }}) + if err != nil { + t.Fatalf("Collect: %v", err) + } + if result.RemovedCount != 0 { + t.Fatalf("sole recovery state was removed: %#v", result) + } + for _, path := range []string{manifest, delta, scratch, filepath.Join(store, "fs", "retired", "only"), filepath.Join(store, "packs", "gen-only")} { + if _, err := os.Stat(path); err != nil { + t.Fatalf("protected path missing %s: %v", path, err) + } + } +} + +func TestCollectKeepsTruePreviousManifestAndRemovesAbandonedFuture(t *testing.T) { + store := filepath.Join(t.TempDir(), "store") + manifestRoot := filepath.Join(store, "manifests", "generations", "session") + for generation := 1; generation <= 3; generation++ { + writeJSONFile(t, filepath.Join(manifestRoot, string(rune('0'+generation))+".json"), manifestFixture("session", filepath.Join(store, "native.jsonl"), int64(generation))) + } + sessionDir := filepath.Join(store, "fs", "sessions", "session") + delta := writeSizedFile(t, filepath.Join(sessionDir, "delta-00000000000000000002.jsonl"), 2) + state := stateFixture("session", filepath.Join(manifestRoot, "2.json"), 2, delta, "", "") + state["generation"] = 2 + writeJSONFile(t, filepath.Join(sessionDir, "state.json"), state) + writeJSONLine(t, filepath.Join(sessionDir, "journal.jsonl"), map[string]any{"operation_id": "compact-2", "phase": "prepared"}) + + blocked, err := Collect(context.Background(), GCOptions{StoreDir: store, Apply: true}) + if err != nil { + t.Fatal(err) + } + if blocked.RemovedCount != 0 { + t.Fatalf("pending compaction allowed manifest cleanup: %#v", blocked) + } + writeJSONLine(t, filepath.Join(sessionDir, "journal.jsonl"), map[string]any{"operation_id": "compact-2", "phase": "rolled-back"}) + collected, err := Collect(context.Background(), GCOptions{StoreDir: store, Apply: true}) + if err != nil { + t.Fatal(err) + } + if collected.RemovedCount != 1 { + t.Fatalf("abandoned future manifest was not collected: %#v", collected) + } + if _, err := os.Stat(filepath.Join(manifestRoot, "3.json")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("abandoned future manifest remains: %v", err) + } + for _, name := range []string{"1.json", "2.json"} { + if _, err := os.Stat(filepath.Join(manifestRoot, name)); err != nil { + t.Fatalf("current/previous manifest missing %s: %v", name, err) + } + } +} + +func TestCollectReportsZeroPhysicalReclamationForRemainingHardLink(t *testing.T) { + store := t.TempDir() + keep := writeSizedFile(t, filepath.Join(store, "keep.bin"), 4096) + temporary := filepath.Join(store, ".backing-abandoned.tmp") + if err := os.Link(keep, temporary); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-2 * time.Hour) + setModTime(t, temporary, old) + result, err := Collect(context.Background(), GCOptions{StoreDir: store, Apply: true, TemporaryGrace: time.Hour, Now: time.Now}) + if err != nil { + t.Fatalf("Collect: %v", err) + } + if result.RemovedCount != 1 || result.ProjectedReclaimableBytes != 0 || result.ActualReclaimedBytes != 0 { + t.Fatalf("hard-link reclamation was overstated: %#v", result) + } + if _, err := os.Stat(keep); err != nil { + t.Fatalf("retained hard link missing: %v", err) + } + if _, err := os.Stat(temporary); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("temporary hard link remains: %v", err) + } +} + +func setModTime(t *testing.T, path string, modTime time.Time) { + t.Helper() + if err := os.Chtimes(path, modTime, modTime); err != nil { + t.Fatal(err) + } +} + +func writeMinimalState(t *testing.T, path string, sessionID string) { + t.Helper() + data, err := json.Marshal(map[string]any{"session_id": sessionID}) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } +} diff --git a/internal/storage/guard.go b/internal/storage/guard.go new file mode 100644 index 0000000..b75fe4b --- /dev/null +++ b/internal/storage/guard.go @@ -0,0 +1,99 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "path/filepath" +) + +type Projection struct { + Operation string `json:"operation"` + AdditionalPersistentBytes int64 `json:"additional_persistent_bytes"` + TemporaryBytes int64 `json:"temporary_bytes"` + TemporaryPersistentOverlapBytes int64 `json:"temporary_persistent_overlap_bytes"` + ReclaimableBytes int64 `json:"reclaimable_bytes"` +} + +type SpaceProbe func(path string) (int64, error) + +type Guard struct { + StoreDir string + Limits Limits + Probe SpaceProbe +} + +type VolumeGuard struct { + Path string + Limits Limits + Probe SpaceProbe +} + +type Assessment struct { + Inventory Inventory `json:"inventory"` + Budget BudgetReport `json:"budget"` +} + +func (g Guard) Check(ctx context.Context, projection Projection) (Assessment, error) { + if err := ctx.Err(); err != nil { + return Assessment{}, err + } + if g.StoreDir == "" { + return Assessment{}, errors.New("storage guard store directory is required") + } + store, err := filepath.Abs(g.StoreDir) + if err != nil { + return Assessment{}, err + } + store = filepath.Clean(store) + inventory, err := Scan(ctx, Options{StoreDir: store, AllowMetadataIssues: true}) + if err != nil { + return Assessment{}, err + } + probe := g.Probe + if probe == nil { + probe = AvailableBytes + } + available, err := probe(store) + if err != nil { + return Assessment{Inventory: inventory}, fmt.Errorf("probe available storage bytes: %w", err) + } + report, err := CheckBudget(BudgetRequest{ + Operation: projection.Operation, + CurrentPhysicalBytes: inventory.TotalPhysicalBytes, + AdditionalPersistentBytes: projection.AdditionalPersistentBytes, + TemporaryBytes: projection.TemporaryBytes, + TemporaryPersistentOverlapBytes: projection.TemporaryPersistentOverlapBytes, + ReclaimableBytes: projection.ReclaimableBytes, + AvailableBytes: available, + }, g.Limits) + return Assessment{Inventory: inventory, Budget: report}, err +} + +func (g VolumeGuard) Check(ctx context.Context, projection Projection) (Assessment, error) { + if err := ctx.Err(); err != nil { + return Assessment{}, err + } + if g.Path == "" { + return Assessment{}, errors.New("volume guard path is required") + } + path := filepath.Clean(g.Path) + probe := g.Probe + if probe == nil { + probe = AvailableBytes + } + available, err := probe(path) + if err != nil { + return Assessment{}, fmt.Errorf("probe available storage bytes: %w", err) + } + limits := g.Limits + if limits == (Limits{}) { + limits = DefaultLimits + } + report, err := CheckBudget(BudgetRequest{ + Operation: projection.Operation, AdditionalPersistentBytes: projection.AdditionalPersistentBytes, + TemporaryBytes: projection.TemporaryBytes, TemporaryPersistentOverlapBytes: projection.TemporaryPersistentOverlapBytes, + ReclaimableBytes: projection.ReclaimableBytes, AvailableBytes: available, + }, limits) + return Assessment{Budget: report}, err +} diff --git a/internal/storage/guard_test.go b/internal/storage/guard_test.go new file mode 100644 index 0000000..4999fe9 --- /dev/null +++ b/internal/storage/guard_test.go @@ -0,0 +1,45 @@ +package storage + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" +) + +func TestGuardScansCurrentFootprintAndChecksLiveFreeSpace(t *testing.T) { + store := filepath.Join(t.TempDir(), "store") + if err := os.MkdirAll(store, 0o700); err != nil { + t.Fatal(err) + } + writeSizedFile(t, filepath.Join(store, "metadata.bin"), 32) + var probed string + guard := Guard{ + StoreDir: store, + Limits: Limits{FreeSpaceReserveBytes: 95}, + Probe: func(path string) (int64, error) { + probed = path + return 100, nil + }, + } + assessment, err := guard.Check(context.Background(), Projection{Operation: "materialize", AdditionalPersistentBytes: 6}) + if !errors.Is(err, ErrBudgetExceeded) { + t.Fatalf("Guard.Check error = %v, want ErrBudgetExceeded", err) + } + if probed != filepath.Clean(store) { + t.Fatalf("space probe path = %q, want %q", probed, filepath.Clean(store)) + } + if assessment.Inventory.TotalPhysicalBytes == 0 || assessment.Budget.CurrentPhysicalBytes != assessment.Inventory.TotalPhysicalBytes { + t.Fatalf("assessment did not use scanned physical bytes: %#v", assessment) + } +} + +func TestGuardPropagatesSpaceProbeFailure(t *testing.T) { + store := t.TempDir() + want := errors.New("probe failed") + guard := Guard{StoreDir: store, Probe: func(string) (int64, error) { return 0, want }} + if _, err := guard.Check(context.Background(), Projection{Operation: "pack"}); !errors.Is(err, want) { + t.Fatalf("Guard.Check error = %v, want %v", err, want) + } +} diff --git a/internal/storage/inventory.go b/internal/storage/inventory.go new file mode 100644 index 0000000..ea47db6 --- /dev/null +++ b/internal/storage/inventory.go @@ -0,0 +1,599 @@ +package storage + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "math" + "os" + "path/filepath" + "strings" +) + +type FileUsage struct { + Files int `json:"files"` + ApparentBytes int64 `json:"apparent_bytes"` + PhysicalBytes int64 `json:"physical_bytes"` +} + +type Inventory struct { + StoreDir string `json:"store_dir"` + LogicalSessionBytes int64 `json:"logical_session_bytes"` + UniqueLooseObjects FileUsage `json:"unique_loose_objects"` + Packs FileUsage `json:"packs"` + NativeSources FileUsage `json:"native_sources"` + RetainedSnapshots FileUsage `json:"retained_snapshots"` + CurrentFallbacks FileUsage `json:"current_fallbacks"` + ActiveDeltas FileUsage `json:"active_deltas"` + WritableBackings FileUsage `json:"writable_backings"` + OldGenerations FileUsage `json:"old_generations"` + RetirementState FileUsage `json:"retirement_state"` + JournalRecovery FileUsage `json:"journal_recovery"` + UnownedTemporary FileUsage `json:"unowned_temporary"` + Metadata FileUsage `json:"metadata"` + TotalFiles int `json:"total_files"` + UniquePhysicalFiles int `json:"unique_physical_files"` + HardlinkAliases int `json:"hardlink_aliases"` + TotalApparentBytes int64 `json:"total_apparent_bytes"` + TotalPhysicalBytes int64 `json:"total_physical_bytes"` + IssueCount int `json:"issue_count"` + Issues []string `json:"issues,omitempty"` +} + +type Options struct { + StoreDir string + AllowMetadataIssues bool +} + +type manifestRecord struct { + Session struct { + ID string `json:"id"` + RolloutPath string `json:"rollout_path"` + } `json:"session"` + Source struct { + Bytes int64 `json:"bytes"` + } `json:"source"` +} + +type stateRecord struct { + SessionID string `json:"session_id"` + Generation uint64 `json:"generation"` + ManifestPath string `json:"manifest_path"` + BaseBytes int64 `json:"base_bytes"` + DeltaPath string `json:"delta_path"` + BackingPath string `json:"backing_path"` + Native struct { + Path string `json:"path"` + } `json:"native_snapshot"` +} + +type journalRecord struct { + OperationID string `json:"operation_id"` + Phase string `json:"phase"` + TempPath string `json:"temp_path"` + FinalPath string `json:"final_path"` + Native struct { + Path string `json:"path"` + } `json:"native"` +} + +type scanner struct { + ctx context.Context + store string + canonicalStore string + nestedMounts map[string]struct{} + result Inventory + physicalFiles map[string]struct{} + primaryManifests map[string]manifestRecord + managedStates map[string]stateRecord + activeDeltas map[string]struct{} + backings map[string]struct{} + snapshots map[string]struct{} + journalOwned map[string]struct{} + journalPending map[string]bool + currentPack string + allowMetadataIssues bool +} + +func Scan(ctx context.Context, options Options) (Inventory, error) { + s, exists, err := prepareScanner(ctx, options) + if err != nil { + return Inventory{}, err + } + if !exists { + return Inventory{StoreDir: cleanAbsolutePath(options.StoreDir)}, nil + } + if err := s.calculateLogicalBytes(); err != nil { + return Inventory{}, err + } + if err := s.walkStore(); err != nil { + return Inventory{}, err + } + if err := s.addExternalReferences(); err != nil { + return Inventory{}, err + } + return s.result, nil +} + +func prepareScanner(ctx context.Context, options Options) (*scanner, bool, error) { + if options.StoreDir == "" { + return nil, false, errors.New("storage inventory store directory is required") + } + store, err := filepath.Abs(options.StoreDir) + if err != nil { + return nil, false, fmt.Errorf("resolve storage inventory root: %w", err) + } + store = filepath.Clean(store) + if info, err := os.Stat(store); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + return nil, false, fmt.Errorf("stat storage inventory root: %w", err) + } else if !info.IsDir() { + return nil, false, errors.New("storage inventory root is not a directory") + } + canonicalStore := store + if resolved, err := filepath.EvalSymlinks(store); err == nil { + canonicalStore = filepath.Clean(resolved) + } + nestedMounts, err := nestedMountPoints(canonicalStore) + if err != nil { + return nil, false, fmt.Errorf("inspect nested storage mounts: %w", err) + } + s := &scanner{ + ctx: ctx, store: store, result: Inventory{StoreDir: store}, + canonicalStore: canonicalStore, nestedMounts: nestedMounts, + physicalFiles: make(map[string]struct{}), primaryManifests: make(map[string]manifestRecord), + managedStates: make(map[string]stateRecord), activeDeltas: make(map[string]struct{}), + backings: make(map[string]struct{}), snapshots: make(map[string]struct{}), + journalOwned: make(map[string]struct{}), journalPending: make(map[string]bool), + allowMetadataIssues: options.AllowMetadataIssues, + } + if err := s.loadManifests(); err != nil { + return nil, false, err + } + if err := s.loadStatesAndJournals(); err != nil { + return nil, false, err + } + if err := s.loadCurrentPack(); err != nil { + return nil, false, err + } + return s, true, nil +} + +func (s *scanner) loadManifests() error { + root := filepath.Join(s.store, "manifests") + return walkIfPresent(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := s.ctx.Err(); err != nil { + return err + } + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + return nil + } + manifest, err := decodeManifest(path) + if err != nil { + return s.metadataIssue(err) + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + if filepath.Dir(relative) != "." { + return nil + } + if manifest.Session.ID == "" || manifest.Source.Bytes < 0 { + return s.metadataIssue(fmt.Errorf("invalid primary manifest %s", path)) + } + if _, exists := s.primaryManifests[manifest.Session.ID]; exists { + return s.metadataIssue(fmt.Errorf("duplicate primary manifest for session %s", manifest.Session.ID)) + } + s.primaryManifests[manifest.Session.ID] = manifest + return nil + }) +} + +func (s *scanner) loadStatesAndJournals() error { + root := filepath.Join(s.store, "fs", "sessions") + entries, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read managed session directories: %w", err) + } + for _, entry := range entries { + if err := s.ctx.Err(); err != nil { + return err + } + if !entry.IsDir() { + continue + } + directory := filepath.Join(root, entry.Name()) + statePath := filepath.Join(directory, "state.json") + data, err := os.ReadFile(statePath) + if err != nil { + return fmt.Errorf("read managed session state %s: %w", entry.Name(), err) + } + var state stateRecord + if err := json.Unmarshal(data, &state); err != nil { + return fmt.Errorf("decode managed session state %s: %w", entry.Name(), err) + } + if state.SessionID != entry.Name() || state.Generation == 0 || state.BaseBytes < 0 || state.DeltaPath == "" { + return fmt.Errorf("invalid managed session state %s", statePath) + } + state.DeltaPath, err = cleanPathWithin(directory, state.DeltaPath) + if err != nil { + return fmt.Errorf("invalid managed delta for %s: %w", state.SessionID, err) + } + if state.BackingPath != "" { + state.BackingPath, err = cleanPathWithin(directory, state.BackingPath) + if err != nil { + return fmt.Errorf("invalid writable backing for %s: %w", state.SessionID, err) + } + s.backings[state.BackingPath] = struct{}{} + } else { + s.activeDeltas[state.DeltaPath] = struct{}{} + } + if state.Native.Path != "" { + state.Native.Path = cleanAbsolutePath(state.Native.Path) + s.snapshots[state.Native.Path] = struct{}{} + } + s.managedStates[state.SessionID] = state + if err := s.loadJournal(directory); err != nil { + return err + } + } + return nil +} + +func (s *scanner) loadJournal(directory string) error { + path := filepath.Join(directory, "journal.jsonl") + file, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("open session journal: %w", err) + } + defer file.Close() + latest := make(map[string]journalRecord) + lines := bufio.NewScanner(file) + lines.Buffer(make([]byte, 64*1024), 4*1024*1024) + for lines.Scan() { + var record journalRecord + if err := json.Unmarshal(lines.Bytes(), &record); err != nil { + return fmt.Errorf("decode session journal %s: %w", path, err) + } + if record.OperationID == "" { + return fmt.Errorf("session journal %s contains a record without an operation ID", path) + } + latest[record.OperationID] = record + } + if err := lines.Err(); err != nil { + return fmt.Errorf("read session journal %s: %w", path, err) + } + for _, record := range latest { + if record.Phase == "complete" || record.Phase == "rolled-back" { + continue + } + s.journalPending[filepath.Clean(directory)] = true + for _, candidate := range []string{record.TempPath, record.FinalPath, record.Native.Path} { + if candidate == "" { + continue + } + clean, err := cleanPathWithin(directory, candidate) + if err != nil { + return fmt.Errorf("unsafe journal-owned recovery path: %w", err) + } + s.journalOwned[clean] = struct{}{} + } + } + return nil +} + +func (s *scanner) calculateLogicalBytes() error { + for sessionID, state := range s.managedStates { + var bytes int64 + if state.BackingPath != "" { + info, err := os.Stat(state.BackingPath) + if err != nil { + return fmt.Errorf("stat writable backing for %s: %w", sessionID, err) + } + bytes = info.Size() + } else { + info, err := os.Stat(state.DeltaPath) + if err != nil { + return fmt.Errorf("stat active delta for %s: %w", sessionID, err) + } + var overflow bool + bytes, overflow = addInt64(state.BaseBytes, info.Size()) + if overflow { + return fmt.Errorf("logical bytes overflow for managed session %s", sessionID) + } + } + var overflow bool + s.result.LogicalSessionBytes, overflow = addInt64(s.result.LogicalSessionBytes, bytes) + if overflow { + return errors.New("logical session byte total overflow") + } + } + for sessionID, manifest := range s.primaryManifests { + if _, managed := s.managedStates[sessionID]; managed { + continue + } + var overflow bool + s.result.LogicalSessionBytes, overflow = addInt64(s.result.LogicalSessionBytes, manifest.Source.Bytes) + if overflow { + return errors.New("logical session byte total overflow") + } + } + return nil +} + +func (s *scanner) loadCurrentPack() error { + data, err := os.ReadFile(filepath.Join(s.store, "packs", "CURRENT")) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read current pack generation: %w", err) + } + s.currentPack = strings.TrimSpace(string(data)) + if s.currentPack == "" || filepath.Base(s.currentPack) != s.currentPack || s.currentPack == "." || s.currentPack == ".." { + s.currentPack = "" + return s.metadataIssue(errors.New("invalid current pack generation")) + } + return nil +} + +func (s *scanner) metadataIssue(err error) error { + if !s.allowMetadataIssues { + return err + } + s.result.Issues = append(s.result.Issues, err.Error()) + s.result.IssueCount = len(s.result.Issues) + return nil +} + +func (s *scanner) walkStore() error { + return filepath.WalkDir(s.store, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := s.ctx.Err(); err != nil { + return err + } + if entry.IsDir() { + if s.isNestedMount(path) { + return filepath.SkipDir + } + return nil + } + info, err := entry.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + usage := s.classifyStorePath(path) + return s.addFile(usage, path, info) + }) +} + +func (s *scanner) isNestedMount(path string) bool { + if path == s.store || len(s.nestedMounts) == 0 { + return false + } + relative, err := filepath.Rel(s.store, path) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return false + } + _, mounted := s.nestedMounts[filepath.Clean(filepath.Join(s.canonicalStore, relative))] + return mounted +} + +func (s *scanner) classifyStorePath(path string) *FileUsage { + path = filepath.Clean(path) + if _, ok := s.backings[path]; ok { + return &s.result.WritableBackings + } + if _, ok := s.activeDeltas[path]; ok { + return &s.result.ActiveDeltas + } + if _, ok := s.journalOwned[path]; ok { + return &s.result.JournalRecovery + } + if pathWithin(filepath.Join(s.store, "fs", "retired"), path) { + return &s.result.RetirementState + } + if pathWithin(filepath.Join(s.store, "fs", "snapshots"), path) { + return &s.result.RetainedSnapshots + } + if pathWithin(filepath.Join(s.store, "fs", "fallbacks"), path) && isCurrentFallback(path) { + return &s.result.CurrentFallbacks + } + if isUnownedTemporary(path) { + return &s.result.UnownedTemporary + } + if pathWithin(filepath.Join(s.store, "objects"), path) && filepath.Ext(path) == ".zst" { + return &s.result.UniqueLooseObjects + } + if generation, ok := packGeneration(s.store, path); ok { + if generation == s.currentPack { + return &s.result.Packs + } + return &s.result.OldGenerations + } + if pathWithin(filepath.Join(s.store, "manifests", "generations"), path) { + return &s.result.OldGenerations + } + if isSessionGenerationData(s.store, path) { + return &s.result.OldGenerations + } + return &s.result.Metadata +} + +func (s *scanner) addExternalReferences() error { + for path := range s.snapshots { + if pathWithin(s.store, path) { + continue + } + if err := s.addExternalFile(&s.result.RetainedSnapshots, path, true); err != nil { + return err + } + } + nativePaths := make(map[string]struct{}) + for _, manifest := range s.primaryManifests { + if manifest.Session.RolloutPath != "" { + nativePaths[cleanAbsolutePath(manifest.Session.RolloutPath)] = struct{}{} + } + } + for path := range nativePaths { + if pathWithin(s.store, path) { + continue + } + if err := s.addExternalFile(&s.result.NativeSources, path, false); err != nil { + return err + } + } + return nil +} + +func (s *scanner) addExternalFile(usage *FileUsage, path string, required bool) error { + info, err := os.Stat(path) + if errors.Is(err, os.ErrNotExist) && !required { + return nil + } + if err != nil { + return fmt.Errorf("stat referenced storage file %s: %w", path, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("referenced storage path is not a regular file: %s", path) + } + return s.addFile(usage, path, info) +} + +func (s *scanner) addFile(usage *FileUsage, path string, info os.FileInfo) error { + identity, physicalBytes, err := physicalFile(path, info) + if err != nil { + return err + } + usage.Files++ + usage.ApparentBytes += info.Size() + s.result.TotalFiles++ + s.result.TotalApparentBytes += info.Size() + if _, exists := s.physicalFiles[identity]; exists { + s.result.HardlinkAliases++ + return nil + } + s.physicalFiles[identity] = struct{}{} + usage.PhysicalBytes += physicalBytes + s.result.UniquePhysicalFiles++ + s.result.TotalPhysicalBytes += physicalBytes + return nil +} + +func decodeManifest(path string) (manifestRecord, error) { + data, err := os.ReadFile(path) + if err != nil { + return manifestRecord{}, fmt.Errorf("read manifest %s: %w", path, err) + } + var manifest manifestRecord + if err := json.Unmarshal(data, &manifest); err != nil { + return manifestRecord{}, fmt.Errorf("decode manifest %s: %w", path, err) + } + return manifest, nil +} + +func walkIfPresent(root string, walk fs.WalkDirFunc) error { + err := filepath.WalkDir(root, walk) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err +} + +func cleanAbsolutePath(path string) string { + if absolute, err := filepath.Abs(path); err == nil { + return filepath.Clean(absolute) + } + return filepath.Clean(path) +} + +func cleanPathWithin(root string, path string) (string, error) { + clean := cleanAbsolutePath(path) + if !pathWithin(root, clean) { + return "", errors.New("path escapes its managed root") + } + return clean, nil +} + +func pathWithin(root string, path string) bool { + root = cleanAbsolutePath(root) + path = cleanAbsolutePath(path) + relative, err := filepath.Rel(root, path) + return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} + +func packGeneration(store string, path string) (string, bool) { + relative, err := filepath.Rel(filepath.Join(store, "packs"), path) + if err != nil || relative == "." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", false + } + parts := strings.Split(relative, string(filepath.Separator)) + if len(parts) < 2 || parts[0] == "" || strings.HasPrefix(parts[0], ".") { + return "", false + } + return parts[0], true +} + +func isSessionGenerationData(store string, path string) bool { + relative, err := filepath.Rel(filepath.Join(store, "fs", "sessions"), path) + if err != nil || relative == "." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return false + } + parts := strings.Split(relative, string(filepath.Separator)) + if len(parts) != 2 { + return false + } + name := parts[1] + return strings.HasPrefix(name, "delta-") && strings.HasSuffix(name, ".jsonl") || + strings.HasPrefix(name, "backing-") && strings.HasSuffix(name, ".jsonl") || name == "delta.jsonl" +} + +func isCurrentFallback(path string) bool { + switch filepath.Base(path) { + case "fallback-current.jsonl", "quarantine-current.jsonl": + return true + default: + return false + } +} + +func isUnownedTemporary(path string) bool { + name := filepath.Base(path) + if !strings.HasPrefix(name, ".") { + return false + } + return strings.Contains(name, ".tmp") || strings.HasPrefix(name, ".generation-") || + strings.HasPrefix(name, ".compact-") || strings.HasPrefix(name, ".object-") || + strings.HasPrefix(name, ".manifest-") || strings.HasPrefix(name, ".materialize-") || + strings.HasPrefix(name, ".backing-") || strings.HasPrefix(name, ".CURRENT-") +} + +func addInt64(left int64, right int64) (int64, bool) { + if right > 0 && left > math.MaxInt64-right { + return 0, true + } + if right < 0 && left < math.MinInt64-right { + return 0, true + } + return left + right, false +} diff --git a/internal/storage/inventory_test.go b/internal/storage/inventory_test.go new file mode 100644 index 0000000..0caea14 --- /dev/null +++ b/internal/storage/inventory_test.go @@ -0,0 +1,222 @@ +package storage + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" +) + +func TestScanClassifiesManagedStorageAndDeduplicatesHardLinks(t *testing.T) { + root := t.TempDir() + store := filepath.Join(root, "store") + nativeRoot := filepath.Join(root, "native") + + nativeA := writeSizedFile(t, filepath.Join(nativeRoot, "managed-a.jsonl"), 13) + nativeB := writeSizedFile(t, filepath.Join(nativeRoot, "managed-b.jsonl"), 20) + nativeC := writeSizedFile(t, filepath.Join(nativeRoot, "archived.jsonl"), 50) + + manifestA := filepath.Join(store, "manifests", "managed-a.json") + manifestB := filepath.Join(store, "manifests", "managed-b.json") + manifestC := filepath.Join(store, "manifests", "archived.json") + writeJSONFile(t, manifestA, manifestFixture("managed-a", nativeA, 100)) + writeJSONFile(t, manifestB, manifestFixture("managed-b", nativeB, 20)) + writeJSONFile(t, manifestC, manifestFixture("archived", nativeC, 50)) + + writeSizedFile(t, filepath.Join(store, "objects", "aa", "aaaaaaaa.zst"), 3) + writeSizedFile(t, filepath.Join(store, "packs", "gen-current", "pack-000001.pack"), 4) + writeSizedFile(t, filepath.Join(store, "packs", "gen-current", "index.json"), 2) + writeSizedFile(t, filepath.Join(store, "packs", "gen-old", "pack-000001.pack"), 5) + writeSizedFile(t, filepath.Join(store, "packs", "gen-old", "index.json"), 2) + writeBytesFile(t, filepath.Join(store, "packs", "CURRENT"), []byte("gen-current\n")) + + snapshotA := filepath.Join(store, "fs", "snapshots", "managed-a", "native.jsonl") + if err := os.MkdirAll(filepath.Dir(snapshotA), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Link(nativeA, snapshotA); err != nil { + t.Fatalf("hard-link retained snapshot: %v", err) + } + snapshotB := writeSizedFile(t, filepath.Join(store, "fs", "snapshots", "managed-b", "native.jsonl"), 20) + + sessionA := filepath.Join(store, "fs", "sessions", "managed-a") + deltaA := writeSizedFile(t, filepath.Join(sessionA, "delta.jsonl"), 4) + backingA := writeSizedFile(t, filepath.Join(sessionA, "backing-00000000000000000002.jsonl"), 17) + writeJSONFile(t, filepath.Join(sessionA, "state.json"), stateFixture("managed-a", manifestA, 100, deltaA, backingA, snapshotA)) + + sessionB := filepath.Join(store, "fs", "sessions", "managed-b") + deltaB := writeSizedFile(t, filepath.Join(sessionB, "delta.jsonl"), 5) + writeSizedFile(t, filepath.Join(sessionB, "delta-00000000000000000001.jsonl"), 6) + writeJSONFile(t, filepath.Join(sessionB, "state.json"), stateFixture("managed-b", manifestB, 20, deltaB, "", snapshotB)) + + scratch := writeSizedFile(t, filepath.Join(sessionB, ".compact-00000000000000000001.jsonl"), 25) + stateTemp := writeSizedFile(t, filepath.Join(sessionB, ".state-compact-00000000000000000002.tmp"), 7) + writeJSONLine(t, filepath.Join(sessionB, "journal.jsonl"), map[string]any{ + "operation_id": "compact-00000000000000000001", + "phase": "prepared", + "temp_path": stateTemp, + "native": map[string]any{"path": scratch}, + }) + writeSizedFile(t, filepath.Join(sessionB, ".backing-orphan.tmp"), 8) + + writeSizedFile(t, filepath.Join(store, "fs", "fallbacks", "managed-a", "fallback-current.jsonl"), 9) + writeSizedFile(t, filepath.Join(store, "fs", "retired", "managed-a-1", "state.json"), 11) + writeSizedFile(t, filepath.Join(store, "fs", "retired", "managed-a-1", "retained-native", "native.jsonl"), 12) + + inventory, err := Scan(context.Background(), Options{StoreDir: store}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + + if inventory.LogicalSessionBytes != 92 { + t.Fatalf("logical session bytes = %d, want 92", inventory.LogicalSessionBytes) + } + assertUsage(t, "loose objects", inventory.UniqueLooseObjects, 1, 3) + assertUsage(t, "current packs", inventory.Packs, 2, 6) + assertUsage(t, "native sources", inventory.NativeSources, 3, 83) + assertUsage(t, "retained snapshots", inventory.RetainedSnapshots, 2, 33) + assertUsage(t, "current fallbacks", inventory.CurrentFallbacks, 1, 9) + assertUsage(t, "active deltas", inventory.ActiveDeltas, 1, 5) + assertUsage(t, "writable backings", inventory.WritableBackings, 1, 17) + assertUsage(t, "old generations", inventory.OldGenerations, 4, 17) + assertUsage(t, "retirement state", inventory.RetirementState, 2, 23) + assertUsage(t, "journal recovery", inventory.JournalRecovery, 2, 32) + assertUsage(t, "unowned temporary", inventory.UnownedTemporary, 1, 8) + + if inventory.TotalPhysicalBytes <= 0 { + t.Fatalf("total physical bytes = %d", inventory.TotalPhysicalBytes) + } + if inventory.HardlinkAliases != 1 { + t.Fatalf("hard-link aliases = %d, want 1", inventory.HardlinkAliases) + } +} + +func TestScanRejectsPathsOutsideTheDeclaredStore(t *testing.T) { + root := t.TempDir() + store := filepath.Join(root, "store") + manifest := filepath.Join(store, "manifests", "session.json") + native := writeSizedFile(t, filepath.Join(root, "native.jsonl"), 4) + writeJSONFile(t, manifest, manifestFixture("session", native, 4)) + writeJSONFile(t, filepath.Join(store, "fs", "sessions", "session", "state.json"), stateFixture( + "session", + manifest, + 4, + filepath.Join(root, "unsafe-delta.jsonl"), + "", + native, + )) + + if _, err := Scan(context.Background(), Options{StoreDir: store}); err == nil { + t.Fatal("Scan should reject a managed data path outside its session directory") + } +} + +func TestScannerRecognizesCanonicalNestedMount(t *testing.T) { + store := filepath.Join(t.TempDir(), "store") + canonical := filepath.Join(filepath.Dir(store), "canonical-store") + s := &scanner{ + store: store, + canonicalStore: canonical, + nestedMounts: map[string]struct{}{ + filepath.Join(canonical, "nested", "mount"): {}, + }, + } + if !s.isNestedMount(filepath.Join(store, "nested", "mount")) { + t.Fatal("canonical nested mount was not recognized") + } + if s.isNestedMount(filepath.Join(store, "nested")) || s.isNestedMount(store) { + t.Fatal("ordinary storage directory was treated as a mount") + } +} + +func assertUsage(t *testing.T, name string, usage FileUsage, files int, apparentBytes int64) { + t.Helper() + if usage.Files != files || usage.ApparentBytes != apparentBytes { + t.Fatalf("%s usage = %#v, want files=%d apparent_bytes=%d", name, usage, files, apparentBytes) + } +} + +func manifestFixture(sessionID string, rolloutPath string, sourceBytes int64) map[string]any { + return map[string]any{ + "session": map[string]any{"id": sessionID, "rollout_path": rolloutPath}, + "source": map[string]any{"bytes": sourceBytes}, + } +} + +func stateFixture(sessionID string, manifestPath string, baseBytes int64, deltaPath string, backingPath string, snapshotPath string) map[string]any { + return map[string]any{ + "version": 1, + "session_id": sessionID, + "generation": 2, + "manifest_path": manifestPath, + "base_bytes": baseBytes, + "base_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "delta_path": deltaPath, + "backing_path": backingPath, + "native_snapshot": map[string]any{"path": snapshotPath, "bytes": baseBytes, "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + } +} + +func writeJSONFile(t *testing.T, path string, value any) string { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, append(data, '\n'), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func writeJSONLine(t *testing.T, path string, value any) { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, append(data, '\n'), 0o600); err != nil { + t.Fatal(err) + } +} + +func writeSizedFile(t *testing.T, path string, size int64) string { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + buffer := make([]byte, size) + for i := range buffer { + buffer[i] = byte('a' + i%26) + } + if _, err := file.Write(buffer); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + return path +} + +func writeBytesFile(t *testing.T, path string, data []byte) string { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + return path +} diff --git a/internal/storage/lease.go b/internal/storage/lease.go new file mode 100644 index 0000000..96f5773 --- /dev/null +++ b/internal/storage/lease.go @@ -0,0 +1,157 @@ +package storage + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" +) + +type Lease struct { + file *os.File + path string + closeOnce sync.Once + closeErr error +} + +func AcquireLease(directory string, label string) (*Lease, error) { + if directory == "" || label == "" || filepath.Base(label) != label || strings.ContainsAny(label, "/\\\x00") { + return nil, errors.New("lease directory and safe label are required") + } + if err := os.Mkdir(directory, 0o700); err != nil { + if !errors.Is(err, os.ErrExist) { + return nil, fmt.Errorf("create lease directory: %w", err) + } + info, statErr := os.Lstat(directory) + if statErr != nil { + return nil, fmt.Errorf("inspect lease directory: %w", statErr) + } + if !info.IsDir() { + return nil, errors.New("lease path is not a directory") + } + } + random := make([]byte, 8) + if _, err := rand.Read(random); err != nil { + return nil, err + } + path := filepath.Join(directory, fmt.Sprintf(".lease-%s-%d-%s", label, os.Getpid(), hex.EncodeToString(random))) + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("create lease file: %w", err) + } + locked, err := tryLockLease(file) + if err != nil || !locked { + _ = file.Close() + _ = os.Remove(path) + if err == nil { + err = errors.New("new lease file could not be locked") + } + return nil, err + } + if _, err := fmt.Fprintf(file, "%d\n", os.Getpid()); err != nil { + _ = unlockLease(file) + _ = file.Close() + _ = os.Remove(path) + return nil, err + } + if err := file.Sync(); err != nil { + _ = unlockLease(file) + _ = file.Close() + _ = os.Remove(path) + return nil, err + } + return &Lease{file: file, path: path}, nil +} + +func (l *Lease) Close() error { + if l == nil { + return nil + } + l.closeOnce.Do(func() { + var errs []error + if l.file != nil { + if err := unlockLease(l.file); err != nil { + errs = append(errs, err) + } + if err := l.file.Close(); err != nil { + errs = append(errs, err) + } + } + if err := os.Remove(l.path); err != nil && !errors.Is(err, os.ErrNotExist) { + errs = append(errs, err) + } + l.closeErr = errors.Join(errs...) + }) + return l.closeErr +} + +func DirectoryHasActiveLease(directory string, cleanStale bool) (bool, error) { + entries, err := os.ReadDir(directory) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + active := false + for _, entry := range entries { + if entry.IsDir() || !strings.HasPrefix(entry.Name(), ".lease-") { + continue + } + path := filepath.Join(directory, entry.Name()) + file, err := os.OpenFile(path, os.O_RDWR, 0) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return false, err + } + locked, lockErr := tryLockLease(file) + if lockErr != nil { + _ = file.Close() + return false, lockErr + } + if !locked { + active = true + _ = file.Close() + continue + } + unlockErr := unlockLease(file) + closeErr := file.Close() + if unlockErr != nil || closeErr != nil { + return false, errors.Join(unlockErr, closeErr) + } + if cleanStale { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return false, err + } + } + } + return active, nil +} + +func FileHasActiveLock(path string) (bool, error) { + file, err := os.OpenFile(path, os.O_RDWR, 0) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + locked, lockErr := tryLockLease(file) + if lockErr != nil { + _ = file.Close() + return false, lockErr + } + if !locked { + _ = file.Close() + return true, nil + } + unlockErr := unlockLease(file) + closeErr := file.Close() + return false, errors.Join(unlockErr, closeErr) +} diff --git a/internal/storage/lease_other.go b/internal/storage/lease_other.go new file mode 100644 index 0000000..c4074f6 --- /dev/null +++ b/internal/storage/lease_other.go @@ -0,0 +1,16 @@ +//go:build !darwin && !linux && !windows + +package storage + +import ( + "errors" + "os" +) + +func tryLockLease(*os.File) (bool, error) { + return false, errors.New("storage leases are unsupported on this platform") +} + +func unlockLease(*os.File) error { + return errors.New("storage leases are unsupported on this platform") +} diff --git a/internal/storage/lease_test.go b/internal/storage/lease_test.go new file mode 100644 index 0000000..8831020 --- /dev/null +++ b/internal/storage/lease_test.go @@ -0,0 +1,65 @@ +package storage + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestLeaseReportsActiveUntilClosed(t *testing.T) { + directory := filepath.Join(t.TempDir(), "leases") + lease, err := AcquireLease(directory, "generation") + if err != nil { + t.Fatalf("AcquireLease: %v", err) + } + active, err := DirectoryHasActiveLease(directory, true) + if err != nil { + t.Fatalf("DirectoryHasActiveLease: %v", err) + } + if !active { + t.Fatal("held lease was not reported active") + } + if err := lease.Close(); err != nil { + t.Fatalf("close lease: %v", err) + } + active, err = DirectoryHasActiveLease(directory, true) + if err != nil { + t.Fatalf("DirectoryHasActiveLease after close: %v", err) + } + if active { + t.Fatal("closed lease remained active") + } +} + +func TestAcquireLeaseDoesNotCreateMissingAncestorDirectories(t *testing.T) { + root := t.TempDir() + missingParent := filepath.Join(root, "missing-generation") + if _, err := AcquireLease(filepath.Join(missingParent, "leases"), "reader"); err == nil { + t.Fatal("lease unexpectedly created a missing generation tree") + } + if _, err := os.Lstat(missingParent); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing generation was recreated: %v", err) + } +} + +func TestDirectoryHasActiveLeaseCleansUnlockedStaleFiles(t *testing.T) { + directory := filepath.Join(t.TempDir(), "leases") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + stale := filepath.Join(directory, ".lease-stale") + if err := os.WriteFile(stale, []byte("stale\n"), 0o600); err != nil { + t.Fatal(err) + } + active, err := DirectoryHasActiveLease(directory, true) + if err != nil { + t.Fatalf("DirectoryHasActiveLease: %v", err) + } + if active { + t.Fatal("unlocked stale lease was reported active") + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatalf("stale lease remains: %v", err) + } +} diff --git a/internal/storage/lease_unix.go b/internal/storage/lease_unix.go new file mode 100644 index 0000000..e4ca92a --- /dev/null +++ b/internal/storage/lease_unix.go @@ -0,0 +1,21 @@ +//go:build darwin || linux + +package storage + +import ( + "errors" + "os" + "syscall" +) + +func tryLockLease(file *os.File) (bool, error) { + err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return false, nil + } + return err == nil, err +} + +func unlockLease(file *os.File) error { + return syscall.Flock(int(file.Fd()), syscall.LOCK_UN) +} diff --git a/internal/storage/lease_windows.go b/internal/storage/lease_windows.go new file mode 100644 index 0000000..4981183 --- /dev/null +++ b/internal/storage/lease_windows.go @@ -0,0 +1,23 @@ +//go:build windows + +package storage + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +func tryLockLease(file *os.File) (bool, error) { + overlapped := new(windows.Overlapped) + err := windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, overlapped) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) || errors.Is(err, windows.ERROR_IO_PENDING) { + return false, nil + } + return err == nil, err +} + +func unlockLease(file *os.File) error { + return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, new(windows.Overlapped)) +} diff --git a/internal/storage/mountpoints_darwin.go b/internal/storage/mountpoints_darwin.go new file mode 100644 index 0000000..794ee68 --- /dev/null +++ b/internal/storage/mountpoints_darwin.go @@ -0,0 +1,29 @@ +//go:build darwin + +package storage + +import ( + "path/filepath" + + "golang.org/x/sys/unix" +) + +func nestedMountPoints(root string) (map[string]struct{}, error) { + count, err := unix.Getfsstat(nil, unix.MNT_NOWAIT) + if err != nil { + return nil, err + } + stats := make([]unix.Statfs_t, count+16) + count, err = unix.Getfsstat(stats, unix.MNT_NOWAIT) + if err != nil { + return nil, err + } + result := make(map[string]struct{}) + for _, stat := range stats[:count] { + mountPoint := filepath.Clean(unix.ByteSliceToString(stat.Mntonname[:])) + if mountPoint != root && pathWithin(root, mountPoint) { + result[mountPoint] = struct{}{} + } + } + return result, nil +} diff --git a/internal/storage/mountpoints_linux.go b/internal/storage/mountpoints_linux.go new file mode 100644 index 0000000..8a782b1 --- /dev/null +++ b/internal/storage/mountpoints_linux.go @@ -0,0 +1,36 @@ +//go:build linux + +package storage + +import ( + "bufio" + "os" + "path/filepath" + "strings" +) + +func nestedMountPoints(root string) (map[string]struct{}, error) { + file, err := os.Open("/proc/self/mountinfo") + if err != nil { + return nil, err + } + defer file.Close() + result := make(map[string]struct{}) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 5 { + continue + } + mountPoint := filepath.Clean(unescapeMountInfoPath(fields[4])) + if mountPoint != root && pathWithin(root, mountPoint) { + result[mountPoint] = struct{}{} + } + } + return result, scanner.Err() +} + +func unescapeMountInfoPath(path string) string { + replacer := strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`) + return replacer.Replace(path) +} diff --git a/internal/storage/mountpoints_other.go b/internal/storage/mountpoints_other.go new file mode 100644 index 0000000..82c43d1 --- /dev/null +++ b/internal/storage/mountpoints_other.go @@ -0,0 +1,7 @@ +//go:build !darwin && !linux + +package storage + +func nestedMountPoints(string) (map[string]struct{}, error) { + return nil, nil +} diff --git a/internal/storage/physical_other.go b/internal/storage/physical_other.go new file mode 100644 index 0000000..fe58f8e --- /dev/null +++ b/internal/storage/physical_other.go @@ -0,0 +1,16 @@ +//go:build !darwin && !linux && !windows + +package storage + +import ( + "os" + "path/filepath" +) + +func physicalFile(path string, info os.FileInfo) (string, int64, error) { + return filepath.Clean(path), info.Size(), nil +} + +func physicalLinkCount(_ string, _ os.FileInfo) (uint64, error) { + return 1, nil +} diff --git a/internal/storage/physical_unix.go b/internal/storage/physical_unix.go new file mode 100644 index 0000000..d21cc47 --- /dev/null +++ b/internal/storage/physical_unix.go @@ -0,0 +1,24 @@ +//go:build darwin || linux + +package storage + +import ( + "fmt" + "os" + "syscall" +) + +func physicalFile(path string, info os.FileInfo) (string, int64, error) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return "", 0, fmt.Errorf("read physical file identity for %s", path) + } + return fmt.Sprintf("%d:%d", stat.Dev, stat.Ino), stat.Blocks * 512, nil +} + +func physicalLinkCount(_ string, info os.FileInfo) (uint64, error) { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + return uint64(stat.Nlink), nil + } + return 1, nil +} diff --git a/internal/storage/physical_windows.go b/internal/storage/physical_windows.go new file mode 100644 index 0000000..dd97e7b --- /dev/null +++ b/internal/storage/physical_windows.go @@ -0,0 +1,45 @@ +//go:build windows + +package storage + +import ( + "fmt" + "os" + + "golang.org/x/sys/windows" +) + +func physicalFile(path string, info os.FileInfo) (string, int64, error) { + utf16Path, err := windows.UTF16PtrFromString(path) + if err != nil { + return "", 0, err + } + handle, err := windows.CreateFile(utf16Path, 0, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) + if err != nil { + return "", 0, fmt.Errorf("open physical file identity for %s: %w", path, err) + } + defer windows.CloseHandle(handle) + var identity windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &identity); err != nil { + return "", 0, fmt.Errorf("read physical file identity for %s: %w", path, err) + } + key := fmt.Sprintf("%d:%d:%d", identity.VolumeSerialNumber, identity.FileIndexHigh, identity.FileIndexLow) + return key, info.Size(), nil +} + +func physicalLinkCount(path string, _ os.FileInfo) (uint64, error) { + utf16Path, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, err + } + handle, err := windows.CreateFile(utf16Path, 0, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) + if err != nil { + return 0, err + } + defer windows.CloseHandle(handle) + var identity windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &identity); err != nil { + return 0, err + } + return uint64(identity.NumberOfLinks), nil +} diff --git a/internal/storage/policy.go b/internal/storage/policy.go new file mode 100644 index 0000000..aa458d4 --- /dev/null +++ b/internal/storage/policy.go @@ -0,0 +1,70 @@ +package storage + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +const PolicyFilename = "storage-policy.json" + +var DefaultLimits = Limits{ + MaxPhysicalBytes: 512 << 30, + MaxTemporaryBytes: 16 << 30, + FreeSpaceReserveBytes: 5 << 30, +} + +type policyFile struct { + Version int `json:"version"` + Limits Limits `json:"limits"` +} + +type Checker interface { + Check(context.Context, Projection) (Assessment, error) +} + +func LoadLimits(storeDir string) (Limits, error) { + if storeDir == "" { + return Limits{}, errors.New("storage policy store directory is required") + } + path := filepath.Join(filepath.Clean(storeDir), PolicyFilename) + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return DefaultLimits, nil + } + if err != nil { + return Limits{}, fmt.Errorf("read storage policy: %w", err) + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var policy policyFile + if err := decoder.Decode(&policy); err != nil { + return Limits{}, fmt.Errorf("decode storage policy: %w", err) + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + if err == nil { + err = errors.New("multiple JSON values") + } + return Limits{}, fmt.Errorf("decode storage policy: %w", err) + } + if policy.Version != 1 { + return Limits{}, fmt.Errorf("unsupported storage policy version %d", policy.Version) + } + if policy.Limits.MaxPhysicalBytes <= 0 || policy.Limits.MaxTemporaryBytes <= 0 || policy.Limits.FreeSpaceReserveBytes <= 0 { + return Limits{}, errors.New("storage policy limits must all be positive") + } + return policy.Limits, nil +} + +func DefaultGuard(storeDir string) (Guard, error) { + limits, err := LoadLimits(storeDir) + if err != nil { + return Guard{}, err + } + return Guard{StoreDir: filepath.Clean(storeDir), Limits: limits}, nil +} diff --git a/internal/storage/policy_test.go b/internal/storage/policy_test.go new file mode 100644 index 0000000..738834b --- /dev/null +++ b/internal/storage/policy_test.go @@ -0,0 +1,52 @@ +package storage + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +func TestLoadLimitsUsesBoundedDefaultsAndAllowsStoreOverride(t *testing.T) { + store := t.TempDir() + defaults, err := LoadLimits(store) + if err != nil { + t.Fatalf("LoadLimits defaults: %v", err) + } + if defaults.MaxPhysicalBytes <= 0 || defaults.MaxTemporaryBytes <= 0 || defaults.FreeSpaceReserveBytes <= 0 { + t.Fatalf("default limits are not hard bounds: %#v", defaults) + } + + want := Limits{MaxPhysicalBytes: 900, MaxTemporaryBytes: 80, FreeSpaceReserveBytes: 70} + data, err := json.Marshal(map[string]any{"version": 1, "limits": want}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(store, PolicyFilename), append(data, '\n'), 0o600); err != nil { + t.Fatal(err) + } + got, err := LoadLimits(store) + if err != nil { + t.Fatalf("LoadLimits override: %v", err) + } + if got != want { + t.Fatalf("limits = %#v, want %#v", got, want) + } +} + +func TestLoadLimitsRejectsUnboundedOrUnknownPolicy(t *testing.T) { + tests := []string{ + `{"version":2,"limits":{"max_physical_bytes":1,"max_temporary_bytes":1,"free_space_reserve_bytes":1}}`, + `{"version":1,"limits":{"max_physical_bytes":0,"max_temporary_bytes":1,"free_space_reserve_bytes":1}}`, + `{"version":1,"limits":{"max_physical_bytes":1,"max_temporary_bytes":1,"free_space_reserve_bytes":1},"extra":true}`, + } + for index, data := range tests { + store := t.TempDir() + if err := os.WriteFile(filepath.Join(store, PolicyFilename), []byte(data), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadLimits(store); err == nil { + t.Fatalf("policy %d should fail", index) + } + } +} diff --git a/internal/storage/space.go b/internal/storage/space.go new file mode 100644 index 0000000..3ea3c0f --- /dev/null +++ b/internal/storage/space.go @@ -0,0 +1,23 @@ +package storage + +import ( + "errors" + "os" + "path/filepath" +) + +func existingSpaceProbePath(path string) (string, error) { + path = filepath.Clean(path) + for { + if _, err := os.Stat(path); err == nil { + return path, nil + } else if !errors.Is(err, os.ErrNotExist) { + return "", err + } + parent := filepath.Dir(path) + if parent == path { + return "", os.ErrNotExist + } + path = parent + } +} diff --git a/internal/storage/space_other.go b/internal/storage/space_other.go new file mode 100644 index 0000000..71bd984 --- /dev/null +++ b/internal/storage/space_other.go @@ -0,0 +1,9 @@ +//go:build !darwin && !linux && !windows + +package storage + +import "errors" + +func AvailableBytes(string) (int64, error) { + return 0, errors.New("available-byte probe is unsupported on this platform") +} diff --git a/internal/storage/space_unix.go b/internal/storage/space_unix.go new file mode 100644 index 0000000..34bf00a --- /dev/null +++ b/internal/storage/space_unix.go @@ -0,0 +1,29 @@ +//go:build darwin || linux + +package storage + +import ( + "errors" + "math" + + "golang.org/x/sys/unix" +) + +func AvailableBytes(path string) (int64, error) { + path, err := existingSpaceProbePath(path) + if err != nil { + return 0, err + } + var stat unix.Statfs_t + if err := unix.Statfs(path, &stat); err != nil { + return 0, err + } + if stat.Bsize <= 0 { + return 0, errors.New("filesystem block size is invalid") + } + available := uint64(stat.Bavail) * uint64(stat.Bsize) + if available > math.MaxInt64 { + return math.MaxInt64, nil + } + return int64(available), nil +} diff --git a/internal/storage/space_windows.go b/internal/storage/space_windows.go new file mode 100644 index 0000000..26611bc --- /dev/null +++ b/internal/storage/space_windows.go @@ -0,0 +1,24 @@ +//go:build windows + +package storage + +import "golang.org/x/sys/windows" + +func AvailableBytes(path string) (int64, error) { + path, err := existingSpaceProbePath(path) + if err != nil { + return 0, err + } + utf16Path, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, err + } + var available uint64 + if err := windows.GetDiskFreeSpaceEx(utf16Path, &available, nil, nil); err != nil { + return 0, err + } + if available > uint64(^uint64(0)>>1) { + return int64(^uint64(0) >> 1), nil + } + return int64(available), nil +} diff --git a/internal/testfs/corpus.go b/internal/testfs/corpus.go new file mode 100644 index 0000000..71f3da9 --- /dev/null +++ b/internal/testfs/corpus.go @@ -0,0 +1,150 @@ +package testfs + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +type Options struct { + LargeFieldBytes int + RepeatedRecords int +} + +type Session struct { + ID string `json:"id"` + Path string `json:"path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` +} + +type Corpus struct { + Root string `json:"root"` + Sessions []Session `json:"sessions"` +} + +func Generate(root string, options Options) (Corpus, error) { + if root == "" { + return Corpus{}, errors.New("corpus root is required") + } + if options.LargeFieldBytes <= 0 { + options.LargeFieldBytes = 768 << 10 + } + if options.RepeatedRecords <= 0 { + options.RepeatedRecords = 64 + } + if err := os.MkdirAll(root, 0o700); err != nil { + return Corpus{}, err + } + large := make([]byte, options.LargeFieldBytes) + for index := range large { + large[index] = byte('a' + index%23) + } + repeated := []byte("{\"type\":\"event\",\"payload\":\"exact-repeated-record\"}\n") + prefixPath := filepath.Join(root, "shared-prefix.bin") + prefix, err := os.Create(prefixPath) + if err != nil { + return Corpus{}, err + } + writer := bufio.NewWriterSize(prefix, 1<<20) + _, _ = writer.WriteString("{\"type\":\"session_meta\",\"id\":\"synthetic\"}\n") + _, _ = writer.WriteString("{\"type\":\"large\",\"payload\":\"") + _, _ = writer.Write(large) + _, _ = writer.WriteString("\"}\n") + for index := 0; index < options.RepeatedRecords; index++ { + _, _ = writer.Write(repeated) + } + _, _ = writer.WriteString("not-json-but-valid-rollout-bytes\n") + if err := writer.Flush(); err != nil { + _ = prefix.Close() + return Corpus{}, err + } + if err := prefix.Sync(); err != nil { + _ = prefix.Close() + return Corpus{}, err + } + if err := prefix.Close(); err != nil { + return Corpus{}, err + } + prefixData, err := os.ReadFile(prefixPath) + if err != nil { + return Corpus{}, err + } + definitions := []struct { + id string + body []byte + }{ + {id: "fork-a", body: append(append([]byte(nil), prefixData...), []byte("{\"tail\":\"a\"}\n")...)}, + {id: "fork-b", body: append(append([]byte(nil), prefixData...), []byte("{\"tail\":\"b\"}\n")...)}, + {id: "reordered", body: append(append([]byte(nil), repeated...), append(prefixData, repeated...)...)}, + {id: "empty", body: []byte{}}, + } + corpus := Corpus{Root: root, Sessions: make([]Session, 0, len(definitions))} + for _, definition := range definitions { + path := filepath.Join(root, definition.id+".jsonl") + if err := os.WriteFile(path, definition.body, 0o600); err != nil { + return Corpus{}, err + } + digest := sha256.Sum256(definition.body) + corpus.Sessions = append(corpus.Sessions, Session{ID: definition.id, Path: path, Bytes: int64(len(definition.body)), SHA256: hex.EncodeToString(digest[:])}) + } + _ = os.Remove(prefixPath) + return corpus, nil +} + +func GenerateRollout(path string, targetBytes int64) (Session, error) { + if path == "" || targetBytes < 0 { + return Session{}, errors.New("rollout path and non-negative target size are required") + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return Session{}, err + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return Session{}, err + } + hasher := sha256.New() + writer := bufio.NewWriterSize(io.MultiWriter(file, hasher), 4<<20) + record := []byte("{\"type\":\"event\",\"payload\":\"synthetic-repeat-0123456789abcdefghijklmnopqrstuvwxyz\"}\n") + var written int64 + for targetBytes-written >= int64(len(record)) { + n, writeErr := writer.Write(record) + written += int64(n) + if writeErr != nil { + _ = file.Close() + return Session{}, writeErr + } + } + if remaining := targetBytes - written; remaining > 0 { + padding := make([]byte, remaining) + for index := range padding { + padding[index] = byte('A' + index%26) + } + n, writeErr := writer.Write(padding) + written += int64(n) + if writeErr != nil { + _ = file.Close() + return Session{}, writeErr + } + } + if err := writer.Flush(); err != nil { + _ = file.Close() + return Session{}, err + } + if err := file.Sync(); err != nil { + _ = file.Close() + return Session{}, err + } + if err := file.Close(); err != nil { + return Session{}, err + } + if written != targetBytes { + return Session{}, fmt.Errorf("generated %d bytes, want %d", written, targetBytes) + } + return Session{ID: "large", Path: path, Bytes: written, SHA256: hex.EncodeToString(hasher.Sum(nil))}, nil +} diff --git a/internal/testfs/corpus_test.go b/internal/testfs/corpus_test.go new file mode 100644 index 0000000..27a44af --- /dev/null +++ b/internal/testfs/corpus_test.go @@ -0,0 +1,200 @@ +package testfs + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "math/rand" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/fsctl" + "github.com/samekind/codexfold/internal/pack" + "github.com/samekind/codexfold/internal/vfs" +) + +func TestGenerateIsDeterministicAndContainsForkAndNonPrefixDuplication(t *testing.T) { + first, err := Generate(filepath.Join(t.TempDir(), "first"), Options{}) + if err != nil { + t.Fatal(err) + } + second, err := Generate(filepath.Join(t.TempDir(), "second"), Options{}) + if err != nil { + t.Fatal(err) + } + if len(first.Sessions) != len(second.Sessions) || len(first.Sessions) < 4 { + t.Fatalf("unexpected corpora: %#v %#v", first, second) + } + for index := range first.Sessions { + if first.Sessions[index].ID != second.Sessions[index].ID || first.Sessions[index].SHA256 != second.Sessions[index].SHA256 || first.Sessions[index].Bytes != second.Sessions[index].Bytes { + t.Fatalf("corpus is not deterministic at %d: %#v %#v", index, first.Sessions[index], second.Sessions[index]) + } + } + if first.Sessions[0].SHA256 == first.Sessions[1].SHA256 || first.Sessions[2].Bytes <= first.Sessions[0].Bytes { + t.Fatalf("fork and reordered fixtures are not distinct: %#v", first.Sessions) + } +} + +func TestPackedCorpusShadowRandomReadsAndWritableSessionStress(t *testing.T) { + root := t.TempDir() + corpus, err := Generate(filepath.Join(root, "corpus"), Options{LargeFieldBytes: 768 << 10, RepeatedRecords: 128}) + if err != nil { + t.Fatal(err) + } + store := filepath.Join(root, "store") + for _, fixture := range corpus.Sessions { + _, err := fold.Fold(context.Background(), fold.Session{ID: fixture.ID, RolloutPath: fixture.Path, Archived: true}, fold.FoldOptions{StoreDir: store, Apply: true, FieldThreshold: 32}) + if err != nil { + t.Fatalf("fold %s: %v", fixture.ID, err) + } + } + if _, err := pack.Build(context.Background(), store, pack.BuildOptions{}); err != nil { + t.Fatalf("pack build: %v", err) + } + resolver, err := pack.Open(store, pack.OpenOptions{CacheBytes: 16 << 20}) + if err != nil { + t.Fatal(err) + } + defer resolver.Close() + for _, fixture := range corpus.Sessions { + manifest, err := fold.LoadManifest(store, fixture.ID) + if err != nil { + t.Fatal(err) + } + view, err := vfs.NewView(manifest, resolver) + if err != nil { + t.Fatal(err) + } + shadow, err := fsctl.Shadow(context.Background(), fixture.Path, view, fsctl.ShadowOptions{BlockBytes: 64 << 10, RandomReads: 10000, Seed: 42}) + if err != nil || !shadow.Verified { + t.Fatalf("shadow %s: %#v err=%v", fixture.ID, shadow, err) + } + } + fixture := corpus.Sessions[0] + manifest, _ := fold.LoadManifest(store, fixture.ID) + managed, err := vfs.OpenSession(context.Background(), vfs.SessionOptions{Root: store, ManifestPath: fold.ManifestPath(store, fixture.ID), Manifest: manifest, Reader: resolver, NativeSnapshot: vfs.NativeFile{Path: fixture.Path, Bytes: fixture.Bytes, SHA256: fixture.SHA256}}) + if err != nil { + t.Fatal(err) + } + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + start := time.Now() + for index := 0; index < 100000; index++ { + if _, err := writer.Append(context.Background(), []byte("x")); err != nil { + t.Fatalf("append %d: %v", index, err) + } + } + if err := writer.Sync(); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + if testing.Verbose() { + t.Logf("100000 appends: %s", time.Since(start)) + } + current, err := managed.MaterializeCurrent(context.Background(), filepath.Join(root, "current.jsonl"), false) + if err != nil || current.Bytes != fixture.Bytes+100000 { + t.Fatalf("append result: %#v err=%v", current, err) + } + verifyConcurrentReadAndWrite(t, managed) + writer, err = managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + if _, err := writer.WriteAt(context.Background(), []byte("PATCH"), 7); err != nil { + t.Fatal(err) + } + if err := writer.Truncate(context.Background(), current.Bytes/2); err != nil { + t.Fatal(err) + } + _ = writer.Close() + info, err := managed.VisibleInfo() + if err != nil || info.Size != current.Bytes/2 { + t.Fatalf("COW/truncate result: %#v err=%v", info, err) + } +} + +func TestGenerateRolloutWritesExactRequestedBytes(t *testing.T) { + fixture, err := GenerateRollout(filepath.Join(t.TempDir(), "large.jsonl"), (17<<20)+37) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(fixture.Path) + if err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(data) + if int64(len(data)) != fixture.Bytes || hex.EncodeToString(digest[:]) != fixture.SHA256 { + t.Fatalf("generated rollout metadata differs: %#v", fixture) + } +} + +func TestFaultHookFiresExactlyOnceAtRequestedPhase(t *testing.T) { + faults := NewFaults("state-publish") + if err := faults.Hook("prepare"); err != nil { + t.Fatal(err) + } + if err := faults.Hook("state-publish"); err == nil || !faults.Fired() { + t.Fatalf("fault did not fire: %v", err) + } + if err := faults.Hook("state-publish"); err != nil { + t.Fatalf("fault fired twice: %v", err) + } +} + +func verifyConcurrentReadAndWrite(t *testing.T, managed *vfs.Session) { + t.Helper() + writer, err := managed.OpenWriter() + if err != nil { + t.Fatal(err) + } + var wait sync.WaitGroup + errorsSeen := make(chan error, 5) + for worker := 0; worker < 5; worker++ { + wait.Add(1) + go func(seed int64) { + defer wait.Done() + random := rand.New(rand.NewSource(seed)) + for index := 0; index < 1000; index++ { + reader, err := managed.OpenReader() + if err != nil { + errorsSeen <- err + return + } + if reader.Size() > 0 { + offset := random.Int63n(reader.Size()) + buffer := make([]byte, 1) + if _, err := reader.ReadAt(context.Background(), buffer, offset); err != nil && !errors.Is(err, io.EOF) { + _ = reader.Close() + errorsSeen <- err + return + } + } + _ = reader.Close() + } + }(int64(worker + 1)) + } + for index := 0; index < 1000; index++ { + if _, err := writer.Append(context.Background(), []byte("y")); err != nil { + t.Fatal(err) + } + } + if err := writer.Sync(); err != nil { + t.Fatal(err) + } + _ = writer.Close() + wait.Wait() + close(errorsSeen) + for err := range errorsSeen { + t.Fatal(err) + } +} diff --git a/internal/testfs/faults.go b/internal/testfs/faults.go new file mode 100644 index 0000000..574f225 --- /dev/null +++ b/internal/testfs/faults.go @@ -0,0 +1,30 @@ +package testfs + +import ( + "errors" + "sync" +) + +type Faults struct { + mu sync.Mutex + phase string + fired bool +} + +func NewFaults(phase string) *Faults { return &Faults{phase: phase} } + +func (f *Faults) Hook(phase string) error { + f.mu.Lock() + defer f.mu.Unlock() + if !f.fired && phase == f.phase { + f.fired = true + return errors.New("injected fault at " + phase) + } + return nil +} + +func (f *Faults) Fired() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.fired +} diff --git a/internal/testfs/large_test.go b/internal/testfs/large_test.go new file mode 100644 index 0000000..562e18f --- /dev/null +++ b/internal/testfs/large_test.go @@ -0,0 +1,119 @@ +package testfs + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/fsctl" + "github.com/samekind/codexfold/internal/pack" + "github.com/samekind/codexfold/internal/vfs" +) + +type largeGateReport struct { + SourceBytes int64 `json:"source_bytes"` + FoldDuration time.Duration `json:"fold_duration"` + PackDuration time.Duration `json:"pack_duration"` + ShadowDuration time.Duration `json:"shadow_duration"` + Cold fsctl.BenchmarkReport `json:"cold"` + Warm fsctl.BenchmarkReport `json:"warm"` + MaxRSSBytes uint64 `json:"max_rss_bytes"` + UserCPU time.Duration `json:"user_cpu"` + SystemCPU time.Duration `json:"system_cpu"` + PackCacheBytes int64 `json:"pack_cache_bytes"` + PackCacheBypassApplied bool `json:"pack_os_cache_bypass_applied"` + LooseObjectsOff bool `json:"loose_objects_offline"` +} + +func TestLargePreviewBenchmark(t *testing.T) { + if os.Getenv("CODEXFOLD_RUN_LARGE_TEST") != "1" { + t.Skip("set CODEXFOLD_RUN_LARGE_TEST=1 to run the 758 MiB preview gate") + } + usageBefore := processResourceUsage() + root := t.TempDir() + const sourceBytes = int64(758 << 20) + fixture, err := GenerateRollout(filepath.Join(root, "large.jsonl"), sourceBytes) + if err != nil { + t.Fatal(err) + } + store := filepath.Join(root, "store") + foldStart := time.Now() + if _, err := fold.Fold(context.Background(), fold.Session{ID: fixture.ID, RolloutPath: fixture.Path, Archived: true}, fold.FoldOptions{StoreDir: store, Apply: true, FieldThreshold: 1 << 20}); err != nil { + t.Fatal(err) + } + foldDuration := time.Since(foldStart) + packStart := time.Now() + if _, err := pack.Build(context.Background(), store, pack.BuildOptions{}); err != nil { + t.Fatal(err) + } + packDuration := time.Since(packStart) + const cacheBytes = int64(128 << 20) + manifest, err := fold.LoadManifest(store, fixture.ID) + if err != nil { + t.Fatal(err) + } + objects := filepath.Join(store, "objects") + offlineObjects := filepath.Join(store, "objects.offline") + if err := os.Rename(objects, offlineObjects); err != nil { + t.Fatal(err) + } + coldResolver, err := pack.Open(store, pack.OpenOptions{CacheBytes: cacheBytes, BypassOSCache: true}) + if err != nil { + t.Fatal(err) + } + coldView, err := vfs.NewView(manifest, coldResolver) + if err != nil { + _ = coldResolver.Close() + t.Fatal(err) + } + runtime.GC() + coldOptions := fsctl.BenchmarkOptions{SequentialBlockBytes: 4 << 20, RandomBlockBytes: 4 << 10, RandomReads: 10000, Seed: 42, BypassOSCache: true} + cold, err := fsctl.Benchmark(context.Background(), fixture.Path, coldView, coldOptions) + packBypassApplied := coldResolver.OSCacheBypassApplied() + _ = coldResolver.Close() + if err != nil { + t.Fatal(err) + } + if runtime.GOOS == "darwin" && (!cold.OSCacheBypassApplied || !packBypassApplied) { + t.Fatalf("macOS cold gate did not apply F_NOCACHE: native=%t pack=%t", cold.OSCacheBypassApplied, packBypassApplied) + } + warmResolver, err := pack.Open(store, pack.OpenOptions{CacheBytes: cacheBytes}) + if err != nil { + t.Fatal(err) + } + defer warmResolver.Close() + warmView, err := vfs.NewView(manifest, warmResolver) + if err != nil { + t.Fatal(err) + } + shadowStart := time.Now() + shadow, err := fsctl.Shadow(context.Background(), fixture.Path, warmView, fsctl.ShadowOptions{BlockBytes: 4 << 20, RandomReads: 10000, Seed: 42}) + if err != nil || !shadow.Verified { + t.Fatalf("shadow: %#v err=%v", shadow, err) + } + shadowDuration := time.Since(shadowStart) + runtime.GC() + warmOptions := fsctl.BenchmarkOptions{SequentialBlockBytes: 4 << 20, RandomBlockBytes: 4 << 10, RandomReads: 10000, Seed: 42} + warm, err := fsctl.Benchmark(context.Background(), fixture.Path, warmView, warmOptions) + if err != nil { + t.Fatal(err) + } + usage := subtractUsage(processResourceUsage(), usageBefore) + report := largeGateReport{SourceBytes: sourceBytes, FoldDuration: foldDuration, PackDuration: packDuration, ShadowDuration: shadowDuration, Cold: cold, Warm: warm, MaxRSSBytes: usage.MaxRSSBytes, UserCPU: usage.UserCPU, SystemCPU: usage.SystemCPU, PackCacheBytes: cacheBytes, PackCacheBypassApplied: packBypassApplied, LooseObjectsOff: true} + encoded, _ := json.MarshalIndent(report, "", " ") + t.Logf("large preview gate:\n%s", encoded) + if cold.Virtual.BytesPerSecond < 500<<20 || cold.Virtual.BytesPerSecond < cold.Native.BytesPerSecond*0.70 { + t.Fatalf("cold virtual throughput gate failed: native=%.0f virtual=%.0f", cold.Native.BytesPerSecond, cold.Virtual.BytesPerSecond) + } + if warm.Virtual.BytesPerSecond < 500<<20 || warm.Virtual.BytesPerSecond < warm.Native.BytesPerSecond*0.80 { + t.Fatalf("warm virtual throughput gate failed: native=%.0f virtual=%.0f", warm.Native.BytesPerSecond, warm.Virtual.BytesPerSecond) + } + if report.MaxRSSBytes != 0 && report.MaxRSSBytes > 512<<20 { + t.Fatalf("max RSS exceeded 512 MiB: %d", report.MaxRSSBytes) + } +} diff --git a/internal/testfs/resource.go b/internal/testfs/resource.go new file mode 100644 index 0000000..b4f345c --- /dev/null +++ b/internal/testfs/resource.go @@ -0,0 +1,20 @@ +package testfs + +import "time" + +type resourceUsage struct { + MaxRSSBytes uint64 + UserCPU time.Duration + SystemCPU time.Duration +} + +func subtractUsage(after resourceUsage, before resourceUsage) resourceUsage { + result := after + if after.UserCPU >= before.UserCPU { + result.UserCPU = after.UserCPU - before.UserCPU + } + if after.SystemCPU >= before.SystemCPU { + result.SystemCPU = after.SystemCPU - before.SystemCPU + } + return result +} diff --git a/internal/testfs/rss_darwin.go b/internal/testfs/rss_darwin.go new file mode 100644 index 0000000..dd6ec2a --- /dev/null +++ b/internal/testfs/rss_darwin.go @@ -0,0 +1,17 @@ +//go:build darwin + +package testfs + +import ( + "time" + + "golang.org/x/sys/unix" +) + +func processResourceUsage() resourceUsage { + var usage unix.Rusage + if err := unix.Getrusage(unix.RUSAGE_SELF, &usage); err != nil || usage.Maxrss < 0 { + return resourceUsage{} + } + return resourceUsage{MaxRSSBytes: uint64(usage.Maxrss), UserCPU: time.Duration(unix.TimevalToNsec(usage.Utime)), SystemCPU: time.Duration(unix.TimevalToNsec(usage.Stime))} +} diff --git a/internal/testfs/rss_linux.go b/internal/testfs/rss_linux.go new file mode 100644 index 0000000..c95e983 --- /dev/null +++ b/internal/testfs/rss_linux.go @@ -0,0 +1,17 @@ +//go:build linux + +package testfs + +import ( + "time" + + "golang.org/x/sys/unix" +) + +func processResourceUsage() resourceUsage { + var usage unix.Rusage + if err := unix.Getrusage(unix.RUSAGE_SELF, &usage); err != nil || usage.Maxrss < 0 { + return resourceUsage{} + } + return resourceUsage{MaxRSSBytes: uint64(usage.Maxrss) * 1024, UserCPU: time.Duration(unix.TimevalToNsec(usage.Utime)), SystemCPU: time.Duration(unix.TimevalToNsec(usage.Stime))} +} diff --git a/internal/testfs/rss_other.go b/internal/testfs/rss_other.go new file mode 100644 index 0000000..758214d --- /dev/null +++ b/internal/testfs/rss_other.go @@ -0,0 +1,5 @@ +//go:build !darwin && !linux + +package testfs + +func processResourceUsage() resourceUsage { return resourceUsage{} } diff --git a/internal/vfs/compact.go b/internal/vfs/compact.go new file mode 100644 index 0000000..84c5079 --- /dev/null +++ b/internal/vfs/compact.go @@ -0,0 +1,209 @@ +package vfs + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/samekind/codexfold/internal/fold" +) + +type PreparedGeneration struct { + ManifestPath string + Manifest fold.Manifest + View *View +} + +type CompactOptions struct { + IdleFor time.Duration + Prepare func(context.Context, NativeFile, uint64) (PreparedGeneration, error) + BeforePhase func(string) error +} + +type CompactResult struct { + Generation uint64 `json:"generation"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` +} + +func (s *Session) Compact(ctx context.Context, options CompactOptions) (CompactResult, error) { + if options.Prepare == nil { + return CompactResult{}, errors.New("compact preparation function is required") + } + s.mu.Lock() + if s.writerOpen { + s.mu.Unlock() + return CompactResult{}, errors.New("cannot compact while a writer lease is held") + } + s.writerOpen = true + state := s.state + s.mu.Unlock() + lease, err := acquireWriterLease(filepath.Join(s.directory, "writer.lease")) + if err != nil { + s.mu.Lock() + s.writerOpen = false + s.mu.Unlock() + return CompactResult{}, err + } + defer func() { + _ = unlockWriterFile(lease) + _ = lease.Close() + s.mu.Lock() + s.writerOpen = false + s.mu.Unlock() + }() + activePath := state.DeltaPath + if state.BackingPath != "" { + activePath = state.BackingPath + } + fingerprint, err := captureFingerprint(activePath) + if err != nil { + return CompactResult{}, err + } + if options.IdleFor > 0 && time.Since(fingerprint.ModTime) < options.IdleFor { + return CompactResult{}, errors.New("session is not idle enough for compaction") + } + current, err := s.MaterializeCurrent(ctx, filepath.Join(s.directory, fmt.Sprintf(".compact-%020d.jsonl", state.Generation)), true) + if err != nil { + return CompactResult{}, err + } + defer os.Remove(current.Path) + prepared, err := options.Prepare(ctx, current, state.Generation+1) + if err != nil { + return CompactResult{}, err + } + if prepared.View == nil || prepared.ManifestPath == "" || prepared.Manifest.Source.SHA256 != current.SHA256 || prepared.Manifest.Source.Bytes != current.Bytes { + return CompactResult{}, errors.New("compact preparation returned incomplete generation") + } + if prepared.View.Size() != current.Bytes { + return CompactResult{}, errors.New("prepared generation byte length differs from current view") + } + preparedDigest, err := hashView(ctx, prepared.View) + if err != nil { + return CompactResult{}, err + } + if preparedDigest != current.SHA256 { + return CompactResult{}, errors.New("prepared generation SHA-256 differs from current view") + } + if err := ensureFingerprintUnchanged(activePath, fingerprint); err != nil { + return CompactResult{}, err + } + newDelta := filepath.Join(s.directory, fmt.Sprintf("delta-%020d.jsonl", state.Generation+1)) + delta, err := os.OpenFile(newDelta, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) + if err != nil { + return CompactResult{}, err + } + if err := delta.Sync(); err != nil { + _ = delta.Close() + return CompactResult{}, err + } + if err := delta.Close(); err != nil { + return CompactResult{}, err + } + next := state + next.Generation++ + next.ManifestPath = prepared.ManifestPath + next.BaseBytes = prepared.View.Size() + next.BaseSHA256 = current.SHA256 + next.DeltaPath = newDelta + next.BackingPath = "" + operationID := fmt.Sprintf("compact-%020d", state.Generation) + stateTemporary := filepath.Join(s.directory, fmt.Sprintf(".state-compact-%020d.tmp", next.Generation)) + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "prepared", Candidate: next, TempPath: stateTemporary, FinalPath: newDelta, Native: current}); err != nil { + return CompactResult{}, err + } + if options.BeforePhase != nil { + if err := options.BeforePhase("after-prepare"); err != nil { + return CompactResult{}, err + } + } + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "state-publishing", Candidate: next, TempPath: stateTemporary, FinalPath: newDelta, Native: current}); err != nil { + return CompactResult{}, err + } + if options.BeforePhase != nil { + if err := options.BeforePhase("before-state-publish"); err != nil { + return CompactResult{}, err + } + } + if err := writeSessionStateWithTemporary(s.statePath, stateTemporary, next); err != nil { + return CompactResult{}, err + } + s.mu.Lock() + s.state = next + s.view = prepared.View + s.mu.Unlock() + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "state-published", Candidate: next, TempPath: stateTemporary, FinalPath: newDelta, Native: current}); err != nil { + return CompactResult{}, err + } + if options.BeforePhase != nil { + if err := options.BeforePhase("after-state-publish"); err != nil { + return CompactResult{}, err + } + } + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: state.SessionID, Kind: "compact", Phase: "complete", Candidate: next, TempPath: stateTemporary, FinalPath: newDelta, Native: current}); err != nil { + return CompactResult{}, err + } + return CompactResult{Generation: next.Generation, Bytes: current.Bytes, SHA256: current.SHA256}, nil +} + +func hashView(ctx context.Context, view *View) (string, error) { + hasher := sha256.New() + buffer := make([]byte, 1<<20) + var offset int64 + for offset < view.Size() { + need := len(buffer) + if remaining := view.Size() - offset; int64(need) > remaining { + need = int(remaining) + } + n, err := view.ReadAt(ctx, buffer[:need], offset) + if n > 0 { + _, _ = hasher.Write(buffer[:n]) + offset += int64(n) + } + if err != nil && !errors.Is(err, io.EOF) { + return "", err + } + if n == 0 { + break + } + } + if offset != view.Size() { + return "", errors.New("prepared generation ended before its declared size") + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +type fileFingerprint struct { + Bytes int64 + ModTime time.Time + SHA256 string +} + +func captureFingerprint(path string) (fileFingerprint, error) { + info, err := os.Stat(path) + if err != nil { + return fileFingerprint{}, err + } + current, err := hashNativePath(path) + if err != nil { + return fileFingerprint{}, err + } + return fileFingerprint{Bytes: info.Size(), ModTime: info.ModTime(), SHA256: current.SHA256}, nil +} + +func ensureFingerprintUnchanged(path string, initial fileFingerprint) error { + current, err := captureFingerprint(path) + if err != nil { + return err + } + if current.Bytes != initial.Bytes || !current.ModTime.Equal(initial.ModTime) || current.SHA256 != initial.SHA256 { + return errors.New("active session data changed during compaction") + } + return nil +} diff --git a/internal/vfs/fallback.go b/internal/vfs/fallback.go new file mode 100644 index 0000000..ac416c1 --- /dev/null +++ b/internal/vfs/fallback.go @@ -0,0 +1,9 @@ +package vfs + +import ( + "context" +) + +func (s *Session) CreateCurrentNativeBacking(ctx context.Context, target string) (NativeFile, error) { + return s.MaterializeCurrent(ctx, target, false) +} diff --git a/internal/vfs/handles.go b/internal/vfs/handles.go new file mode 100644 index 0000000..d77ce1c --- /dev/null +++ b/internal/vfs/handles.go @@ -0,0 +1,230 @@ +package vfs + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "sync" +) + +type ReadHandle struct { + session *Session + generation uint64 + base *View + baseBytes int64 + file *os.File + backing bool + deltaBytes int64 + size int64 + closeOnce sync.Once + closeErr error +} + +func (h *ReadHandle) Size() int64 { return h.size } + +func (h *ReadHandle) ReadAt(ctx context.Context, destination []byte, offset int64) (int, error) { + if offset < 0 { + return 0, errors.New("negative session read offset") + } + if len(destination) == 0 { + return 0, nil + } + if err := ctx.Err(); err != nil { + return 0, err + } + if offset >= h.size { + return 0, io.EOF + } + if h.backing { + limit := len(destination) + if remaining := h.size - offset; int64(limit) > remaining { + limit = int(remaining) + } + n, err := h.file.ReadAt(destination[:limit], offset) + if err != nil && !errors.Is(err, io.EOF) { + return n, err + } + if n < len(destination) { + return n, io.EOF + } + return n, nil + } + written := 0 + if offset < h.baseBytes { + need := len(destination) + if remaining := h.baseBytes - offset; int64(need) > remaining { + need = int(remaining) + } + n, err := h.base.ReadAt(ctx, destination[:need], offset) + written += n + offset += int64(n) + if n != need { + if err == nil { + err = io.ErrUnexpectedEOF + } + return written, err + } + if err != nil && !errors.Is(err, io.EOF) { + return written, err + } + } + if written < len(destination) && offset >= h.baseBytes && offset < h.size { + deltaOffset := offset - h.baseBytes + need := len(destination) - written + if remaining := h.deltaBytes - deltaOffset; int64(need) > remaining { + need = int(remaining) + } + n, err := h.file.ReadAt(destination[written:written+need], deltaOffset) + written += n + if n != need { + if err == nil { + err = io.ErrUnexpectedEOF + } + return written, err + } + if err != nil && !errors.Is(err, io.EOF) { + return written, err + } + } + if written < len(destination) { + return written, io.EOF + } + return written, nil +} + +func (h *ReadHandle) Close() error { + h.closeOnce.Do(func() { + h.closeErr = errors.Join(h.file.Close(), h.session.releaseReader(h.generation)) + }) + return h.closeErr +} + +type WriteHandle struct { + session *Session + leasePath string + lease *os.File + mu sync.Mutex + closed bool +} + +func (h *WriteHandle) Append(ctx context.Context, data []byte) (int, error) { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return 0, errors.New("writer is closed") + } + if err := ctx.Err(); err != nil { + return 0, err + } + state := h.session.State() + path := state.DeltaPath + if state.BackingPath != "" { + path = state.BackingPath + } + file, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + return 0, fmt.Errorf("open append target: %w", err) + } + n, writeErr := file.Write(data) + closeErr := file.Close() + if writeErr != nil { + return n, writeErr + } + if closeErr != nil { + return n, closeErr + } + return n, nil +} + +func (h *WriteHandle) WriteAt(ctx context.Context, data []byte, offset int64) (int, error) { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return 0, errors.New("writer is closed") + } + if offset < 0 { + return 0, errors.New("negative write offset") + } + path, err := h.session.ensureBacking(ctx) + if err != nil { + return 0, err + } + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return 0, err + } + n, writeErr := file.WriteAt(data, offset) + closeErr := file.Close() + if writeErr != nil { + return n, writeErr + } + return n, closeErr +} + +func (h *WriteHandle) Truncate(ctx context.Context, size int64) error { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return errors.New("writer is closed") + } + if size < 0 { + return errors.New("negative truncate size") + } + visible, err := h.session.VisibleInfo() + if err != nil { + return err + } + if size == visible.Size { + return nil + } + path, err := h.session.ensureBacking(ctx) + if err != nil { + return err + } + return os.Truncate(path, size) +} + +func (h *WriteHandle) Sync() error { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return errors.New("writer is closed") + } + state := h.session.State() + path := state.DeltaPath + if state.BackingPath != "" { + path = state.BackingPath + } + file, err := os.OpenFile(path, os.O_RDWR, 0) + if err != nil { + return err + } + if err := file.Sync(); err != nil { + _ = file.Close() + return err + } + return file.Close() +} + +func (h *WriteHandle) Close() error { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return nil + } + h.closed = true + unlockErr := unlockWriterFile(h.lease) + closeErr := h.lease.Close() + h.session.mu.Lock() + h.session.writerOpen = false + h.session.mu.Unlock() + if unlockErr != nil { + return unlockErr + } + if closeErr != nil { + return closeErr + } + return nil +} diff --git a/internal/vfs/journal.go b/internal/vfs/journal.go new file mode 100644 index 0000000..6d3c3fc --- /dev/null +++ b/internal/vfs/journal.go @@ -0,0 +1,73 @@ +package vfs + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" +) + +type JournalRecord struct { + OperationID string `json:"operation_id"` + SessionID string `json:"session_id"` + Kind string `json:"kind"` + Phase string `json:"phase"` + At string `json:"at"` + TempPath string `json:"temp_path,omitempty"` + FinalPath string `json:"final_path,omitempty"` + Candidate SessionState `json:"candidate"` + Native NativeFile `json:"native,omitempty"` +} + +func journalPath(directory string) string { return filepath.Join(directory, "journal.jsonl") } + +func appendJournal(directory string, record JournalRecord) error { + record.At = time.Now().UTC().Format(time.RFC3339Nano) + data, err := json.Marshal(record) + if err != nil { + return fmt.Errorf("encode journal record: %w", err) + } + file, err := os.OpenFile(journalPath(directory), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("open session journal: %w", err) + } + if _, err := file.Write(append(data, '\n')); err != nil { + _ = file.Close() + return fmt.Errorf("write session journal: %w", err) + } + if err := file.Sync(); err != nil { + _ = file.Close() + return fmt.Errorf("sync session journal: %w", err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("close session journal: %w", err) + } + return nil +} + +func readJournal(directory string) ([]JournalRecord, error) { + file, err := os.Open(journalPath(directory)) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + defer file.Close() + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 4096), 1<<20) + var records []JournalRecord + for scanner.Scan() { + var record JournalRecord + if err := json.Unmarshal(scanner.Bytes(), &record); err != nil { + return nil, fmt.Errorf("decode session journal: %w", err) + } + records = append(records, record) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read session journal: %w", err) + } + return records, nil +} diff --git a/internal/vfs/recover.go b/internal/vfs/recover.go new file mode 100644 index 0000000..e3b5432 --- /dev/null +++ b/internal/vfs/recover.go @@ -0,0 +1,144 @@ +package vfs + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +func (s *Session) recover(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + records, err := readJournal(s.directory) + if err != nil { + return err + } + latest := make(map[string]JournalRecord) + lastPosition := make(map[string]int) + for index, record := range records { + if record.OperationID == "" { + return errors.New("session journal record has no operation ID") + } + latest[record.OperationID] = record + lastPosition[record.OperationID] = index + } + ordered := make([]JournalRecord, 0, len(latest)) + for _, record := range latest { + ordered = append(ordered, record) + } + sort.Slice(ordered, func(i, j int) bool { + return lastPosition[ordered[i].OperationID] < lastPosition[ordered[j].OperationID] + }) + for _, record := range ordered { + switch record.Phase { + case "complete", "rolled-back": + if record.Kind == "compact" { + if err := s.cleanupCompactArtifacts(record, record.Phase == "rolled-back"); err != nil { + return err + } + } + continue + case "after-file-publish", "state-publishing", "state-published": + if record.Candidate.SessionID != s.state.SessionID || record.Candidate.Generation == 0 || !pathWithin(s.directory, record.Candidate.DeltaPath) || (record.Candidate.BackingPath != "" && !pathWithin(s.directory, record.Candidate.BackingPath)) { + return fmt.Errorf("journal operation %s has unsafe candidate state", record.OperationID) + } + state, err := loadSessionState(s.statePath) + if err != nil { + return err + } + if record.Kind == "compact" { + if state.Generation < record.Candidate.Generation { + resolved := record + resolved.Phase = "rolled-back" + if err := appendJournal(s.directory, resolved); err != nil { + return err + } + if err := s.cleanupCompactArtifacts(resolved, true); err != nil { + return err + } + continue + } + if state.Generation != record.Candidate.Generation || state.ManifestPath != record.Candidate.ManifestPath { + return fmt.Errorf("journal operation %s conflicts with current compacted state", record.OperationID) + } + s.state = state + } else { + if record.FinalPath == "" || record.Candidate.BackingPath == "" { + return fmt.Errorf("journal operation %s has incomplete published state", record.OperationID) + } + verified, err := hashNativePath(record.FinalPath) + if err != nil || verified.Bytes != record.Native.Bytes || verified.SHA256 != record.Native.SHA256 { + return fmt.Errorf("journal operation %s published backing cannot be verified: %w", record.OperationID, err) + } + if state.Generation < record.Candidate.Generation { + if err := writeSessionState(s.statePath, record.Candidate); err != nil { + return err + } + s.state = record.Candidate + } + } + resolved := record + resolved.Phase = "complete" + if err := appendJournal(s.directory, resolved); err != nil { + return err + } + if record.Kind == "compact" { + if err := s.cleanupCompactArtifacts(resolved, false); err != nil { + return err + } + } + case "prepared", "data-synced": + resolved := record + resolved.Phase = "rolled-back" + if err := appendJournal(s.directory, resolved); err != nil { + return err + } + if record.Kind == "compact" { + if err := s.cleanupCompactArtifacts(resolved, true); err != nil { + return err + } + } else if record.TempPath != "" { + _ = os.Remove(record.TempPath) + } + default: + return fmt.Errorf("journal operation %s has unknown phase %q", record.OperationID, record.Phase) + } + } + return nil +} + +func (s *Session) cleanupCompactArtifacts(record JournalRecord, rollback bool) error { + removeOwned := func(candidate string, prefix string, suffix string) error { + if candidate == "" { + return nil + } + candidate = filepath.Clean(candidate) + name := filepath.Base(candidate) + if filepath.Dir(candidate) != filepath.Clean(s.directory) || !strings.HasPrefix(name, prefix) || !strings.HasSuffix(name, suffix) { + return fmt.Errorf("journal operation %s has unsafe compact artifact %q", record.OperationID, candidate) + } + if err := os.Remove(candidate); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil + } + if err := removeOwned(record.Native.Path, ".compact-", ".jsonl"); err != nil { + return err + } + if err := removeOwned(record.TempPath, ".state-compact-", ".tmp"); err != nil { + return err + } + if rollback { + if err := removeOwned(record.FinalPath, "delta-", ".jsonl"); err != nil { + return err + } + } + return nil +} + +func (s *Session) Recover(ctx context.Context) error { return s.recover(ctx) } diff --git a/internal/vfs/recovery_test.go b/internal/vfs/recovery_test.go new file mode 100644 index 0000000..78dd962 --- /dev/null +++ b/internal/vfs/recovery_test.go @@ -0,0 +1,383 @@ +package vfs + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/storage" +) + +func TestRecoverFinishesPublishedCopyOnWriteGeneration(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + stop := errors.New("stop after backing publish") + session := openFixtureSession(t, root, manifest, reader, func(phase string) error { + if phase == "after-file-publish" { + return stop + } + return nil + }) + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if _, err := writer.WriteAt(context.Background(), []byte("X"), 0); !errors.Is(err, stop) { + t.Fatalf("WriteAt error = %v, want %v", err, stop) + } + _ = writer.Close() + if session.State().BackingPath != "" { + t.Fatal("interrupted state should not publish backing before recovery") + } + + reopened := openFixtureSession(t, root, manifest, reader, nil) + if reopened.State().BackingPath == "" || reopened.State().Generation != 2 { + t.Fatalf("recovery did not finish COW state: %#v", reopened.State()) + } + handle, err := reopened.OpenReader() + if err != nil { + t.Fatalf("OpenReader: %v", err) + } + defer handle.Close() + if got := readHandle(t, handle); !bytes.Equal(got, source) { + t.Fatalf("recovered backing differs: got=%q want=%q", got, source) + } +} + +func TestCreateCurrentNativeBackingIncludesVirtualTail(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if _, err := writer.Append(context.Background(), []byte("-new-tail")); err != nil { + t.Fatalf("Append: %v", err) + } + _ = writer.Close() + + target := filepath.Join(root, "fallback", "session.jsonl") + backing, err := session.CreateCurrentNativeBacking(context.Background(), target) + if err != nil { + t.Fatalf("CreateCurrentNativeBacking: %v", err) + } + want := append(append([]byte(nil), source...), []byte("-new-tail")...) + got, err := os.ReadFile(target) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("fallback differs: got=%q err=%v", got, err) + } + if backing.SHA256 != digestBytes(want) || backing.SHA256 == manifest.Source.SHA256 { + t.Fatalf("fallback digest does not represent current bytes: %#v", backing) + } +} + +func TestCompactSwitchesGenerationAndPreservesPinnedReader(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, _ := session.OpenWriter() + _, _ = writer.Append(context.Background(), []byte("-tail")) + _ = writer.Close() + oldReader, err := session.OpenReader() + if err != nil { + t.Fatalf("OpenReader before compact: %v", err) + } + defer oldReader.Close() + oldTime := time.Now().Add(-time.Hour) + if err := os.Chtimes(session.State().DeltaPath, oldTime, oldTime); err != nil { + t.Fatalf("age delta: %v", err) + } + + want := append(append([]byte(nil), source...), []byte("-tail")...) + result, err := session.Compact(context.Background(), CompactOptions{ + IdleFor: 10 * time.Minute, + Prepare: func(_ context.Context, current NativeFile, next uint64) (PreparedGeneration, error) { + data, err := os.ReadFile(current.Path) + if err != nil { + return PreparedGeneration{}, err + } + digest := digestBytes(data) + preparedManifest := fold.Manifest{ + Version: fold.ManifestVersion, Kind: fold.ManifestKind, + Session: fold.ManifestSession{ID: "session", RolloutPath: current.Path}, + Source: fold.ManifestSource{Bytes: int64(len(data)), SHA256: digest}, + Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: digest, RawBytes: int64(len(data))}}}, + } + preparedReader := memoryReader{digest: data} + view, err := NewView(preparedManifest, preparedReader) + return PreparedGeneration{ManifestPath: filepath.Join(root, "manifest-generation-2.json"), Manifest: preparedManifest, View: view}, err + }, + }) + if err != nil { + t.Fatalf("Compact: %v", err) + } + if result.Generation != 2 || session.State().BackingPath != "" { + t.Fatalf("unexpected compact result/state: result=%#v state=%#v", result, session.State()) + } + newReader, err := session.OpenReader() + if err != nil { + t.Fatalf("OpenReader after compact: %v", err) + } + defer newReader.Close() + if got := readHandle(t, newReader); !bytes.Equal(got, want) { + t.Fatalf("compacted generation differs: got=%q want=%q", got, want) + } + if got := readHandle(t, oldReader); !bytes.Equal(got, want) { + t.Fatalf("pinned old reader changed: got=%q want=%q", got, want) + } +} + +func TestStorageGCKeepsOldSessionGenerationUntilReaderLeaseCloses(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, err := session.OpenWriter() + if err != nil { + t.Fatal(err) + } + if _, err := writer.Append(context.Background(), []byte("-tail")); err != nil { + t.Fatal(err) + } + _ = writer.Close() + oldDelta := session.State().DeltaPath + oldReader, err := session.OpenReader() + if err != nil { + t.Fatal(err) + } + want := append(append([]byte(nil), source...), []byte("-tail")...) + _, err = session.Compact(context.Background(), CompactOptions{Prepare: func(_ context.Context, current NativeFile, _ uint64) (PreparedGeneration, error) { + data, err := os.ReadFile(current.Path) + if err != nil { + return PreparedGeneration{}, err + } + digest := digestBytes(data) + prepared := fold.Manifest{ + Version: fold.ManifestVersion, Kind: fold.ManifestKind, + Session: fold.ManifestSession{ID: "session", RolloutPath: current.Path}, + Source: fold.ManifestSource{Bytes: int64(len(data)), SHA256: digest}, + Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: digest, RawBytes: int64(len(data))}}}, + } + view, err := NewView(prepared, memoryReader{digest: data}) + return PreparedGeneration{ManifestPath: filepath.Join(root, "manifest-generation-2.json"), Manifest: prepared, View: view}, err + }}) + if err != nil { + t.Fatal(err) + } + if got := readHandle(t, oldReader); !bytes.Equal(got, want) { + t.Fatalf("old reader changed: %q", got) + } + blocked, err := storage.Collect(context.Background(), storage.GCOptions{StoreDir: root, Apply: true}) + if err != nil { + t.Fatal(err) + } + if blocked.RemovedCount != 0 { + t.Fatalf("active reader generation was collected: %#v", blocked) + } + if _, err := os.Stat(oldDelta); err != nil { + t.Fatalf("old delta missing while reader lease active: %v", err) + } + if err := oldReader.Close(); err != nil { + t.Fatal(err) + } + collected, err := storage.Collect(context.Background(), storage.GCOptions{StoreDir: root, Apply: true}) + if err != nil { + t.Fatal(err) + } + if collected.RemovedCount != 1 { + t.Fatalf("closed reader generation was not collected: %#v", collected) + } + if _, err := os.Stat(oldDelta); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("old delta remains after reader close: %v", err) + } +} + +func TestCompactRejectsDeltaChangedDuringPreparation(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, _ := session.OpenWriter() + _, _ = writer.Append(context.Background(), []byte("initial")) + _ = writer.Close() + oldTime := time.Now().Add(-time.Hour) + _ = os.Chtimes(session.State().DeltaPath, oldTime, oldTime) + + _, err := session.Compact(context.Background(), CompactOptions{ + IdleFor: 10 * time.Minute, + Prepare: func(_ context.Context, current NativeFile, _ uint64) (PreparedGeneration, error) { + file, openErr := os.OpenFile(session.State().DeltaPath, os.O_APPEND|os.O_WRONLY, 0) + if openErr != nil { + return PreparedGeneration{}, openErr + } + _, _ = file.WriteString("changed") + _ = file.Close() + data, _ := os.ReadFile(current.Path) + digest := digestBytes(data) + preparedManifest := fold.Manifest{Version: fold.ManifestVersion, Kind: fold.ManifestKind, Session: fold.ManifestSession{ID: "session"}, Source: fold.ManifestSource{Bytes: int64(len(data)), SHA256: digest}, Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: digest, RawBytes: int64(len(data))}}}} + view, _ := NewView(preparedManifest, memoryReader{digest: data}) + return PreparedGeneration{ManifestPath: filepath.Join(root, "next.json"), Manifest: preparedManifest, View: view}, nil + }, + }) + if err == nil { + t.Fatal("Compact should reject a delta changed during preparation") + } + if session.State().Generation != 1 { + t.Fatalf("failed compact changed generation: %#v", session.State()) + } +} + +func TestCompactBudgetRejectsBeforeScratchOrPreparation(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + checker := &vfsRejectingChecker{} + session, err := OpenSession(context.Background(), SessionOptions{ + Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: reader, + NativeSnapshot: NativeFile{Path: manifest.Session.RolloutPath, Bytes: manifest.Source.Bytes, SHA256: manifest.Source.SHA256}, + Budget: checker, + }) + if err != nil { + t.Fatal(err) + } + prepared := false + if _, err := session.Compact(context.Background(), CompactOptions{Prepare: func(context.Context, NativeFile, uint64) (PreparedGeneration, error) { + prepared = true + return PreparedGeneration{}, nil + }}); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("Compact error = %v, want storage budget rejection", err) + } + if prepared { + t.Fatal("compact preparation ran after budget rejection") + } + entries, err := os.ReadDir(filepath.Join(root, "fs", "sessions", "session")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".compact-") { + t.Fatalf("compact scratch exists after preflight rejection: %s", entry.Name()) + } + } +} + +func TestCompactRejectsWriterLeaseHeldByAnotherSession(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + serving := openFixtureSession(t, root, manifest, reader, nil) + writer, err := serving.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + defer writer.Close() + + maintenance := openFixtureSession(t, root, manifest, reader, nil) + _, err = maintenance.Compact(context.Background(), CompactOptions{ + Prepare: func(context.Context, NativeFile, uint64) (PreparedGeneration, error) { + t.Fatal("compact preparation ran while another process held the writer lease") + return PreparedGeneration{}, nil + }, + }) + if !errors.Is(err, ErrWriterBusy) { + t.Fatalf("Compact error = %v, want %v", err, ErrWriterBusy) + } +} + +func TestCompactHoldsAndReleasesInProcessWriterState(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + stop := errors.New("stop during preparation") + _, err := session.Compact(context.Background(), CompactOptions{ + Prepare: func(context.Context, NativeFile, uint64) (PreparedGeneration, error) { + session.mu.Lock() + held := session.writerOpen + session.mu.Unlock() + if !held { + t.Fatal("compact did not publish its in-process writer state") + } + if writer, writerErr := session.OpenWriter(); !errors.Is(writerErr, ErrWriterBusy) { + if writer != nil { + _ = writer.Close() + } + t.Fatalf("OpenWriter during compact = %v, want %v", writerErr, ErrWriterBusy) + } + return PreparedGeneration{}, stop + }, + }) + if !errors.Is(err, stop) { + t.Fatalf("Compact error = %v, want %v", err, stop) + } + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter after compact failure: %v", err) + } + _ = writer.Close() +} + +func TestRecoverInterruptedCompactRemovesJournalOwnedScratch(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + state := session.State() + next := state + next.Generation++ + next.DeltaPath = filepath.Join(session.directory, "delta-00000000000000000002.jsonl") + scratch := filepath.Join(session.directory, ".compact-00000000000000000001.jsonl") + stateTemporary := filepath.Join(session.directory, ".state-compact-00000000000000000002.tmp") + for path, data := range map[string][]byte{ + next.DeltaPath: nil, + scratch: source, + stateTemporary: []byte("partial state"), + } { + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write interrupted compact artifact %s: %v", path, err) + } + } + if err := appendJournal(session.directory, JournalRecord{ + OperationID: "compact-00000000000000000001", SessionID: state.SessionID, + Kind: "compact", Phase: "state-publishing", Candidate: next, + TempPath: stateTemporary, FinalPath: next.DeltaPath, + Native: NativeFile{Path: scratch, Bytes: int64(len(source)), SHA256: digestBytes(source)}, + }); err != nil { + t.Fatalf("append interrupted compact journal: %v", err) + } + + reopened := openFixtureSession(t, root, manifest, reader, nil) + if reopened.State().Generation != state.Generation || reopened.State().DeltaPath != state.DeltaPath { + t.Fatalf("recovery changed committed state: %#v", reopened.State()) + } + for _, path := range []string{next.DeltaPath, scratch, stateTemporary} { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("recovery left interrupted compact artifact %s: %v", path, err) + } + } + records, err := readJournal(session.directory) + if err != nil { + t.Fatalf("read recovered journal: %v", err) + } + latest := records[len(records)-1] + if latest.Phase != "rolled-back" || latest.TempPath != stateTemporary || latest.Native.Path != scratch { + t.Fatalf("recovery did not preserve cleanup ownership: %#v", latest) + } +} + +func TestOpenSessionCleansUnlockedStaleWriterLease(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + leasePath := filepath.Join(session.directory, "writer.lease") + if err := os.WriteFile(leasePath, []byte("stale\n"), 0o600); err != nil { + t.Fatalf("write stale lease: %v", err) + } + reopened := openFixtureSession(t, root, manifest, reader, nil) + writer, err := reopened.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter after stale lease cleanup: %v", err) + } + _ = writer.Close() +} diff --git a/internal/vfs/resolver.go b/internal/vfs/resolver.go new file mode 100644 index 0000000..6d8da56 --- /dev/null +++ b/internal/vfs/resolver.go @@ -0,0 +1,11 @@ +package vfs + +import ( + "context" + + "github.com/samekind/codexfold/internal/fold" +) + +type ObjectReader interface { + ReadAt(context.Context, fold.ObjectRef, []byte, int64) (int, error) +} diff --git a/internal/vfs/session.go b/internal/vfs/session.go new file mode 100644 index 0000000..f7098b2 --- /dev/null +++ b/internal/vfs/session.go @@ -0,0 +1,591 @@ +package vfs + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" + + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/storage" +) + +type SessionOptions struct { + Root string + ManifestPath string + Manifest fold.Manifest + Reader ObjectReader + NativeSnapshot NativeFile + Budget storage.Checker + BeforeCOWPhase func(string) error +} + +type Session struct { + mu sync.Mutex + state SessionState + statePath string + directory string + view *View + readerLeases map[uint64]int + readerLeaseFiles map[uint64]*storage.Lease + writerOpen bool + budget storage.Checker + beforeCOWPhase func(string) error +} + +type VisibleInfo struct { + Size int64 + ModTime time.Time + Generation uint64 +} + +var ErrWriterBusy = errors.New("session writer lease is already held") + +type WriterLeaseGuard struct { + file *os.File +} + +func OpenSession(ctx context.Context, options SessionOptions) (*Session, error) { + session, _, err := openSession(ctx, options, false) + return session, err +} + +func OpenSessionWithWriter(ctx context.Context, options SessionOptions) (*Session, *WriteHandle, error) { + return openSession(ctx, options, true) +} + +func openSession(ctx context.Context, options SessionOptions, reserveWriter bool) (*Session, *WriteHandle, error) { + if err := ctx.Err(); err != nil { + return nil, nil, err + } + if options.Root == "" || options.ManifestPath == "" || !safeSessionID(options.Manifest.Session.ID) { + return nil, nil, errors.New("session root, manifest path, and safe session ID are required") + } + budget := options.Budget + if budget == nil { + guard, err := storage.DefaultGuard(options.Root) + if err != nil { + return nil, nil, err + } + budget = guard + } + view, err := NewView(options.Manifest, options.Reader) + if err != nil { + return nil, nil, err + } + directory := filepath.Join(options.Root, "fs", "sessions", options.Manifest.Session.ID) + if err := os.MkdirAll(directory, 0o700); err != nil { + return nil, nil, fmt.Errorf("create virtual session directory: %w", err) + } + leasePath := filepath.Join(directory, "writer.lease") + var reservedLease *os.File + if reserveWriter { + reservedLease, err = acquireWriterLease(leasePath) + if err != nil { + return nil, nil, err + } + } else if err := cleanupStaleWriterLease(leasePath); err != nil { + return nil, nil, err + } + cleanupReservedLease := func() { + if reservedLease == nil { + return + } + _ = unlockWriterFile(reservedLease) + _ = reservedLease.Close() + } + statePath := filepath.Join(directory, "state.json") + state, err := loadSessionState(statePath) + if errors.Is(err, os.ErrNotExist) { + if err := verifyNativeFile(options.NativeSnapshot); err != nil { + cleanupReservedLease() + return nil, nil, err + } + deltaPath := filepath.Join(directory, "delta.jsonl") + delta, err := os.OpenFile(deltaPath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) + if err != nil { + cleanupReservedLease() + return nil, nil, fmt.Errorf("create session delta: %w", err) + } + if err := delta.Sync(); err != nil { + _ = delta.Close() + cleanupReservedLease() + return nil, nil, fmt.Errorf("sync session delta: %w", err) + } + if err := delta.Close(); err != nil { + cleanupReservedLease() + return nil, nil, fmt.Errorf("close session delta: %w", err) + } + state = SessionState{Version: sessionStateVersion, SessionID: options.Manifest.Session.ID, Generation: 1, ManifestPath: filepath.Clean(options.ManifestPath), BaseBytes: view.Size(), BaseSHA256: options.Manifest.Source.SHA256, DeltaPath: deltaPath, NativeSnapshot: options.NativeSnapshot} + if err := writeSessionState(statePath, state); err != nil { + cleanupReservedLease() + return nil, nil, err + } + } else if err != nil { + cleanupReservedLease() + return nil, nil, err + } else { + if state.SessionID != options.Manifest.Session.ID || state.ManifestPath != filepath.Clean(options.ManifestPath) || state.BaseBytes != view.Size() || state.BaseSHA256 != options.Manifest.Source.SHA256 || state.NativeSnapshot != options.NativeSnapshot { + cleanupReservedLease() + return nil, nil, errors.New("persisted session state does not match the requested manifest") + } + if !pathWithin(directory, state.DeltaPath) || (state.BackingPath != "" && !pathWithin(directory, state.BackingPath)) { + cleanupReservedLease() + return nil, nil, errors.New("persisted session state contains an unsafe data path") + } + if _, err := os.Stat(state.DeltaPath); err != nil { + cleanupReservedLease() + return nil, nil, fmt.Errorf("stat session delta: %w", err) + } + if state.BackingPath != "" { + if _, err := os.Stat(state.BackingPath); err != nil { + cleanupReservedLease() + return nil, nil, fmt.Errorf("stat session backing: %w", err) + } + } + } + session := &Session{ + state: state, statePath: statePath, directory: directory, view: view, + readerLeases: make(map[uint64]int), readerLeaseFiles: make(map[uint64]*storage.Lease), + budget: budget, beforeCOWPhase: options.BeforeCOWPhase, + } + var writer *WriteHandle + if reservedLease != nil { + session.writerOpen = true + writer = &WriteHandle{session: session, leasePath: leasePath, lease: reservedLease} + } + if err := session.recover(ctx); err != nil { + if writer != nil { + _ = writer.Close() + } + return nil, nil, err + } + if recovered, err := loadSessionState(statePath); err == nil { + session.state = recovered + } + return session, writer, nil +} + +func (s *Session) State() SessionState { + s.mu.Lock() + defer s.mu.Unlock() + return s.state +} + +func (s *Session) MetadataPath() string { + s.mu.Lock() + defer s.mu.Unlock() + if s.state.BackingPath != "" { + return s.state.BackingPath + } + return s.state.DeltaPath +} + +func (s *Session) VisibleInfo() (VisibleInfo, error) { + s.mu.Lock() + state := s.state + s.mu.Unlock() + path := state.DeltaPath + if state.BackingPath != "" { + path = state.BackingPath + } + info, err := os.Stat(path) + if err != nil { + return VisibleInfo{}, err + } + size := info.Size() + if state.BackingPath == "" { + size += state.BaseBytes + } + return VisibleInfo{Size: size, ModTime: info.ModTime(), Generation: state.Generation}, nil +} + +func (s *Session) OpenReader() (*ReadHandle, error) { + s.mu.Lock() + state := s.state + view := s.view + if s.readerLeases[state.Generation] == 0 { + leaseRoot := filepath.Join(s.directory, "leases") + if err := os.Mkdir(leaseRoot, 0o700); err != nil && !errors.Is(err, os.ErrExist) { + s.mu.Unlock() + return nil, fmt.Errorf("create reader lease root: %w", err) + } + leaseDirectory := filepath.Join(s.directory, "leases", fmt.Sprintf("generation-%020d", state.Generation)) + lease, err := storage.AcquireLease(leaseDirectory, "reader") + if err != nil { + s.mu.Unlock() + return nil, fmt.Errorf("acquire reader generation lease: %w", err) + } + s.readerLeaseFiles[state.Generation] = lease + } + s.readerLeases[state.Generation]++ + s.mu.Unlock() + + handle := &ReadHandle{session: s, generation: state.Generation, base: view, baseBytes: state.BaseBytes} + var path string + if state.BackingPath != "" { + path = state.BackingPath + handle.backing = true + } else { + path = state.DeltaPath + } + file, err := os.Open(path) + if err != nil { + _ = s.releaseReader(state.Generation) + return nil, fmt.Errorf("open session reader file: %w", err) + } + info, err := file.Stat() + if err != nil { + _ = file.Close() + _ = s.releaseReader(state.Generation) + return nil, fmt.Errorf("stat session reader file: %w", err) + } + handle.file = file + if handle.backing { + handle.size = info.Size() + } else { + handle.deltaBytes = info.Size() + handle.size = state.BaseBytes + info.Size() + } + return handle, nil +} + +func (s *Session) releaseReader(generation uint64) error { + s.mu.Lock() + var lease *storage.Lease + if s.readerLeases[generation] <= 1 { + delete(s.readerLeases, generation) + lease = s.readerLeaseFiles[generation] + delete(s.readerLeaseFiles, generation) + } else { + s.readerLeases[generation]-- + } + s.mu.Unlock() + return lease.Close() +} + +func (s *Session) OpenWriter() (*WriteHandle, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.writerOpen { + return nil, ErrWriterBusy + } + leasePath := filepath.Join(s.directory, "writer.lease") + lease, err := acquireWriterLease(leasePath) + if err != nil { + return nil, err + } + s.writerOpen = true + return &WriteHandle{session: s, leasePath: leasePath, lease: lease}, nil +} + +func acquireWriterLease(leasePath string) (*os.File, error) { + lease, err := os.OpenFile(leasePath, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("create writer lease: %w", err) + } + locked, err := tryLockWriterFile(lease) + if err != nil { + _ = lease.Close() + return nil, err + } + if !locked { + _ = lease.Close() + return nil, ErrWriterBusy + } + if err := lease.Truncate(0); err != nil { + _ = unlockWriterFile(lease) + _ = lease.Close() + return nil, err + } + if _, err := fmt.Fprintf(lease, "%d\n", os.Getpid()); err != nil { + _ = unlockWriterFile(lease) + _ = lease.Close() + return nil, fmt.Errorf("write writer lease: %w", err) + } + if err := lease.Sync(); err != nil { + _ = unlockWriterFile(lease) + _ = lease.Close() + return nil, fmt.Errorf("sync writer lease: %w", err) + } + return lease, nil +} + +// TryAcquireWriterLeaseGuard reserves the process-wide writer lease without +// changing its diagnostic payload. Recovery uses it to distinguish an +// abandoned transaction from one whose owner is still alive. +func TryAcquireWriterLeaseGuard(root string, sessionID string) (*WriterLeaseGuard, bool, error) { + if root == "" || !safeSessionID(sessionID) { + return nil, false, errors.New("session root and safe session ID are required") + } + leasePath := filepath.Join(filepath.Clean(root), "fs", "sessions", sessionID, "writer.lease") + file, err := os.OpenFile(leasePath, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, false, fmt.Errorf("open writer lease guard: %w", err) + } + locked, err := tryLockWriterFile(file) + if err != nil { + _ = file.Close() + return nil, false, err + } + if !locked { + _ = file.Close() + return nil, false, nil + } + return &WriterLeaseGuard{file: file}, true, nil +} + +func (g *WriterLeaseGuard) Close() error { + if g == nil || g.file == nil { + return nil + } + file := g.file + g.file = nil + return errors.Join(unlockWriterFile(file), file.Close()) +} + +func (s *Session) ensureBacking(ctx context.Context) (string, error) { + s.mu.Lock() + if s.state.BackingPath != "" { + path := s.state.BackingPath + s.mu.Unlock() + return path, nil + } + currentGeneration := s.state.Generation + s.mu.Unlock() + + reader, err := s.OpenReader() + if err != nil { + return "", err + } + defer reader.Close() + if _, err := s.budget.Check(ctx, storage.Projection{ + Operation: "copy-on-write", AdditionalPersistentBytes: reader.Size(), TemporaryBytes: reader.Size(), + TemporaryPersistentOverlapBytes: reader.Size(), + }); err != nil { + return "", err + } + temporary, err := os.CreateTemp(s.directory, ".backing-*.tmp") + if err != nil { + return "", fmt.Errorf("create temporary backing: %w", err) + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return "", err + } + sourceHash := sha256.New() + buffer := make([]byte, 1<<20) + var offset int64 + for offset < reader.Size() { + if err := ctx.Err(); err != nil { + _ = temporary.Close() + return "", err + } + need := len(buffer) + if remaining := reader.Size() - offset; int64(need) > remaining { + need = int(remaining) + } + n, readErr := reader.ReadAt(ctx, buffer[:need], offset) + if n > 0 { + _, _ = sourceHash.Write(buffer[:n]) + if _, err := temporary.Write(buffer[:n]); err != nil { + _ = temporary.Close() + return "", fmt.Errorf("write temporary backing: %w", err) + } + offset += int64(n) + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + _ = temporary.Close() + return "", readErr + } + if n == 0 { + break + } + } + if offset != reader.Size() { + _ = temporary.Close() + return "", errors.New("temporary backing source read ended early") + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return "", fmt.Errorf("sync temporary backing: %w", err) + } + if err := temporary.Close(); err != nil { + return "", fmt.Errorf("close temporary backing: %w", err) + } + verified, err := hashNativePath(temporaryPath) + if err != nil { + return "", err + } + if verified.Bytes != reader.Size() || verified.SHA256 != hex.EncodeToString(sourceHash.Sum(nil)) { + return "", errors.New("temporary backing verification failed") + } + operationID := fmt.Sprintf("cow-%020d", currentGeneration) + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: s.state.SessionID, Kind: "copy-on-write", Phase: "data-synced", TempPath: temporaryPath, Native: verified}); err != nil { + return "", err + } + if s.beforeCOWPhase != nil { + if err := s.beforeCOWPhase("before-publish"); err != nil { + return "", err + } + } + backingPath := filepath.Join(s.directory, fmt.Sprintf("backing-%020d.jsonl", currentGeneration+1)) + if err := replaceStateFile(temporaryPath, backingPath); err != nil { + return "", fmt.Errorf("publish session backing: %w", err) + } + if err := syncStateDirectory(s.directory); err != nil { + return "", err + } + candidate := s.state + candidate.Generation = currentGeneration + 1 + candidate.BackingPath = backingPath + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: s.state.SessionID, Kind: "copy-on-write", Phase: "after-file-publish", Candidate: candidate, FinalPath: backingPath, Native: verified}); err != nil { + return "", err + } + if s.beforeCOWPhase != nil { + if err := s.beforeCOWPhase("after-file-publish"); err != nil { + return "", err + } + } + s.mu.Lock() + defer s.mu.Unlock() + if s.state.Generation != currentGeneration || s.state.BackingPath != "" { + return "", errors.New("session generation changed during copy-on-write") + } + next := s.state + next.Generation++ + next.BackingPath = backingPath + if err := writeSessionState(s.statePath, next); err != nil { + return "", err + } + s.state = next + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: next.SessionID, Kind: "copy-on-write", Phase: "state-published", Candidate: next, FinalPath: backingPath, Native: verified}); err != nil { + return "", err + } + if err := appendJournal(s.directory, JournalRecord{OperationID: operationID, SessionID: next.SessionID, Kind: "copy-on-write", Phase: "complete", Candidate: next, FinalPath: backingPath, Native: verified}); err != nil { + return "", err + } + return backingPath, nil +} + +func (s *Session) MaterializeCurrent(ctx context.Context, target string, overwrite bool) (NativeFile, error) { + if target == "" { + return NativeFile{}, errors.New("materialize target is required") + } + if !overwrite { + if _, err := os.Stat(target); err == nil { + return NativeFile{}, fmt.Errorf("materialize target already exists: %s", target) + } else if !errors.Is(err, os.ErrNotExist) { + return NativeFile{}, err + } + } + reader, err := s.OpenReader() + if err != nil { + return NativeFile{}, err + } + defer reader.Close() + reclaimableBytes := int64(0) + if overwrite { + if info, err := os.Stat(target); err == nil && info.Mode().IsRegular() { + reclaimableBytes = info.Size() + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return NativeFile{}, err + } + } + if _, err := s.budget.Check(ctx, storage.Projection{ + Operation: "materialize-current", AdditionalPersistentBytes: reader.Size(), TemporaryBytes: reader.Size(), + TemporaryPersistentOverlapBytes: reader.Size(), ReclaimableBytes: reclaimableBytes, + }); err != nil { + return NativeFile{}, err + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return NativeFile{}, err + } + temporary, err := os.CreateTemp(filepath.Dir(target), ".materialize-*.tmp") + if err != nil { + return NativeFile{}, err + } + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return NativeFile{}, err + } + hasher := sha256.New() + buffer := make([]byte, 1<<20) + var offset int64 + for offset < reader.Size() { + if err := ctx.Err(); err != nil { + _ = temporary.Close() + return NativeFile{}, err + } + need := len(buffer) + if remaining := reader.Size() - offset; int64(need) > remaining { + need = int(remaining) + } + n, readErr := reader.ReadAt(ctx, buffer[:need], offset) + if n > 0 { + _, _ = hasher.Write(buffer[:n]) + if _, err := temporary.Write(buffer[:n]); err != nil { + _ = temporary.Close() + return NativeFile{}, err + } + offset += int64(n) + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + _ = temporary.Close() + return NativeFile{}, readErr + } + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return NativeFile{}, err + } + if err := temporary.Close(); err != nil { + return NativeFile{}, err + } + if overwrite { + if err := replaceStateFile(temporaryPath, target); err != nil { + return NativeFile{}, err + } + } else if err := os.Rename(temporaryPath, target); err != nil { + return NativeFile{}, err + } + if err := syncStateDirectory(filepath.Dir(target)); err != nil { + return NativeFile{}, err + } + expected := NativeFile{Path: target, Bytes: offset, SHA256: hex.EncodeToString(hasher.Sum(nil))} + verified, err := hashNativePath(target) + if err != nil { + return NativeFile{}, err + } + if verified.Bytes != expected.Bytes || verified.SHA256 != expected.SHA256 { + return NativeFile{}, errors.New("materialized current session verification failed") + } + return expected, nil +} + +func hashNativePath(path string) (NativeFile, error) { + file, err := os.Open(path) + if err != nil { + return NativeFile{}, err + } + hasher := sha256.New() + bytesRead, copyErr := io.Copy(hasher, file) + closeErr := file.Close() + if copyErr != nil { + return NativeFile{}, copyErr + } + if closeErr != nil { + return NativeFile{}, closeErr + } + return NativeFile{Path: path, Bytes: bytesRead, SHA256: hex.EncodeToString(hasher.Sum(nil))}, nil +} diff --git a/internal/vfs/session_test.go b/internal/vfs/session_test.go new file mode 100644 index 0000000..6e4358e --- /dev/null +++ b/internal/vfs/session_test.go @@ -0,0 +1,366 @@ +package vfs + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/samekind/codexfold/internal/fold" + "github.com/samekind/codexfold/internal/storage" +) + +func TestSessionAppendPersistsWithoutHydratingBase(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + + oldReader, err := session.OpenReader() + if err != nil { + t.Fatalf("OpenReader before append: %v", err) + } + defer oldReader.Close() + + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if _, err := writer.Append(context.Background(), []byte("-durable-tail")); err != nil { + t.Fatalf("Append: %v", err) + } + if err := writer.Sync(); err != nil { + t.Fatalf("Sync: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("Close writer: %v", err) + } + + if got := readHandle(t, oldReader); !bytes.Equal(got, source) { + t.Fatalf("old generation changed after append: %q", got) + } + newReader, err := session.OpenReader() + if err != nil { + t.Fatalf("OpenReader after append: %v", err) + } + want := append(append([]byte(nil), source...), []byte("-durable-tail")...) + if got := readHandle(t, newReader); !bytes.Equal(got, want) { + t.Fatalf("new generation bytes differ: got=%q want=%q", got, want) + } + _ = newReader.Close() + + state := session.State() + if state.BackingPath != "" { + t.Fatalf("append hydrated a backing file: %#v", state) + } + if info, err := os.Stat(state.DeltaPath); err != nil || info.Size() != int64(len("-durable-tail")) { + t.Fatalf("delta state differs: info=%v err=%v", info, err) + } + + reopened := openFixtureSession(t, root, manifest, reader, nil) + reopenedReader, err := reopened.OpenReader() + if err != nil { + t.Fatalf("reopened OpenReader: %v", err) + } + defer reopenedReader.Close() + if got := readHandle(t, reopenedReader); !bytes.Equal(got, want) { + t.Fatalf("reopened bytes differ: got=%q want=%q", got, want) + } +} + +func TestSessionAllowsOnlyOneWriterLease(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + first, err := session.OpenWriter() + if err != nil { + t.Fatalf("first OpenWriter: %v", err) + } + if _, err := session.OpenWriter(); err == nil { + t.Fatal("second OpenWriter should fail while the lease is held") + } + if err := first.Close(); err != nil { + t.Fatalf("close first writer: %v", err) + } + second, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter after release: %v", err) + } + _ = second.Close() +} + +func TestSessionReaderHoldsGenerationLeaseUntilClose(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + handle, err := session.OpenReader() + if err != nil { + t.Fatal(err) + } + leaseDirectory := filepath.Join(root, "fs", "sessions", "session", "leases", "generation-00000000000000000001") + active, err := storage.DirectoryHasActiveLease(leaseDirectory, false) + if err != nil || !active { + t.Fatalf("reader generation lease: active=%t err=%v", active, err) + } + if err := handle.Close(); err != nil { + t.Fatal(err) + } + active, err = storage.DirectoryHasActiveLease(leaseDirectory, true) + if err != nil || active { + t.Fatalf("closed reader generation lease: active=%t err=%v", active, err) + } +} + +func TestSessionReaderDoesNotRecreateRetiredStateDirectory(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + stateDirectory := filepath.Dir(session.State().DeltaPath) + retired := stateDirectory + ".retired" + if err := os.Rename(stateDirectory, retired); err != nil { + t.Fatal(err) + } + if _, err := session.OpenReader(); err == nil { + t.Fatal("reader unexpectedly opened after the state directory was retired") + } + if _, err := os.Lstat(stateDirectory); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("retired state directory was recreated: %v", err) + } +} + +func TestSessionRandomWriteTransitionsToVerifiedBacking(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if _, err := writer.Append(context.Background(), []byte("-tail")); err != nil { + t.Fatalf("Append: %v", err) + } + if _, err := writer.WriteAt(context.Background(), []byte("PATCH"), 3); err != nil { + t.Fatalf("WriteAt: %v", err) + } + if err := writer.Sync(); err != nil { + t.Fatalf("Sync: %v", err) + } + _ = writer.Close() + + want := append(append([]byte(nil), source...), []byte("-tail")...) + copy(want[3:], []byte("PATCH")) + current, err := session.MaterializeCurrent(context.Background(), filepath.Join(root, "current.jsonl"), false) + if err != nil { + t.Fatalf("MaterializeCurrent: %v", err) + } + if current.SHA256 != digestBytes(want) || current.Bytes != int64(len(want)) { + t.Fatalf("materialized metadata differs: %#v", current) + } + got, err := os.ReadFile(current.Path) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("materialized bytes differ: bytes=%q err=%v", got, err) + } + if session.State().BackingPath == "" { + t.Fatal("random write did not activate a backing file") + } +} + +func TestSessionBudgetRejectsCopyOnWriteBeforeCreatingBacking(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + checker := &vfsRejectingChecker{} + session, err := OpenSession(context.Background(), SessionOptions{ + Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: reader, + NativeSnapshot: NativeFile{Path: manifest.Session.RolloutPath, Bytes: manifest.Source.Bytes, SHA256: manifest.Source.SHA256}, + Budget: checker, + }) + if err != nil { + t.Fatalf("OpenSession: %v", err) + } + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if _, err := writer.WriteAt(context.Background(), []byte("X"), 0); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("WriteAt error = %v, want storage budget rejection", err) + } + _ = writer.Close() + if checker.Calls != 1 || checker.Projection.Operation != "copy-on-write" || checker.Projection.TemporaryBytes != int64(len(source)) { + t.Fatalf("unexpected COW budget projection: %#v", checker) + } + if state := session.State(); state.BackingPath != "" || state.Generation != 1 { + t.Fatalf("budget rejection changed session state: %#v", state) + } + entries, err := os.ReadDir(filepath.Join(root, "fs", "sessions", "session")) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.Contains(entry.Name(), "backing") { + t.Fatalf("backing artifact exists after preflight rejection: %s", entry.Name()) + } + } +} + +func TestSessionBudgetRejectsMaterializeBeforeCreatingTarget(t *testing.T) { + root := t.TempDir() + manifest, reader, _ := sessionFixture(t, root) + checker := &vfsRejectingChecker{} + session, err := OpenSession(context.Background(), SessionOptions{ + Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: reader, + NativeSnapshot: NativeFile{Path: manifest.Session.RolloutPath, Bytes: manifest.Source.Bytes, SHA256: manifest.Source.SHA256}, + Budget: checker, + }) + if err != nil { + t.Fatalf("OpenSession: %v", err) + } + target := filepath.Join(root, "new", "current.jsonl") + if _, err := session.MaterializeCurrent(context.Background(), target, false); !errors.Is(err, storage.ErrBudgetExceeded) { + t.Fatalf("MaterializeCurrent error = %v, want storage budget rejection", err) + } + if checker.Calls != 1 || checker.Projection.Operation != "materialize-current" { + t.Fatalf("unexpected materialize budget projection: %#v", checker) + } + if _, err := os.Stat(filepath.Dir(target)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("materialize target directory exists after preflight rejection: %v", err) + } +} + +func TestSessionTruncateTransitionsToBacking(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + newSize := int64(len(source) - 4) + if err := writer.Truncate(context.Background(), newSize); err != nil { + t.Fatalf("Truncate: %v", err) + } + _ = writer.Close() + readerHandle, err := session.OpenReader() + if err != nil { + t.Fatalf("OpenReader: %v", err) + } + defer readerHandle.Close() + if got := readHandle(t, readerHandle); !bytes.Equal(got, source[:newSize]) { + t.Fatalf("truncated bytes differ: got=%q want=%q", got, source[:newSize]) + } +} + +func TestSessionEqualLengthTruncateDoesNotCreateBacking(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + session := openFixtureSession(t, root, manifest, reader, nil) + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if err := writer.Truncate(context.Background(), int64(len(source))); err != nil { + t.Fatalf("equal-length Truncate: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("Close writer: %v", err) + } + if state := session.State(); state.BackingPath != "" { + t.Fatalf("equal-length truncate created backing: %#v", state) + } + if info, err := os.Stat(session.State().DeltaPath); err != nil || info.Size() != 0 { + t.Fatalf("equal-length truncate changed delta: info=%#v err=%v", info, err) + } +} + +func TestSessionInterruptedCopyOnWriteKeepsPreviousGeneration(t *testing.T) { + root := t.TempDir() + manifest, reader, source := sessionFixture(t, root) + stop := errors.New("stop before COW publish") + session := openFixtureSession(t, root, manifest, reader, func(phase string) error { + if phase == "before-publish" { + return stop + } + return nil + }) + writer, err := session.OpenWriter() + if err != nil { + t.Fatalf("OpenWriter: %v", err) + } + if _, err := writer.WriteAt(context.Background(), []byte("X"), 0); !errors.Is(err, stop) { + t.Fatalf("WriteAt error = %v, want %v", err, stop) + } + _ = writer.Close() + if session.State().BackingPath != "" { + t.Fatalf("interrupted COW published backing: %#v", session.State()) + } + handle, err := session.OpenReader() + if err != nil { + t.Fatalf("OpenReader: %v", err) + } + defer handle.Close() + if got := readHandle(t, handle); !bytes.Equal(got, source) { + t.Fatalf("previous generation changed after interrupted COW: %q", got) + } +} + +func sessionFixture(t *testing.T, root string) (fold.Manifest, memoryReader, []byte) { + t.Helper() + parts := [][]byte{[]byte("first-line\n"), bytes.Repeat([]byte("middle"), 11), []byte("\nlast-line\n")} + reader := memoryReader{} + manifest := fold.Manifest{Version: fold.ManifestVersion, Kind: fold.ManifestKind, Session: fold.ManifestSession{ID: "session", RolloutPath: filepath.Join(root, "native.jsonl")}} + var source []byte + for _, partBytes := range parts { + digest := digestBytes(partBytes) + reader[digest] = partBytes + manifest.Parts = append(manifest.Parts, fold.Part{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: digest, RawBytes: int64(len(partBytes))}}) + source = append(source, partBytes...) + } + manifest.Source = fold.ManifestSource{Bytes: int64(len(source)), SHA256: digestBytes(source)} + if err := os.WriteFile(manifest.Session.RolloutPath, source, 0o600); err != nil { + t.Fatalf("write native snapshot: %v", err) + } + return manifest, reader, source +} + +func openFixtureSession(t *testing.T, root string, manifest fold.Manifest, reader memoryReader, hook func(string) error) *Session { + t.Helper() + session, err := OpenSession(context.Background(), SessionOptions{ + Root: root, ManifestPath: filepath.Join(root, "manifest.json"), Manifest: manifest, Reader: reader, + NativeSnapshot: NativeFile{Path: manifest.Session.RolloutPath, Bytes: manifest.Source.Bytes, SHA256: manifest.Source.SHA256}, + BeforeCOWPhase: hook, + }) + if err != nil { + t.Fatalf("OpenSession returned error: %v", err) + } + return session +} + +func readHandle(t *testing.T, handle *ReadHandle) []byte { + t.Helper() + buffer := make([]byte, handle.Size()) + n, err := handle.ReadAt(context.Background(), buffer, 0) + if err != nil && !errors.Is(err, io.EOF) { + t.Fatalf("ReadAt returned error: %v", err) + } + return buffer[:n] +} + +func digestBytes(data []byte) string { + digest := sha256.Sum256(data) + return hex.EncodeToString(digest[:]) +} + +type vfsRejectingChecker struct { + Calls int + Projection storage.Projection +} + +func (c *vfsRejectingChecker) Check(_ context.Context, projection storage.Projection) (storage.Assessment, error) { + c.Calls++ + c.Projection = projection + return storage.Assessment{}, storage.ErrBudgetExceeded +} diff --git a/internal/vfs/state.go b/internal/vfs/state.go new file mode 100644 index 0000000..4ade7d9 --- /dev/null +++ b/internal/vfs/state.go @@ -0,0 +1,198 @@ +package vfs + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" +) + +const sessionStateVersion = 1 + +type NativeFile struct { + Path string `json:"path"` + Bytes int64 `json:"bytes"` + SHA256 string `json:"sha256"` +} + +type SessionState struct { + Version int `json:"version"` + SessionID string `json:"session_id"` + Generation uint64 `json:"generation"` + ManifestPath string `json:"manifest_path"` + BaseBytes int64 `json:"base_bytes"` + BaseSHA256 string `json:"base_sha256"` + DeltaPath string `json:"delta_path"` + BackingPath string `json:"backing_path,omitempty"` + NativeSnapshot NativeFile `json:"native_snapshot"` +} + +func loadSessionState(path string) (SessionState, error) { + data, err := os.ReadFile(path) + if err != nil { + return SessionState{}, err + } + var state SessionState + if err := json.Unmarshal(data, &state); err != nil { + return SessionState{}, fmt.Errorf("decode session state: %w", err) + } + if state.Version != sessionStateVersion || !safeSessionID(state.SessionID) || state.Generation == 0 || state.BaseBytes < 0 || len(state.BaseSHA256) != 64 || state.DeltaPath == "" { + return SessionState{}, errors.New("invalid virtual session state") + } + return state, nil +} + +func LoadSessionState(path string) (SessionState, error) { + state, err := loadSessionState(path) + if err != nil { + return SessionState{}, err + } + directory := filepath.Dir(filepath.Clean(path)) + if filepath.Base(directory) != state.SessionID || filepath.Base(filepath.Dir(directory)) != "sessions" { + return SessionState{}, errors.New("session state path does not match its session ID") + } + if !pathWithin(directory, state.DeltaPath) || (state.BackingPath != "" && !pathWithin(directory, state.BackingPath)) { + return SessionState{}, errors.New("session state contains an unsafe data path") + } + return state, nil +} + +func RepublishSessionState(path string) (SessionState, error) { + state, err := LoadSessionState(path) + if err != nil { + return SessionState{}, err + } + if state.Generation == ^uint64(0) { + return SessionState{}, errors.New("session generation cannot advance") + } + state.Generation++ + if err := writeSessionState(path, state); err != nil { + return SessionState{}, err + } + return state, nil +} + +func DiscoverSessionStates(root string) ([]SessionState, error) { + directory := filepath.Join(root, "fs", "sessions") + entries, err := os.ReadDir(directory) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read managed session states: %w", err) + } + states := make([]SessionState, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + state, err := LoadSessionState(filepath.Join(directory, entry.Name(), "state.json")) + if err != nil { + return nil, fmt.Errorf("load managed session %s: %w", entry.Name(), err) + } + states = append(states, state) + } + sort.Slice(states, func(i, j int) bool { return states[i].SessionID < states[j].SessionID }) + return states, nil +} + +func writeSessionState(path string, state SessionState) error { + data, err := encodeSessionState(state) + if err != nil { + return err + } + directory := filepath.Dir(path) + temporary, err := os.CreateTemp(directory, ".state-*.tmp") + if err != nil { + return fmt.Errorf("create temporary session state: %w", err) + } + return commitSessionState(path, data, temporary) +} + +func writeSessionStateWithTemporary(path string, temporaryPath string, state SessionState) error { + directory := filepath.Clean(filepath.Dir(path)) + temporaryPath = filepath.Clean(temporaryPath) + if filepath.Dir(temporaryPath) != directory { + return errors.New("temporary session state must be in the state directory") + } + data, err := encodeSessionState(state) + if err != nil { + return err + } + temporary, err := os.OpenFile(temporaryPath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) + if err != nil { + return fmt.Errorf("create temporary session state: %w", err) + } + return commitSessionState(path, data, temporary) +} + +func encodeSessionState(state SessionState) ([]byte, error) { + data, err := json.MarshalIndent(state, "", " ") + if err != nil { + return nil, fmt.Errorf("encode session state: %w", err) + } + return append(data, '\n'), nil +} + +func commitSessionState(path string, data []byte, temporary *os.File) error { + directory := filepath.Dir(path) + temporaryPath := temporary.Name() + defer func() { _ = os.Remove(temporaryPath) }() + if err := temporary.Chmod(0o600); err != nil { + _ = temporary.Close() + return fmt.Errorf("chmod temporary session state: %w", err) + } + if _, err := temporary.Write(data); err != nil { + _ = temporary.Close() + return fmt.Errorf("write temporary session state: %w", err) + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return fmt.Errorf("sync temporary session state: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close temporary session state: %w", err) + } + if err := replaceStateFile(temporaryPath, path); err != nil { + return fmt.Errorf("commit session state: %w", err) + } + return syncStateDirectory(directory) +} + +func verifyNativeFile(file NativeFile) error { + if file.Path == "" || file.Bytes < 0 || len(file.SHA256) != 64 { + return errors.New("native snapshot metadata is incomplete") + } + opened, err := os.Open(file.Path) + if err != nil { + return fmt.Errorf("open native snapshot: %w", err) + } + hasher := sha256.New() + bytesRead, copyErr := io.Copy(hasher, opened) + closeErr := opened.Close() + if copyErr != nil { + return fmt.Errorf("hash native snapshot: %w", copyErr) + } + if closeErr != nil { + return fmt.Errorf("close native snapshot: %w", closeErr) + } + if bytesRead != file.Bytes || hex.EncodeToString(hasher.Sum(nil)) != file.SHA256 { + return errors.New("native snapshot bytes or SHA-256 differ from metadata") + } + return nil +} + +func safeSessionID(sessionID string) bool { + return sessionID != "" && sessionID != "." && sessionID != ".." && !strings.ContainsAny(sessionID, "/\\\x00") +} + +func pathWithin(directory string, path string) bool { + relative, err := filepath.Rel(directory, path) + return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} diff --git a/internal/vfs/state_discovery_test.go b/internal/vfs/state_discovery_test.go new file mode 100644 index 0000000..aa8c113 --- /dev/null +++ b/internal/vfs/state_discovery_test.go @@ -0,0 +1,92 @@ +package vfs + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDiscoverSessionStatesReturnsValidatedStatesInSessionOrder(t *testing.T) { + root := t.TempDir() + for _, sessionID := range []string{"beta", "alpha"} { + directory := filepath.Join(root, "fs", "sessions", sessionID) + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatalf("create state directory: %v", err) + } + state := SessionState{ + Version: sessionStateVersion, SessionID: sessionID, Generation: 1, + ManifestPath: filepath.Join(root, "manifests", sessionID+".json"), + BaseBytes: 1, BaseSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + DeltaPath: filepath.Join(directory, "delta.jsonl"), + NativeSnapshot: NativeFile{Path: filepath.Join(root, sessionID+".jsonl"), Bytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + } + if err := writeSessionState(filepath.Join(directory, "state.json"), state); err != nil { + t.Fatalf("write state: %v", err) + } + } + states, err := DiscoverSessionStates(root) + if err != nil { + t.Fatalf("DiscoverSessionStates: %v", err) + } + if len(states) != 2 || states[0].SessionID != "alpha" || states[1].SessionID != "beta" { + t.Fatalf("unexpected states: %#v", states) + } +} + +func TestLoadSessionStateRejectsStateOutsideManagedSessionDirectory(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "fs", "sessions", "session") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + state := SessionState{ + Version: sessionStateVersion, SessionID: "session", Generation: 1, + ManifestPath: filepath.Join(root, "manifest.json"), BaseBytes: 1, + BaseSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + DeltaPath: filepath.Join(root, "outside.jsonl"), + NativeSnapshot: NativeFile{Path: filepath.Join(root, "native.jsonl"), Bytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + } + if err := writeSessionState(filepath.Join(directory, "state.json"), state); err != nil { + t.Fatal(err) + } + if _, err := LoadSessionState(filepath.Join(directory, "state.json")); err == nil { + t.Fatal("LoadSessionState should reject data paths outside the managed session directory") + } +} + +func TestRepublishSessionStateAdvancesGeneration(t *testing.T) { + root := t.TempDir() + directory := filepath.Join(root, "fs", "sessions", "session") + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + statePath := filepath.Join(directory, "state.json") + state := SessionState{ + Version: sessionStateVersion, SessionID: "session", Generation: 7, + ManifestPath: filepath.Join(root, "manifest.json"), BaseBytes: 1, + BaseSHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + DeltaPath: filepath.Join(directory, "delta.jsonl"), + NativeSnapshot: NativeFile{Path: filepath.Join(root, "native.jsonl"), Bytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + } + if err := os.WriteFile(state.DeltaPath, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := writeSessionState(statePath, state); err != nil { + t.Fatal(err) + } + + republished, err := RepublishSessionState(statePath) + if err != nil { + t.Fatalf("RepublishSessionState: %v", err) + } + if republished.Generation != 8 { + t.Fatalf("republished generation = %d, want 8", republished.Generation) + } + loaded, err := LoadSessionState(statePath) + if err != nil { + t.Fatal(err) + } + if loaded.Generation != 8 { + t.Fatalf("persisted generation = %d, want 8", loaded.Generation) + } +} diff --git a/internal/vfs/state_replace_unix.go b/internal/vfs/state_replace_unix.go new file mode 100644 index 0000000..09521fe --- /dev/null +++ b/internal/vfs/state_replace_unix.go @@ -0,0 +1,22 @@ +//go:build !windows + +package vfs + +import ( + "fmt" + "os" +) + +func replaceStateFile(source string, target string) error { return os.Rename(source, target) } + +func syncStateDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + if err := directory.Sync(); err != nil { + _ = directory.Close() + return fmt.Errorf("sync directory %s: %w", path, err) + } + return directory.Close() +} diff --git a/internal/vfs/state_replace_windows.go b/internal/vfs/state_replace_windows.go new file mode 100644 index 0000000..dd9b25f --- /dev/null +++ b/internal/vfs/state_replace_windows.go @@ -0,0 +1,29 @@ +//go:build windows + +package vfs + +import ( + "fmt" + "syscall" + "unsafe" +) + +var moveFileExW = syscall.NewLazyDLL("kernel32.dll").NewProc("MoveFileExW") + +func replaceStateFile(source string, target string) error { + sourcePointer, err := syscall.UTF16PtrFromString(source) + if err != nil { + return err + } + targetPointer, err := syscall.UTF16PtrFromString(target) + if err != nil { + return err + } + result, _, callErr := moveFileExW.Call(uintptr(unsafe.Pointer(sourcePointer)), uintptr(unsafe.Pointer(targetPointer)), 0x1|0x8) + if result == 0 { + return fmt.Errorf("replace file: %w", callErr) + } + return nil +} + +func syncStateDirectory(string) error { return nil } diff --git a/internal/vfs/view.go b/internal/vfs/view.go new file mode 100644 index 0000000..fa4d7e0 --- /dev/null +++ b/internal/vfs/view.go @@ -0,0 +1,134 @@ +package vfs + +import ( + "context" + "errors" + "fmt" + "io" + "sort" + + "github.com/samekind/codexfold/internal/fold" +) + +type View struct { + manifest fold.Manifest + ends []int64 + reader ObjectReader +} + +func NewView(manifest fold.Manifest, reader ObjectReader) (*View, error) { + if reader == nil { + return nil, errors.New("virtual view object reader is required") + } + if manifest.Version != fold.ManifestVersion || manifest.Kind != fold.ManifestKind { + return nil, fmt.Errorf("unsupported fold manifest version=%d kind=%q", manifest.Version, manifest.Kind) + } + view := &View{manifest: manifest, reader: reader, ends: make([]int64, len(manifest.Parts))} + var total int64 + for index, part := range manifest.Parts { + if part.Kind != fold.PartResidual && part.Kind != fold.PartField { + return nil, fmt.Errorf("manifest part %d has unsupported kind %q", index, part.Kind) + } + if len(part.Object.SHA256) != 64 || part.Object.RawBytes <= 0 { + return nil, fmt.Errorf("manifest part %d has invalid object reference", index) + } + if part.Object.RawBytes > int64(^uint64(0)>>1)-total { + return nil, errors.New("manifest byte length overflows int64") + } + total += part.Object.RawBytes + view.ends[index] = total + } + if total != manifest.Source.Bytes { + return nil, fmt.Errorf("manifest parts total %d bytes, source records %d", total, manifest.Source.Bytes) + } + return view, nil +} + +func (v *View) Size() int64 { return v.manifest.Source.Bytes } + +func (v *View) ReadAt(ctx context.Context, destination []byte, offset int64) (int, error) { + if offset < 0 { + return 0, errors.New("negative virtual read offset") + } + if len(destination) == 0 { + return 0, nil + } + if err := ctx.Err(); err != nil { + return 0, err + } + if offset >= v.Size() { + return 0, io.EOF + } + partIndex := sort.Search(len(v.ends), func(index int) bool { return v.ends[index] > offset }) + if partIndex == len(v.ends) { + return 0, fmt.Errorf("manifest has no part for offset %d", offset) + } + written := 0 + for written < len(destination) && offset < v.Size() { + if err := ctx.Err(); err != nil { + return written, err + } + if partIndex == len(v.ends) { + return written, fmt.Errorf("manifest has no part for offset %d", offset) + } + partStart := int64(0) + if partIndex > 0 { + partStart = v.ends[partIndex-1] + } + part := v.manifest.Parts[partIndex] + inside := offset - partStart + remaining := part.Object.RawBytes - inside + need := len(destination) - written + if int64(need) > remaining { + need = int(remaining) + } + partDestinationStart := written + n, err := v.reader.ReadAt(ctx, part.Object, destination[written:written+need], inside) + if n < 0 || n > need { + return written, fmt.Errorf("object reader returned invalid byte count %d for request %d", n, need) + } + written += n + offset += int64(n) + if n != need { + if err == nil { + err = io.ErrUnexpectedEOF + } + return written, fmt.Errorf("read manifest part %d: %w", partIndex, err) + } + if err != nil && !errors.Is(err, io.EOF) { + return written, fmt.Errorf("read manifest part %d: %w", partIndex, err) + } + if inside == 0 && int64(need) == part.Object.RawBytes { + source := destination[partDestinationStart : partDestinationStart+need] + for written < len(destination) && partIndex+1 < len(v.manifest.Parts) { + if err := ctx.Err(); err != nil { + return written, err + } + next := v.manifest.Parts[partIndex+1] + if !sameObjectBytes(part.Object, next.Object) { + break + } + repeatedBytes := len(destination) - written + if int64(repeatedBytes) > next.Object.RawBytes { + repeatedBytes = int(next.Object.RawBytes) + } + copy(destination[written:written+repeatedBytes], source[:repeatedBytes]) + written += repeatedBytes + offset += int64(repeatedBytes) + partIndex++ + if int64(repeatedBytes) < next.Object.RawBytes { + break + } + } + } + partIndex++ + } + if written < len(destination) { + return written, io.EOF + } + return written, nil +} + +func sameObjectBytes(first fold.ObjectRef, second fold.ObjectRef) bool { + return first.SHA256 == second.SHA256 && first.RawBytes == second.RawBytes +} diff --git a/internal/vfs/view_test.go b/internal/vfs/view_test.go new file mode 100644 index 0000000..8ffb46a --- /dev/null +++ b/internal/vfs/view_test.go @@ -0,0 +1,181 @@ +package vfs + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "math/rand" + "testing" + + "github.com/samekind/codexfold/internal/fold" +) + +func TestViewReadsExactBytesAcrossPartBoundaries(t *testing.T) { + view, source := fixtureView(t) + buffer := make([]byte, 19) + n, err := view.ReadAt(context.Background(), buffer, 4) + if err != nil { + t.Fatalf("ReadAt returned error: %v", err) + } + if !bytes.Equal(buffer[:n], source[4:23]) { + t.Fatalf("cross-part bytes differ: got=%q want=%q", buffer[:n], source[4:23]) + } + if view.Size() != int64(len(source)) { + t.Fatalf("Size = %d, want %d", view.Size(), len(source)) + } +} + +func TestViewMatchesNativeBytesForTenThousandRandomReads(t *testing.T) { + view, source := fixtureView(t) + random := rand.New(rand.NewSource(42)) + for iteration := 0; iteration < 10000; iteration++ { + offset := random.Intn(len(source) + 5) + length := random.Intn(80) + buffer := make([]byte, length) + n, err := view.ReadAt(context.Background(), buffer, int64(offset)) + if length == 0 { + if n != 0 || err != nil { + t.Fatalf("iteration %d zero read = (%d, %v)", iteration, n, err) + } + continue + } + if offset >= len(source) { + if n != 0 || !errors.Is(err, io.EOF) { + t.Fatalf("iteration %d past EOF = (%d, %v)", iteration, n, err) + } + continue + } + end := offset + length + if end > len(source) { + end = len(source) + } + if !bytes.Equal(buffer[:n], source[offset:end]) { + t.Fatalf("iteration %d bytes differ offset=%d length=%d", iteration, offset, length) + } + if end < offset+length { + if !errors.Is(err, io.EOF) { + t.Fatalf("iteration %d error = %v, want EOF", iteration, err) + } + } else if err != nil { + t.Fatalf("iteration %d unexpected error: %v", iteration, err) + } + } +} + +func TestViewRejectsInconsistentManifestLength(t *testing.T) { + manifest := fold.Manifest{ + Version: fold.ManifestVersion, + Kind: fold.ManifestKind, + Source: fold.ManifestSource{Bytes: 5, SHA256: string(make([]byte, 64))}, + Parts: []fold.Part{{Kind: fold.PartResidual, Object: fold.ObjectRef{ + SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + RawBytes: 4, + }}}, + } + if _, err := NewView(manifest, memoryReader{}); err == nil { + t.Fatal("NewView should reject inconsistent manifest bytes") + } +} + +func TestViewPropagatesCancellation(t *testing.T) { + view, _ := fixtureView(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := view.ReadAt(ctx, make([]byte, 1), 0); !errors.Is(err, context.Canceled) { + t.Fatalf("ReadAt error = %v, want context.Canceled", err) + } +} + +func TestViewReadsAdjacentRepeatedObjectsOncePerRequest(t *testing.T) { + object := bytes.Repeat([]byte("repeated-object-"), 4096) + digest := sha256.Sum256(object) + ref := fold.ObjectRef{ + SHA256: hex.EncodeToString(digest[:]), + RawBytes: int64(len(object)), + } + const repeats = 512 + manifest := fold.Manifest{ + Version: fold.ManifestVersion, + Kind: fold.ManifestKind, + Source: fold.ManifestSource{Bytes: int64(len(object) * repeats)}, + Parts: make([]fold.Part, repeats), + } + for index := range manifest.Parts { + manifest.Parts[index] = fold.Part{Kind: fold.PartResidual, Object: ref} + } + reader := &countingMemoryReader{objects: memoryReader{ref.SHA256: object}} + view, err := NewView(manifest, reader) + if err != nil { + t.Fatal(err) + } + got := make([]byte, len(object)*repeats) + if n, err := view.ReadAt(context.Background(), got, 0); err != nil || n != len(got) { + t.Fatalf("ReadAt repeated objects = %d, %v", n, err) + } + if reader.calls != 1 { + t.Fatalf("object reader calls = %d, want 1", reader.calls) + } + for index := 0; index < repeats; index++ { + start := index * len(object) + if !bytes.Equal(got[start:start+len(object)], object) { + t.Fatalf("repeated object %d changed", index) + } + } +} + +type memoryReader map[string][]byte + +type countingMemoryReader struct { + objects memoryReader + calls int +} + +func (r *countingMemoryReader) ReadAt(ctx context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { + r.calls++ + return r.objects.ReadAt(ctx, ref, destination, offset) +} + +func (r memoryReader) ReadAt(_ context.Context, ref fold.ObjectRef, destination []byte, offset int64) (int, error) { + data, ok := r[ref.SHA256] + if !ok { + return 0, errors.New("missing object") + } + if offset >= int64(len(data)) { + return 0, io.EOF + } + n := copy(destination, data[offset:]) + if n < len(destination) { + return n, io.EOF + } + return n, nil +} + +func fixtureView(t *testing.T) (*View, []byte) { + t.Helper() + parts := [][]byte{ + []byte("alpha-"), + bytes.Repeat([]byte("B"), 33), + []byte("-gamma-"), + bytes.Repeat([]byte("delta"), 17), + } + reader := memoryReader{} + manifest := fold.Manifest{Version: fold.ManifestVersion, Kind: fold.ManifestKind} + var source []byte + for _, data := range parts { + digest := sha256.Sum256(data) + hexDigest := hex.EncodeToString(digest[:]) + reader[hexDigest] = data + manifest.Parts = append(manifest.Parts, fold.Part{Kind: fold.PartResidual, Object: fold.ObjectRef{SHA256: hexDigest, RawBytes: int64(len(data))}}) + source = append(source, data...) + } + sourceDigest := sha256.Sum256(source) + manifest.Source = fold.ManifestSource{Bytes: int64(len(source)), SHA256: hex.EncodeToString(sourceDigest[:])} + view, err := NewView(manifest, reader) + if err != nil { + t.Fatalf("NewView returned error: %v", err) + } + return view, source +} diff --git a/internal/vfs/writer_lock_unix.go b/internal/vfs/writer_lock_unix.go new file mode 100644 index 0000000..a65c726 --- /dev/null +++ b/internal/vfs/writer_lock_unix.go @@ -0,0 +1,40 @@ +//go:build !windows + +package vfs + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func tryLockWriterFile(file *os.File) (bool, error) { + err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if errors.Is(err, unix.EWOULDBLOCK) { + return false, nil + } + return err == nil, err +} + +func unlockWriterFile(file *os.File) error { return unix.Flock(int(file.Fd()), unix.LOCK_UN) } + +func cleanupStaleWriterLease(path string) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return err + } + locked, err := tryLockWriterFile(file) + if err != nil { + _ = file.Close() + return err + } + if !locked { + return file.Close() + } + if err := unlockWriterFile(file); err != nil { + _ = file.Close() + return err + } + return file.Close() +} diff --git a/internal/vfs/writer_lock_windows.go b/internal/vfs/writer_lock_windows.go new file mode 100644 index 0000000..ce721e7 --- /dev/null +++ b/internal/vfs/writer_lock_windows.go @@ -0,0 +1,43 @@ +//go:build windows + +package vfs + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +func tryLockWriterFile(file *os.File) (bool, error) { + overlapped := new(windows.Overlapped) + err := windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, overlapped) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return false, nil + } + return err == nil, err +} + +func unlockWriterFile(file *os.File) error { + return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, new(windows.Overlapped)) +} + +func cleanupStaleWriterLease(path string) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return err + } + locked, err := tryLockWriterFile(file) + if err != nil { + _ = file.Close() + return err + } + if !locked { + return file.Close() + } + if err := unlockWriterFile(file); err != nil { + _ = file.Close() + return err + } + return file.Close() +} diff --git a/platform/darwin/fskit/CodexFoldFSKit.entitlements b/platform/darwin/fskit/CodexFoldFSKit.entitlements new file mode 100644 index 0000000..24446b6 --- /dev/null +++ b/platform/darwin/fskit/CodexFoldFSKit.entitlements @@ -0,0 +1,10 @@ + + + + + com.apple.security.application-groups + + group.vip.jstar.codexfold + + + diff --git a/platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.pbxproj b/platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.pbxproj new file mode 100644 index 0000000..d2161c3 --- /dev/null +++ b/platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.pbxproj @@ -0,0 +1,427 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 77; + objects = { + +/* Begin PBXBuildFile section */ + 54CC8E546672A2B2EB8778AE /* ProfileModule.swift in Sources */ = {isa = PBXBuildFile; fileRef = A321270CF90754EFB5FB4F82 /* ProfileModule.swift */; }; + 5A3E15ED0688984A0382281F /* Host.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0299A73CFDD46513958B3173 /* Host.swift */; }; + AC5E7B1EB1279405828432A1 /* Wire.swift in Sources */ = {isa = PBXBuildFile; fileRef = 442319991E8D02494E8E8966 /* Wire.swift */; }; + D146B22FE62339A25BA1F531 /* CodexFoldFSKitModule.appex in Embed ExtensionKit Extensions */ = {isa = PBXBuildFile; fileRef = 0EB68F3D6D26A8642A137511 /* CodexFoldFSKitModule.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; + F5B87A303BF4BD2D5651CFF8 /* ReadCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = E70FB31455A1202A1C719C64 /* ReadCache.swift */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + AE33723DBF591F025AAAE018 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 708C773DF138FE6212E0FC9A /* Project object */; + proxyType = 1; + remoteGlobalIDString = 00016AAD22571AA1F375B40A; + remoteInfo = CodexFoldFSKitModule; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXCopyFilesBuildPhase section */ + 7F086318D92AAA862935176E /* Embed ExtensionKit Extensions */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = "$(EXTENSIONS_FOLDER_PATH)"; + dstSubfolderSpec = 16; + files = ( + D146B22FE62339A25BA1F531 /* CodexFoldFSKitModule.appex in Embed ExtensionKit Extensions */, + ); + name = "Embed ExtensionKit Extensions"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + 0299A73CFDD46513958B3173 /* Host.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Host.swift; sourceTree = ""; }; + 0EB68F3D6D26A8642A137511 /* CodexFoldFSKitModule.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.extensionkit-extension"; includeInIndex = 0; path = CodexFoldFSKitModule.appex; sourceTree = BUILT_PRODUCTS_DIR; }; + 12B2D089AE4A8AE24344C2F8 /* CodexFoldFSKit.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = CodexFoldFSKit.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 442319991E8D02494E8E8966 /* Wire.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Wire.swift; sourceTree = ""; }; + 45646F5D9CFE136ED2DC3D6E /* CodexFoldFSKitModule.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = CodexFoldFSKitModule.entitlements; sourceTree = ""; }; + 683C4E970391FE59129607C0 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = ""; }; + 8B37F589115C048AA0DA2C76 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = ""; }; + A321270CF90754EFB5FB4F82 /* ProfileModule.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileModule.swift; sourceTree = ""; }; + E70FB31455A1202A1C719C64 /* ReadCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ReadCache.swift; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXGroup section */ + 00AEBFA549CEA3A2EF2670ED /* Products */ = { + isa = PBXGroup; + children = ( + 12B2D089AE4A8AE24344C2F8 /* CodexFoldFSKit.app */, + 0EB68F3D6D26A8642A137511 /* CodexFoldFSKitModule.appex */, + ); + name = Products; + sourceTree = ""; + }; + 4DED45333D2D92F454B627C0 = { + isa = PBXGroup; + children = ( + B470688CF65E5B72C59AEBB3 /* Extension */, + A88D2555D5D45400A73FD862 /* Host */, + 00AEBFA549CEA3A2EF2670ED /* Products */, + ); + sourceTree = ""; + }; + A88D2555D5D45400A73FD862 /* Host */ = { + isa = PBXGroup; + children = ( + 0299A73CFDD46513958B3173 /* Host.swift */, + 8B37F589115C048AA0DA2C76 /* Info.plist */, + ); + path = Host; + sourceTree = ""; + }; + B470688CF65E5B72C59AEBB3 /* Extension */ = { + isa = PBXGroup; + children = ( + 45646F5D9CFE136ED2DC3D6E /* CodexFoldFSKitModule.entitlements */, + 683C4E970391FE59129607C0 /* Info.plist */, + A321270CF90754EFB5FB4F82 /* ProfileModule.swift */, + E70FB31455A1202A1C719C64 /* ReadCache.swift */, + 442319991E8D02494E8E8966 /* Wire.swift */, + ); + path = Extension; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 00016AAD22571AA1F375B40A /* CodexFoldFSKitModule */ = { + isa = PBXNativeTarget; + buildConfigurationList = 2AA9F9298B9AF2C59CCB9ED4 /* Build configuration list for PBXNativeTarget "CodexFoldFSKitModule" */; + buildPhases = ( + C77A846ECB9F07A35A0A7952 /* Sources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = CodexFoldFSKitModule; + packageProductDependencies = ( + ); + productName = CodexFoldFSKitModule; + productReference = 0EB68F3D6D26A8642A137511 /* CodexFoldFSKitModule.appex */; + productType = "com.apple.product-type.extensionkit-extension"; + }; + 07E85731B7BEF398A8C4E4EC /* CodexFoldFSKit */ = { + isa = PBXNativeTarget; + buildConfigurationList = 830559E5E3522260B246A010 /* Build configuration list for PBXNativeTarget "CodexFoldFSKit" */; + buildPhases = ( + 0E5DA05244BDCF249A088B8C /* Sources */, + 7F086318D92AAA862935176E /* Embed ExtensionKit Extensions */, + ); + buildRules = ( + ); + dependencies = ( + 74BE85F55E9506D33987AFBF /* PBXTargetDependency */, + ); + name = CodexFoldFSKit; + packageProductDependencies = ( + ); + productName = CodexFoldFSKit; + productReference = 12B2D089AE4A8AE24344C2F8 /* CodexFoldFSKit.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 708C773DF138FE6212E0FC9A /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = YES; + LastUpgradeCheck = 1430; + TargetAttributes = { + 00016AAD22571AA1F375B40A = { + DevelopmentTeam = Y987FUR837; + ProvisioningStyle = Automatic; + }; + 07E85731B7BEF398A8C4E4EC = { + DevelopmentTeam = Y987FUR837; + ProvisioningStyle = Automatic; + }; + }; + }; + buildConfigurationList = 16739CA6CFCBE5CD8461F74B /* Build configuration list for PBXProject "CodexFoldFSKit" */; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + Base, + en, + ); + mainGroup = 4DED45333D2D92F454B627C0; + minimizedProjectReferenceProxies = 1; + preferredProjectObjectVersion = 77; + productRefGroup = 00AEBFA549CEA3A2EF2670ED /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 07E85731B7BEF398A8C4E4EC /* CodexFoldFSKit */, + 00016AAD22571AA1F375B40A /* CodexFoldFSKitModule */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXSourcesBuildPhase section */ + 0E5DA05244BDCF249A088B8C /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 5A3E15ED0688984A0382281F /* Host.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + C77A846ECB9F07A35A0A7952 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 54CC8E546672A2B2EB8778AE /* ProfileModule.swift in Sources */, + F5B87A303BF4BD2D5651CFF8 /* ReadCache.swift in Sources */, + AC5E7B1EB1279405828432A1 /* Wire.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 74BE85F55E9506D33987AFBF /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 00016AAD22571AA1F375B40A /* CodexFoldFSKitModule */; + targetProxy = AE33723DBF591F025AAAE018 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin XCBuildConfiguration section */ + 07A6AEE6F5B5E6DA978DAA00 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_ENTITLEMENTS = CodexFoldFSKit.entitlements; + COMBINE_HIDPI_IMAGES = YES; + INFOPLIST_FILE = Host/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = vip.jstar.codexfold.fskitprofileprobe; + REGISTER_APP_GROUPS = YES; + SDKROOT = macosx; + }; + name = Release; + }; + 109F1AB456D7EEE08D96BA16 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CODE_SIGN_ENTITLEMENTS = Extension/CodexFoldFSKitModule.entitlements; + COMBINE_HIDPI_IMAGES = YES; + ENABLE_APP_SANDBOX = YES; + INFOPLIST_FILE = Extension/Info.plist; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = vip.jstar.codexfold.fskitprofileprobe.module; + REGISTER_APP_GROUPS = YES; + SDKROOT = macosx; + SKIP_INSTALL = YES; + }; + name = Debug; + }; + 2ABE4C480A0163130E06B856 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGN_STYLE = Automatic; + COPY_PHASE_STRIP = NO; + CURRENT_PROJECT_VERSION = 103; + DEBUG_INFORMATION_FORMAT = dwarf; + DEVELOPMENT_TEAM = Y987FUR837; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "$(inherited)", + "DEBUG=1", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 27.0; + MARKETING_VERSION = 0.3.0; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 5.0; + }; + name = Debug; + }; + 39F4FAD3CDE67351EA64468B /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_ENTITLEMENTS = CodexFoldFSKit.entitlements; + COMBINE_HIDPI_IMAGES = YES; + INFOPLIST_FILE = Host/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = vip.jstar.codexfold.fskitprofileprobe; + REGISTER_APP_GROUPS = YES; + SDKROOT = macosx; + }; + name = Debug; + }; + B61FA346F14A61E65CC7F596 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGN_STYLE = Automatic; + COPY_PHASE_STRIP = NO; + CURRENT_PROJECT_VERSION = 103; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + DEVELOPMENT_TEAM = Y987FUR837; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 27.0; + MARKETING_VERSION = 0.3.0; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + SWIFT_VERSION = 5.0; + }; + name = Release; + }; + ED8C7963C890ACC5C7FF9A70 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CODE_SIGN_ENTITLEMENTS = Extension/CodexFoldFSKitModule.entitlements; + COMBINE_HIDPI_IMAGES = YES; + ENABLE_APP_SANDBOX = YES; + INFOPLIST_FILE = Extension/Info.plist; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = vip.jstar.codexfold.fskitprofileprobe.module; + REGISTER_APP_GROUPS = YES; + SDKROOT = macosx; + SKIP_INSTALL = YES; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 16739CA6CFCBE5CD8461F74B /* Build configuration list for PBXProject "CodexFoldFSKit" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 2ABE4C480A0163130E06B856 /* Debug */, + B61FA346F14A61E65CC7F596 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Debug; + }; + 2AA9F9298B9AF2C59CCB9ED4 /* Build configuration list for PBXNativeTarget "CodexFoldFSKitModule" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 109F1AB456D7EEE08D96BA16 /* Debug */, + ED8C7963C890ACC5C7FF9A70 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Debug; + }; + 830559E5E3522260B246A010 /* Build configuration list for PBXNativeTarget "CodexFoldFSKit" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 39F4FAD3CDE67351EA64468B /* Debug */, + 07A6AEE6F5B5E6DA978DAA00 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Debug; + }; +/* End XCConfigurationList section */ + }; + rootObject = 708C773DF138FE6212E0FC9A /* Project object */; +} diff --git a/platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..919434a --- /dev/null +++ b/platform/darwin/fskit/CodexFoldFSKit.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/platform/darwin/fskit/Extension/CodexFoldFSKitModule.entitlements b/platform/darwin/fskit/Extension/CodexFoldFSKitModule.entitlements new file mode 100644 index 0000000..f7535bd --- /dev/null +++ b/platform/darwin/fskit/Extension/CodexFoldFSKitModule.entitlements @@ -0,0 +1,16 @@ + + + + + com.apple.developer.fskit.fsmodule + + com.apple.security.app-sandbox + + com.apple.security.application-groups + + group.vip.jstar.codexfold + + com.apple.security.network.client + + + diff --git a/platform/darwin/fskit/Extension/Info.plist b/platform/darwin/fskit/Extension/Info.plist new file mode 100644 index 0000000..3d6b31b --- /dev/null +++ b/platform/darwin/fskit/Extension/Info.plist @@ -0,0 +1,54 @@ + + + + + CFBundleDisplayName + CodexFold Native FSKit Module + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + XPC! + CFBundleShortVersionString + $(MARKETING_VERSION) + CFBundleVersion + $(CURRENT_PROJECT_VERSION) + EXAppExtensionAttributes + + EXExtensionPointIdentifier + com.apple.fskit.fsmodule + FSActivateOptionSyntax + + shortOptions + o: + + FSMediaTypes + + FSPersonalities + + FSRequiresSecurityScopedPathURLResources + + FSShortName + codexfoldnative + FSSupportedSchemes + + codexfoldnative + + FSSupportsBlockResources + + FSSupportsGenericURLResources + + FSSupportsPathURLs + + FSSupportsServerURLs + + + LSMinimumSystemVersion + $(MACOSX_DEPLOYMENT_TARGET) + + diff --git a/platform/darwin/fskit/Extension/ProfileModule.swift b/platform/darwin/fskit/Extension/ProfileModule.swift new file mode 100644 index 0000000..ea45cf2 --- /dev/null +++ b/platform/darwin/fskit/Extension/ProfileModule.swift @@ -0,0 +1,1771 @@ +import Darwin +import Dispatch +import ExtensionFoundation +import Foundation +import FSKit +import OSLog + +@main +struct CodexFoldFSKitModule: UnaryFileSystemExtension { + var fileSystem: FSUnaryFileSystem & FSUnaryFileSystemOperations { + CodexFoldFileSystem() + } +} + +final class CodexFoldFileSystem: FSUnaryFileSystem, FSUnaryFileSystemOperations { + private let logger = Logger( + subsystem: "vip.jstar.codexfold.fskitprofileprobe.module", + category: "resource" + ) + private let volumeID = FSVolume.Identifier(uuid: UUID(uuidString: "5D0CF927-75A7-48B0-BDAE-621D8F2E695B")!) + private let lock = NSLock() + private var activeResourceURL: URL? + private weak var activeVolume: CodexFoldVolume? + + func probeResource( + resource: FSResource, + replyHandler: @escaping (FSProbeResult?, (any Error)?) -> Void + ) { + let containerID = FSContainerIdentifier(uuid: volumeID.uuid) + replyHandler(.usable(name: "CodexFold", containerID: containerID), nil) + } + + func loadResource( + resource: FSResource, + options: FSTaskOptions, + replyHandler: @escaping (FSVolume?, (any Error)?) -> Void + ) { + guard let pathResource = resource as? FSPathURLResource else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + let url = pathResource.url + let scoped = url.startAccessingSecurityScopedResource() + logger.notice("loadResource started scoped=\(scoped, privacy: .public)") + do { + var isDirectory = ObjCBool(false) + guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory) else { + logger.error("resource path is missing") + throw POSIXError(.ENOENT) + } + logger.notice("resource inspected directory=\(isDirectory.boolValue, privacy: .public)") + let descriptorURL = isDirectory.boolValue + ? url.appendingPathComponent("descriptor.bin", isDirectory: false) + : url + let descriptorData = try Data(contentsOf: descriptorURL) + logger.notice("descriptor read bytes=\(descriptorData.count, privacy: .public)") + let descriptor = try WireDescriptor(data: descriptorData) + logger.notice("descriptor decoded generation=\(descriptor.generation, privacy: .public)") + logger.notice("connecting to daemon socket") + let client = try DaemonClient(descriptor: descriptor) + logger.notice("daemon socket connected; sending ping") + try client.ping() + logger.notice("daemon ping succeeded") + let volume = try CodexFoldVolume(volumeID: volumeID, client: client) + lock.lock() + activeResourceURL = scoped ? url : nil + activeVolume = volume + lock.unlock() + containerStatus = .ready + replyHandler(volume, nil) + } catch { + logger.error("loadResource failed: \(String(describing: error), privacy: .public)") + if scoped { + url.stopAccessingSecurityScopedResource() + } + replyHandler(nil, error) + } + } + + func unloadResource(resource: FSResource, options: FSTaskOptions) async throws { + let (volume, url) = lock.withLock { () -> (CodexFoldVolume?, URL?) in + let volume = activeVolume + let url = activeResourceURL + activeVolume = nil + activeResourceURL = nil + return (volume, url) + } + try volume?.synchronizeNow() + url?.stopAccessingSecurityScopedResource() + containerStatus = .notReady(status: POSIXError(.EAGAIN)) + } +} + +private final class CodexFoldPrefetchChannel { + let connection: WireConnection + let handle: UInt64 + let generation: UInt64 + let sharedReadWindow: WireSharedReadWindow? + private(set) var nativeReadFD: Int32? + private var closed = false + + init(connection: WireConnection, opened: WireOpenResult, generation: UInt64) { + self.connection = connection + self.handle = opened.handle + self.generation = generation + self.nativeReadFD = opened.nativeReadFD + self.sharedReadWindow = opened.sharedReadWindow + } + + func close(client: DaemonClient) { + guard !closed else { return } + closed = true + if let nativeReadFD { + Darwin.close(nativeReadFD) + self.nativeReadFD = nil + } + try? client.handleOperation(.release, handle: handle, connection: connection) + connection.close() + } + + deinit { + if let nativeReadFD { + Darwin.close(nativeReadFD) + } + connection.close() + } +} + +private final class CodexFoldIO { + // Keep the cache bounded while amortizing the FSKit-to-daemon round trip. + // A 12 MiB block aligns with the mounted reader's 4 MiB requests and lets + // eight workers maintain a 96 MiB horizon when the kernel cache is off. + + let client: DaemonClient + let connection: WireConnection + let handle: UInt64 + let path: String + let writable: Bool + private let foregroundFetchLock = NSLock() + private let cacheLock = NSLock() + private let nativeFDLock = NSLock() + private let prefetchPoolLock = NSLock() + private let prefetchQueue: OperationQueue + private var cachedBlocks: [Int64: CodexFoldCachedReadBlock] = [:] + private var prefetchingBlocks: Set = [] + private var prefetchGroups: [Int64: DispatchGroup] = [:] + private var prefetchFrontier: Int64 = 0 + private var sequentialHintBlockOffset: Int64? + private var cachePruneOffset: Int64 = 0 + private var cacheGeneration: UInt64 = 1 + private var fileSize: Int64 + private var closed = false + private var nativeReadFD: Int32? + private let sharedReadWindow: WireSharedReadWindow? + private var idlePrefetchChannels: [CodexFoldPrefetchChannel] = [] + private var prefetchChannelCount = 0 + private var prefetchPoolGeneration: UInt64 = 1 + private var prefetchPoolClosed = false + private let readAheadBytes: Int + private let concurrentPrefetchCount: Int + private let scheduledPrefetchCount: Int + private let maxCachedBlocks: Int + + init( + client: DaemonClient, + connection: WireConnection, + handle: UInt64, + path: String, + writable: Bool, + size: UInt64, + nativeReadFD: Int32? = nil, + sharedReadWindow: WireSharedReadWindow? = nil + ) { + self.client = client + self.connection = connection + self.handle = handle + self.path = path + self.writable = writable + self.fileSize = Self.normalizedFileSize(size) + let readAheadPolicy = CodexFoldReadAheadPolicy( + negotiatedReadBytes: connection.maximumReadBytes + ) + self.readAheadBytes = readAheadPolicy.readAheadBytes + self.concurrentPrefetchCount = readAheadPolicy.concurrentPrefetchCount + self.scheduledPrefetchCount = readAheadPolicy.scheduledPrefetchCount + self.maxCachedBlocks = readAheadPolicy.maxCachedBlocks + let prefetchQueue = OperationQueue() + prefetchQueue.name = "vip.jstar.codexfold.fskit.readahead.\(handle)" + prefetchQueue.qualityOfService = .userInitiated + prefetchQueue.maxConcurrentOperationCount = concurrentPrefetchCount + self.prefetchQueue = prefetchQueue + if writable { + if let nativeReadFD { + Darwin.close(nativeReadFD) + } + self.nativeReadFD = nil + self.sharedReadWindow = nil + } else { + self.nativeReadFD = nativeReadFD + self.sharedReadWindow = sharedReadWindow + } + primePrefetchChannelsIfBeneficial() + primeReadAheadIfBeneficial() + } + + func read( + client: DaemonClient, + offset: Int64, + length: Int, + into buffer: FSMutableFileDataBuffer + ) throws -> Int { + guard offset >= 0, length >= 0, buffer.length >= length else { + throw POSIXError(.EINVAL) + } + guard length > 0 else { return 0 } + if !writable { + nativeFDLock.lock() + if let nativeReadFD { + defer { nativeFDLock.unlock() } + return try Self.readNativeDescriptor( + nativeReadFD, + offset: offset, + length: length, + into: buffer + ) + } + nativeFDLock.unlock() + } + if !writable, length < readAheadBytes { + let blockSize = Int64(readAheadBytes) + let fetchOffset = offset / blockSize * blockSize + if let copied = copyCachedRange(offset: offset, length: length, into: buffer) { + schedulePrefetch(after: fetchOffset, generation: currentCacheGeneration()) + return copied + } + + foregroundFetchLock.lock() + defer { foregroundFetchLock.unlock() } + if let copied = copyCachedRange(offset: offset, length: length, into: buffer) { + schedulePrefetch(after: fetchOffset, generation: currentCacheGeneration()) + return copied + } + waitForPrefetch(offset: fetchOffset) + if let copied = copyCachedRange(offset: offset, length: length, into: buffer) { + schedulePrefetch(after: fetchOffset, generation: currentCacheGeneration()) + return copied + } + guard !isClosed() else { throw POSIXError(.EBADF) } + + if readAheadLength(offset: fetchOffset) > 0 { + let generation = currentCacheGeneration() + schedulePrefetchBlock(offset: fetchOffset, generation: generation) + waitForPrefetch(offset: fetchOffset) + if let copied = copyCachedRange(offset: offset, length: length, into: buffer) { + schedulePrefetch(after: fetchOffset, generation: generation) + return copied + } + } + } + let data = try client.read( + handle: handle, + offset: offset, + length: length, + connection: connection, + sharedWindow: sharedReadWindow + ) + return buffer.withUnsafeMutableBytes { destination in + _ = data.copyBytes(to: destination.bindMemory(to: UInt8.self)) + return data.count + } + } + + func invalidateReadCache() { + cacheLock.lock() + resetReadCacheLocked() + cacheLock.unlock() + invalidatePrefetchChannels() + } + + var usesNativeReadFD: Bool { + nativeFDLock.lock() + defer { nativeFDLock.unlock() } + return nativeReadFD != nil + } + + func updateFileSize(_ size: UInt64) { + let updated = Self.normalizedFileSize(size) + var changed = false + cacheLock.lock() + if updated != fileSize { + fileSize = updated + resetReadCacheLocked() + changed = true + } + cacheLock.unlock() + if changed { + invalidatePrefetchChannels() + } + } + + func shutdown() { + cacheLock.lock() + closed = true + resetReadCacheLocked() + cacheLock.unlock() + prefetchQueue.cancelAllOperations() + prefetchQueue.waitUntilAllOperationsAreFinished() + + nativeFDLock.lock() + if let nativeReadFD { + Darwin.close(nativeReadFD) + self.nativeReadFD = nil + } + nativeFDLock.unlock() + + closePrefetchPool() + } + + private func readNativeDescriptorIfAvailable(offset: Int64, length: Int) throws -> Data? { + nativeFDLock.lock() + defer { nativeFDLock.unlock() } + guard let nativeReadFD else { return nil } + return try Self.readNativeDescriptor(nativeReadFD, offset: offset, length: length) + } + + private static func normalizedFileSize(_ size: UInt64) -> Int64 { + size > UInt64(Int64.max) ? Int64.max : Int64(size) + } + + private func resetReadCacheLocked() { + cacheGeneration &+= 1 + cachedBlocks.removeAll(keepingCapacity: false) + prefetchingBlocks.removeAll(keepingCapacity: false) + prefetchGroups.removeAll(keepingCapacity: false) + prefetchFrontier = 0 + sequentialHintBlockOffset = nil + cachePruneOffset = 0 + } + + private func checkoutPrefetchChannel() throws -> CodexFoldPrefetchChannel? { + prefetchPoolLock.lock() + if prefetchPoolClosed { + prefetchPoolLock.unlock() + return nil + } + if let channel = idlePrefetchChannels.popLast() { + prefetchPoolLock.unlock() + return channel + } + guard prefetchChannelCount < maxCachedBlocks else { + prefetchPoolLock.unlock() + return nil + } + let generation = prefetchPoolGeneration + prefetchChannelCount += 1 + prefetchPoolLock.unlock() + + let connection: WireConnection + do { + connection = try client.newConnection() + } catch { + releasePrefetchChannelReservation() + throw error + } + let channel: CodexFoldPrefetchChannel + do { + let opened = try client.open(path, flags: O_RDONLY, connection: connection) + channel = CodexFoldPrefetchChannel( + connection: connection, + opened: opened, + generation: generation + ) + } catch { + connection.close() + releasePrefetchChannelReservation() + throw error + } + + prefetchPoolLock.lock() + let accepted = !prefetchPoolClosed && generation == prefetchPoolGeneration + if !accepted { + prefetchChannelCount -= 1 + } + prefetchPoolLock.unlock() + if !accepted { + channel.close(client: client) + return nil + } + return channel + } + + private func primePrefetchChannelsIfBeneficial() { + guard shouldPrimeReadAhead else { + return + } + + var channels: [CodexFoldPrefetchChannel] = [] + channels.reserveCapacity(maxCachedBlocks) + for _ in 0..= Int64(readAheadBytes * maxCachedBlocks) + } + + private func returnPrefetchChannel(_ channel: CodexFoldPrefetchChannel, healthy: Bool) { + prefetchPoolLock.lock() + let retain = healthy && + !prefetchPoolClosed && + channel.generation == prefetchPoolGeneration + if retain { + idlePrefetchChannels.append(channel) + } else { + prefetchChannelCount -= 1 + } + prefetchPoolLock.unlock() + if !retain { + channel.close(client: client) + } + } + + private func releasePrefetchChannelReservation() { + prefetchPoolLock.lock() + prefetchChannelCount -= 1 + prefetchPoolLock.unlock() + } + + private func invalidatePrefetchChannels() { + prefetchPoolLock.lock() + prefetchPoolGeneration &+= 1 + let channels = idlePrefetchChannels + idlePrefetchChannels.removeAll(keepingCapacity: false) + prefetchChannelCount -= channels.count + prefetchPoolLock.unlock() + for channel in channels { + channel.close(client: client) + } + } + + private func closePrefetchPool() { + prefetchPoolLock.lock() + prefetchPoolClosed = true + prefetchPoolGeneration &+= 1 + let channels = idlePrefetchChannels + idlePrefetchChannels.removeAll(keepingCapacity: false) + prefetchChannelCount -= channels.count + prefetchPoolLock.unlock() + for channel in channels { + channel.close(client: client) + } + } + + private func readAheadLength(offset: Int64) -> Int { + cacheLock.lock() + defer { cacheLock.unlock() } + return readAheadLengthLocked(offset: offset) + } + + private func readAheadLengthLocked(offset: Int64) -> Int { + guard offset >= 0, offset < fileSize else { return 0 } + return Int(min(Int64(readAheadBytes), fileSize - offset)) + } + + private static func readNativeDescriptor(_ descriptor: Int32, offset: Int64, length: Int) throws -> Data { + guard descriptor >= 0, offset >= 0, length >= 0 else { + throw POSIXError(.EINVAL) + } + var result = Data(count: length) + var completed = 0 + while completed < length { + let amount = result.withUnsafeMutableBytes { bytes in + Darwin.pread( + descriptor, + bytes.baseAddress!.advanced(by: completed), + length - completed, + off_t(offset + Int64(completed)) + ) + } + if amount == 0 { + break + } + if amount < 0 { + if errno == EINTR { continue } + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + completed += amount + } + if completed < result.count { + result.removeSubrange(completed.. Int { + guard descriptor >= 0, offset >= 0, length >= 0, buffer.length >= length else { + throw POSIXError(.EINVAL) + } + return try buffer.withUnsafeMutableBytes { destination in + var completed = 0 + while completed < length { + let amount = Darwin.pread( + descriptor, + destination.baseAddress!.advanced(by: completed), + length - completed, + off_t(offset + Int64(completed)) + ) + if amount == 0 { + break + } + if amount < 0 { + if errno == EINTR { continue } + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + completed += amount + } + return completed + } + } + + private func copyCachedRange( + offset: Int64, + length: Int, + into buffer: FSMutableFileDataBuffer + ) -> Int? { + cacheLock.lock() + guard !closed, offset >= 0, length >= 0, offset < fileSize else { + cacheLock.unlock() + return nil + } + let requestedEnd = Int64(length) > Int64.max - offset ? Int64.max : offset + Int64(length) + let end = min(fileSize, requestedEnd) + guard end >= offset, end - offset <= Int64(buffer.length) else { + cacheLock.unlock() + return nil + } + let blockSize = Int64(readAheadBytes) + let firstBlockOffset = offset / blockSize * blockSize + if let cached = cachedBlocks[firstBlockOffset] { + let lower = offset - firstBlockOffset + let available = Int64(cached.count) - lower + if lower >= 0, available >= end - offset { + pruneCacheLocked(before: firstBlockOffset) + cacheLock.unlock() + return copyCachedSpan( + cached, + sourceOffset: Int(lower), + length: Int(end - offset), + into: buffer + ) + } + } + + var spans: [(block: CodexFoldCachedReadBlock, offset: Int, length: Int)] = [] + var current = offset + while current < end { + let blockOffset = current / blockSize * blockSize + guard let cached = cachedBlocks[blockOffset] else { + cacheLock.unlock() + return nil + } + let lower = current - blockOffset + guard lower >= 0, lower < Int64(cached.count) else { + cacheLock.unlock() + return nil + } + let available = min(Int64(cached.count) - lower, end - current) + guard available > 0 else { + cacheLock.unlock() + return nil + } + spans.append((cached, Int(lower), Int(available))) + current += available + } + pruneCacheLocked(before: firstBlockOffset) + cacheLock.unlock() + + let written = buffer.withUnsafeMutableBytes { destination in + var written = 0 + for span in spans { + guard span.block.copyBytes( + from: span.offset, + count: span.length, + to: destination.baseAddress!.advanced(by: written) + ) else { + return -1 + } + written += span.length + } + return written + } + return written >= 0 ? written : nil + } + + private func copyCachedSpan( + _ block: CodexFoldCachedReadBlock, + sourceOffset: Int, + length: Int, + into buffer: FSMutableFileDataBuffer + ) -> Int? { + let copied = buffer.withUnsafeMutableBytes { destination in + block.copyBytes( + from: sourceOffset, + count: length, + to: destination.baseAddress! + ) + } + return copied ? length : nil + } + + private func pruneCacheLocked(before offset: Int64) { + guard offset > cachePruneOffset else { return } + let staleOffsets = cachedBlocks.keys.filter { $0 < offset } + for staleOffset in staleOffsets { + cachedBlocks.removeValue(forKey: staleOffset) + } + cachePruneOffset = offset + } + + private func currentCacheGeneration() -> UInt64 { + cacheLock.lock() + defer { cacheLock.unlock() } + return cacheGeneration + } + + private func isClosed() -> Bool { + cacheLock.lock() + defer { cacheLock.unlock() } + return closed + } + + private func waitForPrefetch(offset: Int64) { + cacheLock.lock() + let group = prefetchGroups[offset] + cacheLock.unlock() + guard let group else { return } + // Failed and cancelled prefetches also leave the group, allowing the + // foreground request to retry through the ordinary path. + _ = group.wait(timeout: .now() + .seconds(5)) + } + + private func trimCacheLocked() { + while cachedBlocks.count > maxCachedBlocks { + guard let oldest = cachedBlocks.keys.min() else { return } + cachedBlocks.removeValue(forKey: oldest) + } + } + + private func schedulePrefetchBlock(offset: Int64, generation: UInt64) { + var request: (offset: Int64, group: DispatchGroup)? + cacheLock.lock() + if !closed, + generation == cacheGeneration, + readAheadLengthLocked(offset: offset) > 0, + cachedBlocks[offset] == nil, + !prefetchingBlocks.contains(offset), + prefetchingBlocks.count < scheduledPrefetchCount { + let group = DispatchGroup() + group.enter() + prefetchingBlocks.insert(offset) + prefetchGroups[offset] = group + request = (offset, group) + } + cacheLock.unlock() + guard let request else { return } + prefetchQueue.addOperation { [weak self] in + guard let self else { + request.group.leave() + return + } + self.prefetch(offset: request.offset, generation: generation, completion: request.group) + } + } + + private func schedulePrefetch(after blockOffset: Int64, generation: UInt64) { + guard !writable else { return } + let blockSize = Int64(readAheadBytes) + var requests: [(offset: Int64, group: DispatchGroup)] = [] + cacheLock.lock() + guard !closed, generation == cacheGeneration else { + cacheLock.unlock() + return + } + guard fileSize > 0 else { + cacheLock.unlock() + return + } + let lastBlockOffset = (fileSize - 1) / blockSize * blockSize + if blockOffset >= lastBlockOffset { + cacheLock.unlock() + return + } + if let previous = sequentialHintBlockOffset, blockOffset < previous { + // A backwards seek starts a new read-ahead horizon without invalidating + // already fetched blocks that may still satisfy another open reader. + prefetchFrontier = blockOffset + blockSize + } + sequentialHintBlockOffset = blockOffset + let span = Int64(scheduledPrefetchCount) * blockSize + let requestedHorizon = blockOffset > Int64.max - span ? Int64.max : blockOffset + span + let horizon = min(lastBlockOffset, requestedHorizon) + if prefetchFrontier <= blockOffset { + prefetchFrontier = blockOffset + blockSize + } + while prefetchingBlocks.count < scheduledPrefetchCount && prefetchFrontier <= horizon && prefetchFrontier < fileSize { + let offset = prefetchFrontier + prefetchFrontier = offset >= lastBlockOffset ? fileSize : offset + blockSize + if cachedBlocks[offset] != nil || prefetchingBlocks.contains(offset) { + continue + } + let group = DispatchGroup() + group.enter() + prefetchingBlocks.insert(offset) + prefetchGroups[offset] = group + requests.append((offset, group)) + } + cacheLock.unlock() + + for request in requests { + prefetchQueue.addOperation { [weak self] in + guard let self else { + request.group.leave() + return + } + self.prefetch(offset: request.offset, generation: generation, completion: request.group) + } + } + } + + private func prefetch(offset: Int64, generation: UInt64, completion: DispatchGroup) { + var nextHint: Int64? + defer { + cacheLock.lock() + if prefetchGroups[offset] === completion { + prefetchingBlocks.remove(offset) + prefetchGroups.removeValue(forKey: offset) + } + if !closed && generation == cacheGeneration { + nextHint = sequentialHintBlockOffset + } + cacheLock.unlock() + completion.leave() + if let nextHint { + schedulePrefetch(after: nextHint, generation: generation) + } + } + cacheLock.lock() + let allowed = !closed && generation == cacheGeneration + let fetchLength = allowed ? readAheadLengthLocked(offset: offset) : 0 + cacheLock.unlock() + guard allowed, fetchLength > 0 else { return } + + do { + if let data = try readNativeDescriptorIfAvailable(offset: offset, length: fetchLength) { + cacheLock.lock() + if !closed && generation == cacheGeneration { + cachedBlocks[offset] = CodexFoldCachedReadBlock(data: data) + trimCacheLocked() + } + cacheLock.unlock() + return + } + } catch { + // The ordinary daemon path below remains the compatibility fallback. + } + + var channel: CodexFoldPrefetchChannel? + var healthy = false + defer { + if let channel { + returnPrefetchChannel(channel, healthy: healthy) + } + } + do { + guard let checkedOut = try checkoutPrefetchChannel() else { return } + channel = checkedOut + let block: CodexFoldCachedReadBlock + if let nativeReadFD = checkedOut.nativeReadFD { + block = CodexFoldCachedReadBlock( + data: try Self.readNativeDescriptor(nativeReadFD, offset: offset, length: fetchLength) + ) + } else if let sharedWindow = checkedOut.sharedReadWindow { + let result = try client.readBorrowingSharedWindow( + handle: checkedOut.handle, + offset: offset, + length: fetchLength, + connection: checkedOut.connection, + sharedWindow: sharedWindow + ) + switch result { + case .copied(let data): + block = CodexFoldCachedReadBlock(data: data) + case .sharedWindow(let count): + block = try CodexFoldCachedReadBlock( + sharedWindow: sharedWindow, + count: count, + release: { [weak self, checkedOut] in + self?.returnPrefetchChannel(checkedOut, healthy: true) + } + ) + channel = nil + } + } else { + block = CodexFoldCachedReadBlock( + data: try client.read( + handle: checkedOut.handle, + offset: offset, + length: fetchLength, + connection: checkedOut.connection + ) + ) + } + healthy = true + cacheLock.lock() + if !closed && generation == cacheGeneration { + cachedBlocks[offset] = block + trimCacheLocked() + } + cacheLock.unlock() + } catch { + return + } + } +} + +private final class CodexFoldItem: FSItem { + private let lock = NSLock() + private var storedEntry: WireEntry + private var storedIO: CodexFoldIO? + private(set) var deleted = false + + init(entry: WireEntry) { + storedEntry = entry + super.init() + } + + var entry: WireEntry { + lock.lock() + defer { lock.unlock() } + return storedEntry + } + + func update(_ entry: WireEntry) { + lock.lock() + storedEntry = entry + deleted = false + let currentIO = storedIO + lock.unlock() + currentIO?.updateFileSize(entry.size) + } + + func markDeleted() { + lock.lock() + deleted = true + lock.unlock() + } + + func io() -> CodexFoldIO? { + lock.lock() + defer { lock.unlock() } + return storedIO + } + + func replaceIO(_ io: CodexFoldIO?) -> CodexFoldIO? { + lock.lock() + let previous = storedIO + storedIO = io + lock.unlock() + return previous + } + + func invalidateReadCache() { + lock.lock() + let current = storedIO + lock.unlock() + current?.invalidateReadCache() + } +} + +private final class CodexFoldVolume: FSVolume, FSVolume.Handler, FSVolume.ReadWriteHandler, FSVolume.DataCacheHandler, FSVolume.XattrHandler { + private let logger = Logger( + subsystem: "vip.jstar.codexfold.fskitprofileprobe.module", + category: "coherency" + ) + private let client: DaemonClient + private let itemLock = NSLock() + private var items: [UInt64: CodexFoldItem] = [:] + private var rootItem: CodexFoldItem + private var namespaceVersion: UInt64 + private var namespaceTimer: DispatchSourceTimer? + + init(volumeID: FSVolume.Identifier, client: DaemonClient) throws { + self.client = client + let rootEntry = try client.getattr("/") + rootItem = CodexFoldItem(entry: rootEntry) + namespaceVersion = rootEntry.namespaceID + items[rootEntry.nodeID] = rootItem + super.init(volumeID: volumeID, volumeName: FSFileName(string: "CodexFold")) + } + + var supportedVolumeCapabilities: FSVolume.SupportedCapabilities { + let capabilities = FSVolume.SupportedCapabilities() + capabilities.supportsPersistentObjectIDs = false + capabilities.supportsSymbolicLinks = false + capabilities.supportsHardLinks = false + capabilities.supportsJournal = false + capabilities.supportsActiveJournal = false + capabilities.supportsSparseFiles = false + // A statfs round trip crosses the extension/daemon boundary, so let + // FSKit cache it instead of treating it as a local constant-time call. + capabilities.supportsFastStatFS = false + capabilities.supports2TBFiles = true + capabilities.supports64BitObjectIDs = true + capabilities.supportsHiddenFiles = true + capabilities.caseFormat = .sensitive + return capabilities + } + + var volumeStatistics: FSStatFSResult { + let result = FSStatFSResult(fileSystemTypeName: "codexfold") + do { + let stat = try client.statfs() + result.blockSize = Int(stat.blockSize) + result.ioSize = Int(stat.ioSize) + result.totalBytes = stat.totalBytes + result.availableBytes = stat.availableBytes + result.freeBytes = stat.freeBytes + result.usedBytes = stat.usedBytes + result.totalFiles = stat.totalFiles + result.freeFiles = stat.freeFiles + } catch { + result.blockSize = 4096 + result.ioSize = 4 * 1024 * 1024 + result.totalBytes = 1 << 40 + result.availableBytes = 1 << 39 + result.freeBytes = 1 << 39 + result.usedBytes = 1 << 39 + result.totalFiles = 1 << 32 + result.freeFiles = 1 << 31 + } + return result + } + + var maximumLinkCount: Int { 1 } + var maximumNameLength: Int { 255 } + var maximumFileSize: UInt64 { UInt64.max >> 1 } + var maximumXattrSize: Int { 16 * 1024 * 1024 - 8192 } + var restrictsOwnershipChanges: Bool { false } + var truncatesLongNames: Bool { false } + var enableOpenUnlinkEmulation: Bool { true } + + func activate( + options: FSTaskOptions, + replyHandler: @escaping (FSActivateResult?, (any Error)?) -> Void + ) { + do { + let entry = try client.getattr("/") + rootItem.update(entry) + itemLock.lock() + items = [entry.nodeID: rootItem] + namespaceVersion = entry.namespaceID + itemLock.unlock() + startNamespaceMonitor() + replyHandler(FSActivateResult(rootItem: rootItem), nil) + } catch { + replyHandler(nil, error) + } + } + + func deactivate(options: FSDeactivateOptions, replyHandler: @escaping ((any Error)?) -> Void) { + stopNamespaceMonitor() + closeAllItems() + replyHandler(nil) + } + + func mount(options: FSTaskOptions, replyHandler: @escaping ((any Error)?) -> Void) { + do { + try client.ping() + replyHandler(nil) + } catch { + replyHandler(error) + } + } + + func unmount(replyHandler: @escaping () -> Void) { + stopNamespaceMonitor() + try? synchronizeNow() + closeAllItems() + replyHandler() + } + + func synchronize(flags: FSSyncFlags, replyHandler: @escaping ((any Error)?) -> Void) { + do { + try synchronizeNow() + replyHandler(nil) + } catch { + replyHandler(error) + } + } + + func synchronizeNow() throws { + try client.sync() + } + + func lookupItem( + named name: FSFileName, + in directory: FSItem, + context: FSContext, + replyHandler: @escaping (FSLookupItemResult?, (any Error)?) -> Void + ) { + guard let directory = directory as? CodexFoldItem, directory.entry.type == .directory else { + replyHandler(nil, POSIXError(.ENOTDIR)) + return + } + guard let nameString = name.string else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let entry = try client.getattr(join(directory.entry.path, nameString)) + let item = item(for: entry) + replyHandler( + FSLookupItemResult(foundItem: item, itemName: FSFileName(string: entry.name), itemAttributes: attributes(for: entry)), + nil + ) + } catch { + replyHandler(nil, error) + } + } + + func reclaimItem(_ item: FSItem, replyHandler: @escaping ((any Error)?) -> Void) { + guard let item = item as? CodexFoldItem else { + replyHandler(POSIXError(.EINVAL)) + return + } + itemLock.lock() + let reclaimed = item.tryReclaim { [self] in + closeIO(item.replaceIO(nil)) + if items[item.entry.nodeID] === item && item !== rootItem { + items.removeValue(forKey: item.entry.nodeID) + } + } + itemLock.unlock() + replyHandler(reclaimed ? nil : nil) + } + + func createItem( + named name: FSFileName, + type: FSItem.ItemType, + in directory: FSItem, + attributes newAttributes: FSItem.SetAttributesRequest, + context: FSContext, + replyHandler: @escaping (FSCreateItemResult?, (any Error)?) -> Void + ) { + guard let directory = directory as? CodexFoldItem, directory.entry.type == .directory else { + replyHandler(nil, POSIXError(.ENOTDIR)) + return + } + guard let nameString = name.string else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + let itemPath = join(directory.entry.path, nameString) + do { + var entry: WireEntry + switch type { + case .file: + let created = try client.create(itemPath, flags: O_RDWR | O_APPEND) + try client.handleOperation(.release, handle: created.1, connection: created.0) + created.0.close() + entry = created.2 + case .directory: + try client.mkdir(itemPath, mode: newAttributes.isValid(.mode) ? newAttributes.mode : 0o700) + entry = try client.getattr(itemPath) + default: + throw POSIXError(.ENOTSUP) + } + try applyAttributes(newAttributes, path: itemPath, type: type) + entry = try client.getattr(itemPath) + let item = item(for: entry) + let directoryEntry = try client.getattr(directory.entry.path) + directory.update(directoryEntry) + replyHandler( + FSCreateItemResult( + newItem: item, + newItemName: FSFileName(string: entry.name), + newItemAttributes: attributes(for: entry), + directoryAttributes: attributes(for: directoryEntry), + freeSpace: freeSpaceSnapshot() + ), + nil + ) + } catch { + replyHandler(nil, error) + } + } + + func createSymbolicLink( + named name: FSFileName, + in directory: FSItem, + attributes: FSItem.SetAttributesRequest, + linkContents: FSFileName, + context: FSContext, + replyHandler: @escaping (FSCreateSymlinkResult?, (any Error)?) -> Void + ) { + replyHandler(nil, POSIXError(.ENOTSUP)) + } + + func createLink( + to item: FSItem, + named name: FSFileName, + in directory: FSItem, + context: FSContext, + replyHandler: @escaping (FSCreateLinkResult?, (any Error)?) -> Void + ) { + replyHandler(nil, POSIXError(.ENOTSUP)) + } + + func renameItem( + _ item: FSItem, + inDirectory sourceDirectory: FSItem, + named sourceName: FSFileName, + to destinationName: FSFileName, + inDirectory destinationDirectory: FSItem, + overItem: FSItem?, + context: FSContext, + replyHandler: @escaping (FSRenameItemResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem, + let sourceDirectory = sourceDirectory as? CodexFoldItem, + let destinationDirectory = destinationDirectory as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + guard let sourceNameString = sourceName.string, let destinationNameString = destinationName.string else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + let sourcePath = join(sourceDirectory.entry.path, sourceNameString) + let destinationPath = join(destinationDirectory.entry.path, destinationNameString) + do { + try client.rename(sourcePath, destinationPath) + let renamedEntry = try client.getattr(destinationPath) + item.update(renamedEntry) + let sourceEntry = try client.getattr(sourceDirectory.entry.path) + let destinationEntry = sourceDirectory === destinationDirectory ? sourceEntry : try client.getattr(destinationDirectory.entry.path) + sourceDirectory.update(sourceEntry) + destinationDirectory.update(destinationEntry) + if let overItem = overItem as? CodexFoldItem { + overItem.markDeleted() + removeCached(overItem) + } + replyHandler( + FSRenameItemResult( + newName: FSFileName(string: renamedEntry.name), + renamedItemAttributes: attributes(for: renamedEntry), + sourceDirectoryAttributes: attributes(for: sourceEntry), + destinationDirectoryAttributes: attributes(for: destinationEntry), + overItemAttributes: nil, + freeSpace: freeSpaceSnapshot() + ), + nil + ) + } catch { + replyHandler(nil, error) + } + } + + func removeItem( + _ item: FSItem, + named name: FSFileName, + from directory: FSItem, + context: FSContext, + replyHandler: @escaping (FSRemoveItemResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem, let directory = directory as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + guard let nameString = name.string else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + let removedEntry = item.entry + do { + try client.remove(join(directory.entry.path, nameString), directory: removedEntry.type == .directory) + item.markDeleted() + closeIO(item.replaceIO(nil)) + removeCached(item) + let directoryEntry = try client.getattr(directory.entry.path) + directory.update(directoryEntry) + replyHandler( + FSRemoveItemResult( + itemAttributes: attributes(for: removedEntry), + directoryAttributes: attributes(for: directoryEntry), + freeSpace: freeSpaceSnapshot() + ), + nil + ) + } catch { + replyHandler(nil, error) + } + } + + func getAttributes( + _ desiredAttributes: FSItem.GetAttributesRequest, + of item: FSItem, + context: FSContext, + replyHandler: @escaping (FSGetAttributesResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let entry = try client.getattr(item.entry.path) + item.update(entry) + replyHandler(FSGetAttributesResult(attributes: attributes(for: entry)), nil) + } catch { + replyHandler(nil, error) + } + } + + func setAttributes( + _ newAttributes: FSItem.SetAttributesRequest, + on item: FSItem, + context: FSContext, + replyHandler: @escaping (FSSetAttributesResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + item.invalidateReadCache() + try applyAttributes(newAttributes, path: item.entry.path, type: itemType(item.entry.type)) + let entry = try client.getattr(item.entry.path) + item.update(entry) + replyHandler(FSSetAttributesResult(attributes: attributes(for: entry), freeSpace: freeSpaceSnapshot()), nil) + } catch { + replyHandler(nil, error) + } + } + + func enumerateDirectory( + _ directory: FSItem, + startingAt cookie: FSDirectoryCookie, + verifier: FSDirectoryVerifier, + attributes desiredAttributes: FSItem.GetAttributesRequest?, + packer: FSDirectoryEntryPacker, + context: FSContext, + replyHandler: @escaping (FSEnumerateDirectoryResult?, (any Error)?) -> Void + ) { + guard let directory = directory as? CodexFoldItem, directory.entry.type == .directory else { + replyHandler(nil, POSIXError(.ENOTDIR)) + return + } + do { + let before = try client.getattr(directory.entry.path) + let entries = try client.readDir(directory.entry.path) + let after = try client.getattr(directory.entry.path) + guard before.contentGeneration == after.contentGeneration else { + throw POSIXError(.ESTALE) + } + directory.update(after) + let currentVersion = after.contentGeneration + let start = Int(cookie.rawValue) + guard start >= 0, start <= entries.count else { + throw POSIXError(.EINVAL) + } + if verifier != .initial, verifier.rawValue != currentVersion { + throw POSIXError(.ESTALE) + } + for index in start.. Void + ) { + replyHandler(nil, POSIXError(.ENOTSUP)) + } + + func getXattr( + named name: FSFileName, + of item: FSItem, + context: FSContext, + replyHandler: @escaping (FSGetXattrResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem, let nameString = name.string else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let value = try client.getXattr(item.entry.path, name: nameString) + guard let result = FSGetXattrResult(xattrValue: value) else { + throw POSIXError(.EIO) + } + replyHandler(result, nil) + } catch { + replyHandler(nil, error) + } + } + + func setXattr( + named name: FSFileName, + to value: Data?, + on item: FSItem, + policy: FSVolume.SetXattrPolicy, + context: FSContext, + replyHandler: @escaping (FSSetXattrResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem, let nameString = name.string else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + try client.setXattr(item.entry.path, name: nameString, value: value ?? Data(), policy: UInt32(policy.rawValue)) + if let entry = try? client.getattr(item.entry.path) { + item.update(entry) + } + guard let result = FSSetXattrResult(freeSpace: freeSpaceSnapshot()) else { + throw POSIXError(.EIO) + } + replyHandler(result, nil) + } catch { + replyHandler(nil, error) + } + } + + func listXattrs( + of item: FSItem, + context: FSContext, + replyHandler: @escaping (FSListXattrsResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let names = try client.listXattrs(item.entry.path).map { FSFileName(string: $0) } + guard let result = FSListXattrsResult(xattrNames: names) else { + throw POSIXError(.EIO) + } + replyHandler(result, nil) + } catch { + replyHandler(nil, error) + } + } + + func read( + from item: FSItem, + at offset: off_t, + length: Int, + into buffer: FSMutableFileDataBuffer, + replyHandler: @escaping (FSReadFileResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem, item.entry.type == .file, offset >= 0 else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let io = try ensureIO(item, writable: false) + let bytesRead = try io.read(client: client, offset: Int64(offset), length: length, into: buffer) + let entry = item.entry + replyHandler(FSReadFileResult(bytesRead: bytesRead, itemAttributes: attributes(for: entry)), nil) + } catch { + replyHandler(nil, error) + } + } + + func write( + contents: Data, + to item: FSItem, + at offset: off_t, + replyHandler: @escaping (FSWriteFileResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem, item.entry.type == .file, offset >= 0 else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let io = try ensureIO(item, writable: true) + let previousSize = item.entry.size + io.invalidateReadCache() + let count = try client.write(handle: io.handle, offset: Int64(offset), data: contents, connection: io.connection) + let entry = try client.getattr(item.entry.path) + let invalidateKernelCache = writeRequiresKernelCacheInvalidation( + previousSize: previousSize, + offset: Int64(offset), + writtenBytes: count, + visibleSize: entry.size + ) + item.update(entry) + replyHandler(FSWriteFileResult(bytesWritten: count, itemAttributes: attributes(for: entry), freeSpace: freeSpaceSnapshot()), nil) + if invalidateKernelCache, + let error = setCacheState( + for: item, + cacheMode: .none, + coherencyType: .noCache, + action: .revoke + ) { + logger.error("normalized write cache revoke failed: \(String(describing: error), privacy: .public)") + } + } catch { + replyHandler(nil, error) + } + } + + func open( + _ item: FSItem, + modes: FSVolume.OpenModes, + cacheMode: FSVolume.DataCacheMode, + context: FSContext, + replyHandler: @escaping (FSOpenItemResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let writable = modes.contains(.write) + var nativePassthrough = false + if item.entry.type == .file { + let fileIO = try ensureIO(item, writable: writable) + nativePassthrough = fileIO.usesNativeReadFD + } + let coherency: FSVolume.KernelCacheCoherencyType = !writable && !nativePassthrough && cacheMode != .none ? .readCache : .noCache + replyHandler(FSOpenItemResult(grantedCoherency: coherency), nil) + } catch { + replyHandler(nil, error) + } + } + + func close(_ item: FSItem, context: FSContext, replyHandler: @escaping () -> Void) { + if let item = item as? CodexFoldItem { + let closed = item.replaceIO(nil) + closeIO(closed) + if closed?.writable == true { + if let entry = try? client.getattr(item.entry.path) { + item.update(entry) + } + } + } + replyHandler() + } + + func upgrade( + _ item: FSItem, + cacheMode: FSVolume.DataCacheMode, + context: FSContext, + replyHandler: @escaping (FSUpgradeItemResult?, (any Error)?) -> Void + ) { + guard let item = item as? CodexFoldItem else { + replyHandler(nil, POSIXError(.EINVAL)) + return + } + do { + let writable = cacheMode == .readWriteWithCache + var nativePassthrough = false + if item.entry.type == .file { + let fileIO = try ensureIO(item, writable: writable) + nativePassthrough = fileIO.usesNativeReadFD + } + let coherency: FSVolume.KernelCacheCoherencyType = !writable && !nativePassthrough && cacheMode != .none ? .readCache : .noCache + replyHandler(FSUpgradeItemResult(grantedCoherency: coherency), nil) + } catch { + replyHandler(nil, error) + } + } + + private func item(for entry: WireEntry) -> CodexFoldItem { + itemLock.lock() + defer { itemLock.unlock() } + if let existing = items[entry.nodeID] { + existing.update(entry) + return existing + } + let item = CodexFoldItem(entry: entry) + items[entry.nodeID] = item + return item + } + + private func removeCached(_ item: CodexFoldItem) { + itemLock.lock() + if items[item.entry.nodeID] === item { + items.removeValue(forKey: item.entry.nodeID) + } + itemLock.unlock() + } + + private func ensureIO(_ item: CodexFoldItem, writable: Bool) throws -> CodexFoldIO { + if let existing = item.io(), !writable || existing.writable { + return existing + } + closeIO(item.replaceIO(nil)) + let connection = try client.newConnection() + var flags: Int32 = writable ? O_RDWR : O_RDONLY + if writable && item.entry.path.hasSuffix(".jsonl") && !item.entry.name.hasPrefix("._") { + flags |= O_APPEND + flags |= Int32(bitPattern: 1 << 31) + } + do { + let opened = try client.open(item.entry.path, flags: flags, connection: connection) + let io = CodexFoldIO( + client: client, + connection: connection, + handle: opened.handle, + path: item.entry.path, + writable: writable, + size: item.entry.size, + nativeReadFD: opened.nativeReadFD, + sharedReadWindow: opened.sharedReadWindow + ) + closeIO(item.replaceIO(io)) + return io + } catch { + connection.close() + throw error + } + } + + private func applyAttributes( + _ request: FSItem.SetAttributesRequest, + path: String, + type: FSItem.ItemType + ) throws { + if type == .file, request.isValid(.size) { + try client.truncate(path, size: request.size) + request.consumedAttributes.insert(.size) + } + + let hasMode = request.isValid(.mode) + let hasUID = request.isValid(.uid) + let hasGID = request.isValid(.gid) + let hasAccessTime = request.isValid(.accessTime) + let hasModifyTime = request.isValid(.modifyTime) + var valid: UInt32 = 0 + if hasMode { valid |= 1 << 0 } + if hasUID { valid |= 1 << 1 } + if hasGID { valid |= 1 << 2 } + if hasAccessTime { valid |= 1 << 3 } + if hasModifyTime { valid |= 1 << 4 } + guard valid != 0 else { return } + + try client.setAttributes( + path, + valid: valid, + mode: hasMode ? request.mode : 0, + uid: hasUID ? request.uid : 0, + gid: hasGID ? request.gid : 0, + accessTime: hasAccessTime ? request.accessTime : timespec(), + modifyTime: hasModifyTime ? request.modifyTime : timespec() + ) + if hasMode { request.consumedAttributes.insert(.mode) } + if hasUID { request.consumedAttributes.insert(.uid) } + if hasGID { request.consumedAttributes.insert(.gid) } + if hasAccessTime { request.consumedAttributes.insert(.accessTime) } + if hasModifyTime { request.consumedAttributes.insert(.modifyTime) } + } + + private func closeIO(_ io: CodexFoldIO?) { + guard let io else { return } + io.shutdown() + if io.writable { + try? client.handleOperation(.fsync, handle: io.handle, connection: io.connection) + } + try? client.handleOperation(.release, handle: io.handle, connection: io.connection) + io.connection.close() + } + + private func closeAllItems() { + itemLock.lock() + let snapshot = Array(items.values) + itemLock.unlock() + for item in snapshot { + closeIO(item.replaceIO(nil)) + } + } + + private func startNamespaceMonitor() { + stopNamespaceMonitor() + let timer = DispatchSource.makeTimerSource(queue: DispatchQueue(label: "vip.jstar.codexfold.fskit.namespace")) + timer.schedule(deadline: .now() + .milliseconds(250), repeating: .milliseconds(500), leeway: .milliseconds(100)) + timer.setEventHandler { [weak self] in self?.refreshNamespace() } + namespaceTimer = timer + timer.resume() + } + + private func stopNamespaceMonitor() { + namespaceTimer?.cancel() + namespaceTimer = nil + } + + private func refreshNamespace() { + guard let current = try? client.namespaceVersion() else { return } + itemLock.lock() + let previousVersion = namespaceVersion + if current == namespaceVersion { + itemLock.unlock() + return + } + namespaceVersion = current + let candidates = items.compactMap { nodeID, item -> (nodeID: UInt64, item: CodexFoldItem)? in + item === rootItem ? nil : (nodeID, item) + } + itemLock.unlock() + let candidatePaths = candidates.map { $0.item.entry.path }.sorted().joined(separator: ",") + logger.info( + "namespace refresh previous=\(previousVersion) current=\(current) candidates=\(candidates.count) paths=\(candidatePaths, privacy: .public)" + ) + if let rootEntry = try? client.getattr("/") { + rootItem.update(rootEntry) + } + + var changedItems: [CodexFoldItem] = [] + var changedDirectories: [CodexFoldItem] = [] + var staleItems: [(nodeID: UInt64, item: CodexFoldItem)] = [] + for candidate in candidates { + let item = candidate.item + let previous = item.entry + guard let refreshed = try? client.getattr(previous.path), + previous.hasSameObjectIdentity(as: refreshed) else { + staleItems.append((candidate.nodeID, item)) + continue + } + if previous.type == .directory && + !previous.hasSameCachedDirectoryContents(as: refreshed) { + item.update(refreshed) + changedDirectories.append(item) + continue + } + if previous.type == .file && !previous.hasSameCachedFileData(as: refreshed) { + changedItems.append(item) + } + item.update(refreshed) + } + let changedDirectoryPaths = changedDirectories.map { $0.entry.path }.sorted().joined(separator: ",") + logger.info( + "namespace delta directories=\(changedDirectories.count) directory_paths=\(changedDirectoryPaths, privacy: .public) files=\(changedItems.count) stale=\(staleItems.count)" + ) + + var removedItems: [CodexFoldItem] = [] + itemLock.lock() + for stale in staleItems where items[stale.nodeID] === stale.item { + items.removeValue(forKey: stale.nodeID) + removedItems.append(stale.item) + } + itemLock.unlock() + for item in changedDirectories { + if let error = setCacheState( + for: item, + cacheMode: .none, + coherencyType: .noCache, + // The directory still exists. Revoke is reserved for an item + // that disappeared; invalidate asks the kernel to discard its + // cached directory state without invalidating the live vnode. + action: .invalidate + ) { + logger.error("directory contents cache invalidation failed: \(String(describing: error), privacy: .public)") + } + } + for item in changedItems { + item.invalidateReadCache() + if let error = setCacheState( + for: item, + cacheMode: .none, + coherencyType: .noCache, + // A plain data-cache invalidation does not evict stale vnode + // attributes after an external file-size change. + action: .revoke + ) { + logger.error("external change cache revoke failed: \(String(describing: error), privacy: .public)") + } + } + for item in removedItems { + item.invalidateReadCache() + if let error = setCacheState( + for: item, + cacheMode: .none, + coherencyType: .noCache, + action: .revoke + ) { + logger.error("external removal cache revoke failed: \(String(describing: error), privacy: .public)") + } + } + } + + private func attributes(for entry: WireEntry) -> FSItem.Attributes { + let result = FSItem.Attributes() + result.uid = entry.uid + result.gid = entry.gid + result.linkCount = 1 + result.fileID = FSItem.Identifier(rawValue: entry.nodeID)! + result.parentID = FSItem.Identifier(rawValue: entry.parentID)! + result.mode = entry.mode + result.type = itemType(entry.type) + result.size = entry.type == .directory ? 0 : entry.size + result.allocSize = entry.type == .directory ? 0 : entry.allocSize + result.modifyTime = entry.modifyTime + result.changeTime = entry.changeTime + result.accessTime = entry.accessTime + return result + } + + private func freeSpaceSnapshot() -> FSFreeSpace { + guard let stat = try? client.statfs() else { + return FSFreeSpace.noUpdate + } + let freeSpace = FSFreeSpace() + freeSpace.populate(bytes: stat.availableBytes) + return freeSpace + } + + private func itemType(_ type: WireEntryType) -> FSItem.ItemType { + switch type { + case .file: return .file + case .directory: return .directory + case .symlink: return .symlink + case .unknown: return .unknown + } + } + + private func join(_ directory: String, _ name: String) -> String { + if directory == "/" { return "/" + name } + return directory + "/" + name + } +} diff --git a/platform/darwin/fskit/Extension/ReadCache.swift b/platform/darwin/fskit/Extension/ReadCache.swift new file mode 100644 index 0000000..fc378b6 --- /dev/null +++ b/platform/darwin/fskit/Extension/ReadCache.swift @@ -0,0 +1,150 @@ +import Darwin +import Foundation + +struct CodexFoldReadAheadPolicy { + private static let preferredReadAheadBytes = 12 * 1024 * 1024 + private static let fallbackReadAheadBytes = 11 * 1024 * 1024 + private static let maximumPrefetchBytes = 96 * 1024 * 1024 + private static let maximumConcurrentPrefetchCount = 8 + private static let maximumScheduledPrefetchCount = 8 + + let readAheadBytes: Int + let concurrentPrefetchCount: Int + let scheduledPrefetchCount: Int + let maxCachedBlocks: Int + + init(negotiatedReadBytes: Int) { + if negotiatedReadBytes >= Self.preferredReadAheadBytes { + readAheadBytes = Self.preferredReadAheadBytes + } else { + readAheadBytes = min(Self.fallbackReadAheadBytes, negotiatedReadBytes) + } + scheduledPrefetchCount = max( + 1, + min( + Self.maximumScheduledPrefetchCount, + Self.maximumPrefetchBytes / max(1, readAheadBytes) + ) + ) + concurrentPrefetchCount = min( + Self.maximumConcurrentPrefetchCount, + scheduledPrefetchCount + ) + maxCachedBlocks = scheduledPrefetchCount + 1 + } +} + +extension WireEntry { + func hasSameObjectIdentity(as other: WireEntry) -> Bool { + type == other.type && + path == other.path && + nodeID == other.nodeID + } + + func hasSameCachedFileData(as other: WireEntry) -> Bool { + type == .file && + other.type == .file && + hasSameObjectIdentity(as: other) && + size == other.size && + allocSize == other.allocSize && + modifyTime.tv_sec == other.modifyTime.tv_sec && + modifyTime.tv_nsec == other.modifyTime.tv_nsec && + changeTime.tv_sec == other.changeTime.tv_sec && + changeTime.tv_nsec == other.changeTime.tv_nsec + } + + func hasSameCachedDirectoryContents(as other: WireEntry) -> Bool { + type == .directory && + other.type == .directory && + hasSameObjectIdentity(as: other) && + contentGeneration == other.contentGeneration && + modifyTime.tv_sec == other.modifyTime.tv_sec && + modifyTime.tv_nsec == other.modifyTime.tv_nsec && + changeTime.tv_sec == other.changeTime.tv_sec && + changeTime.tv_nsec == other.changeTime.tv_nsec + } +} + +func writeRequiresKernelCacheInvalidation( + previousSize: UInt64, + offset: Int64, + writtenBytes: Int, + visibleSize: UInt64 +) -> Bool { + guard offset >= 0, writtenBytes >= 0 else { return true } + let (literalEnd, overflow) = UInt64(offset).addingReportingOverflow(UInt64(writtenBytes)) + guard !overflow else { return true } + return visibleSize != max(previousSize, literalEnd) +} + +final class CodexFoldCachedReadBlock { + let count: Int + + private enum Storage { + case data(Data) + case sharedWindow(WireSharedReadWindow, CodexFoldReadLease) + } + + private let storage: Storage + + init(data: Data) { + self.count = data.count + self.storage = .data(data) + } + + init( + sharedWindow: WireSharedReadWindow, + count: Int, + release: @escaping () -> Void + ) throws { + guard count >= 0, count <= sharedWindow.capacity else { + throw POSIXError(.EPROTO) + } + self.count = count + self.storage = .sharedWindow(sharedWindow, CodexFoldReadLease(release: release)) + } + + func copyBytes( + from sourceOffset: Int, + count requestedCount: Int, + to destination: UnsafeMutableRawPointer + ) -> Bool { + guard sourceOffset >= 0, + requestedCount >= 0, + sourceOffset <= count, + requestedCount <= count - sourceOffset else { + return false + } + guard requestedCount > 0 else { return true } + + switch storage { + case .data(let data): + _ = data.withUnsafeBytes { source in + Darwin.memcpy( + destination, + source.baseAddress!.advanced(by: sourceOffset), + requestedCount + ) + } + return true + case .sharedWindow(let window, _): + return window.copyBytes( + from: sourceOffset, + count: requestedCount, + to: destination + ) + } + } +} + +private final class CodexFoldReadLease { + private let release: () -> Void + + init(release: @escaping () -> Void) { + self.release = release + } + + deinit { + release() + } +} diff --git a/platform/darwin/fskit/Extension/Wire.swift b/platform/darwin/fskit/Extension/Wire.swift new file mode 100644 index 0000000..7c3061a --- /dev/null +++ b/platform/darwin/fskit/Extension/Wire.swift @@ -0,0 +1,1300 @@ +import Darwin +import Foundation + +private let wireMagic = Data([0x43, 0x46, 0x53, 0x50]) +private let descriptorMagic = Data([0x43, 0x46, 0x53, 0x52]) +private let wireVersion: UInt16 = 2 +private let wireHeaderSize = 40 +private let defaultMaxPayload = 32 * 1024 * 1024 +private let capabilityNativeReadFD: UInt32 = 1 << 0 +private let capabilitySharedReadFD: UInt32 = 1 << 1 +private let capabilitySharedWindow: UInt32 = 1 << 2 +private let capabilitySharedFileWindow: UInt32 = 1 << 3 +private let capabilityContentGeneration: UInt32 = 1 << 4 +private let flagNativeReadFD: UInt32 = 1 << 0 +private let flagSharedReadFD: UInt32 = 1 << 1 +private let flagSharedWindow: UInt32 = 1 << 2 +private let flagSharedFileWindow: UInt32 = 1 << 3 +private let nativeReadFDMarker: UInt8 = 0x46 +private let sharedReadFDMarker: UInt8 = 0x53 +private let sharedWindowFDMarker: UInt8 = 0x57 +private let sharedFileWindowFDMarker: UInt8 = 0x52 +private let socketBufferBytes: Int32 = 4 * 1024 * 1024 + +enum WireOperation: UInt8 { + case hello = 1 + case ping + case getattr + case readDir + case open + case create + case read + case write + case fsync + case flush + case release + case truncate + case mkdir + case rename + case unlink + case rmdir + case statfs + case sync + case namespaceVersion + case setattr + case getXattr + case setXattr + case listXattrs +} + +enum WireEntryType: UInt8 { + case unknown = 0 + case file + case directory + case symlink +} + +struct WireDescriptor { + let generation: UInt64 + let socketPath: String + let token: Data + + init(resourceURL: URL) throws { + var isDirectory = ObjCBool(false) + guard FileManager.default.fileExists(atPath: resourceURL.path, isDirectory: &isDirectory) else { + throw POSIXError(.ENOENT) + } + let descriptorURL = isDirectory.boolValue + ? resourceURL.appendingPathComponent("descriptor.bin", isDirectory: false) + : resourceURL + try self.init(data: Data(contentsOf: descriptorURL)) + } + + init(data: Data) throws { + var reader = WireReader(data) + guard try reader.raw(count: 4) == descriptorMagic else { + throw POSIXError(.EPROTO) + } + guard try reader.uint16() == wireVersion else { + throw POSIXError(.EPROTONOSUPPORT) + } + _ = try reader.uint16() + generation = try reader.uint64() + socketPath = try reader.string(limit: 4096) + token = try reader.bytes(limit: 256) + try reader.finish() + guard generation != 0, !socketPath.isEmpty, token.count >= 16 else { + throw POSIXError(.EINVAL) + } + } +} + +struct WireEntry { + let path: String + let name: String + let nodeID: UInt64 + let parentID: UInt64 + let type: WireEntryType + let mode: UInt32 + let uid: UInt32 + let gid: UInt32 + let size: UInt64 + let allocSize: UInt64 + let modifyTime: timespec + let changeTime: timespec + let accessTime: timespec + let namespaceID: UInt64 + let contentGeneration: UInt64 + + init(reader: inout WireReader, includesContentGeneration: Bool = false) throws { + path = try reader.string(limit: 1 << 20) + name = try reader.string(limit: 4096) + nodeID = try reader.uint64() + parentID = try reader.uint64() + guard let type = WireEntryType(rawValue: try reader.uint8()) else { + throw POSIXError(.EPROTO) + } + self.type = type + mode = try reader.uint32() + uid = try reader.uint32() + gid = try reader.uint32() + size = try reader.uint64() + allocSize = try reader.uint64() + modifyTime = try reader.time() + changeTime = try reader.time() + accessTime = try reader.time() + namespaceID = try reader.uint64() + contentGeneration = includesContentGeneration ? try reader.uint64() : 0 + } +} + +struct WireStatFS { + let blockSize: UInt32 + let ioSize: UInt32 + let totalBytes: UInt64 + let availableBytes: UInt64 + let freeBytes: UInt64 + let usedBytes: UInt64 + let totalFiles: UInt64 + let freeFiles: UInt64 + + init(reader: inout WireReader) throws { + blockSize = try reader.uint32() + ioSize = try reader.uint32() + totalBytes = try reader.uint64() + availableBytes = try reader.uint64() + freeBytes = try reader.uint64() + usedBytes = try reader.uint64() + totalFiles = try reader.uint64() + freeFiles = try reader.uint64() + } +} + +struct WireWriter { + private(set) var data = Data() + + mutating func raw(_ value: Data) { + data.append(value) + } + + mutating func uint8(_ value: UInt8) { + data.append(value) + } + + mutating func uint16(_ value: UInt16) { + appendFixed(value) + } + + mutating func uint32(_ value: UInt32) { + appendFixed(value) + } + + mutating func uint64(_ value: UInt64) { + appendFixed(value) + } + + mutating func int64(_ value: Int64) { + appendFixed(UInt64(bitPattern: value)) + } + + mutating func bytes(_ value: Data) { + uint32(UInt32(value.count)) + raw(value) + } + + mutating func string(_ value: String) { + bytes(Data(value.utf8)) + } + + mutating func time(_ value: timespec) { + int64(Int64(value.tv_sec)) + uint32(UInt32(value.tv_nsec)) + } + + private mutating func appendFixed(_ value: T) { + var littleEndian = value.littleEndian + withUnsafeBytes(of: &littleEndian) { data.append(contentsOf: $0) } + } +} + +struct WireReader { + private let data: Data + private var offset = 0 + + init(_ data: Data) { + self.data = data + } + + var remaining: Int { + data.count - offset + } + + mutating func raw(count: Int) throws -> Data { + guard count >= 0, offset <= data.count, data.count - offset >= count else { + throw POSIXError(.EPROTO) + } + let result = data[offset..<(offset + count)] + offset += count + return result + } + + mutating func uint8() throws -> UInt8 { + let value = try raw(count: 1) + return value[value.startIndex] + } + + mutating func uint16() throws -> UInt16 { + try readFixed(UInt16.self) + } + + mutating func uint32() throws -> UInt32 { + try readFixed(UInt32.self) + } + + mutating func uint64() throws -> UInt64 { + try readFixed(UInt64.self) + } + + mutating func int64() throws -> Int64 { + Int64(bitPattern: try uint64()) + } + + mutating func bytes(limit: Int) throws -> Data { + let count = Int(try uint32()) + guard count <= limit else { + throw POSIXError(.E2BIG) + } + return try raw(count: count) + } + + mutating func string(limit: Int) throws -> String { + guard let value = String(data: try bytes(limit: limit), encoding: .utf8) else { + throw POSIXError(.EILSEQ) + } + return value + } + + mutating func time() throws -> timespec { + let seconds = try int64() + let nanoseconds = try uint32() + guard nanoseconds < 1_000_000_000 else { + throw POSIXError(.EPROTO) + } + return timespec(tv_sec: Int(seconds), tv_nsec: Int(nanoseconds)) + } + + mutating func finish() throws { + guard offset == data.count else { + throw POSIXError(.EPROTO) + } + } + + private mutating func readFixed(_ type: T.Type) throws -> T { + let value = try raw(count: MemoryLayout.size) + return value.withUnsafeBytes { bytes in + T(littleEndian: bytes.loadUnaligned(as: T.self)) + } + } +} + +struct WireResponse { + let operation: WireOperation + let flags: UInt32 + let requestID: UInt64 + let generation: UInt64 + let status: Int32 + let payload: Data + let nativeReadFD: Int32? + let sharedReadFD: Int32? + let sharedWindowFD: Int32? +} + +struct WireOpenResult { + let handle: UInt64 + let nativeReadFD: Int32? + let sharedReadWindow: WireSharedReadWindow? +} + +enum WireReadResult { + case copied(Data) + case sharedWindow(count: Int) +} + +final class WireSharedReadWindow { + let capacity: Int + let wireFlag: UInt32 + + private enum Storage { + case mapped(UnsafeMutableRawPointer, Int) + case mappedFile(UnsafeMutableRawPointer, Int, Int32) + } + + private let storage: Storage + + init(descriptor: Int32, capacity: Int) throws { + guard descriptor >= 0, capacity > 0 else { + if descriptor >= 0 { + Darwin.close(descriptor) + } + throw POSIXError(.EINVAL) + } + defer { Darwin.close(descriptor) } + + let pageSize = Int64(Darwin.getpagesize()) + let requestedBytes = Int64(capacity) + let roundedBytes = ((requestedBytes + pageSize - 1) / pageSize) * pageSize + guard roundedBytes > 0, roundedBytes <= Int64(Int.max) else { + throw POSIXError(.EOVERFLOW) + } + var metadata = stat() + guard Darwin.fstat(descriptor, &metadata) == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + guard Int64(metadata.st_size) == roundedBytes else { + throw POSIXError(.EPROTO) + } + let mappingLength = Int(roundedBytes) + let mapped = Darwin.mmap(nil, mappingLength, PROT_READ, MAP_SHARED, descriptor, 0) + let mapError = errno + guard mapped != MAP_FAILED, let mapped else { + throw POSIXError(POSIXErrorCode(rawValue: mapError) ?? .EIO) + } + Self.prefaultReadMapping(mapped, length: mappingLength) + self.capacity = capacity + self.wireFlag = flagSharedWindow + self.storage = .mapped(mapped, mappingLength) + } + + init(fileDescriptor descriptor: Int32, capacity: Int) throws { + guard descriptor >= 0, capacity > 0 else { + if descriptor >= 0 { + Darwin.close(descriptor) + } + throw POSIXError(.EINVAL) + } + var metadata = stat() + guard Darwin.fstat(descriptor, &metadata) == 0 else { + let code = errno + Darwin.close(descriptor) + throw POSIXError(POSIXErrorCode(rawValue: code) ?? .EIO) + } + let pageSize = Int64(Darwin.getpagesize()) + let requestedBytes = Int64(capacity) + let roundedBytes = ((requestedBytes + pageSize - 1) / pageSize) * pageSize + guard roundedBytes > 0, + roundedBytes <= Int64(Int.max), + Int64(metadata.st_size) == roundedBytes else { + Darwin.close(descriptor) + throw POSIXError(.EPROTO) + } + let mappingLength = Int(roundedBytes) + let mapped = Darwin.mmap(nil, mappingLength, PROT_READ, MAP_SHARED, descriptor, 0) + let mapError = errno + guard mapped != MAP_FAILED, let mapped else { + Darwin.close(descriptor) + throw POSIXError(POSIXErrorCode(rawValue: mapError) ?? .EIO) + } + Self.prefaultReadMapping(mapped, length: mappingLength) + self.capacity = capacity + self.wireFlag = flagSharedFileWindow + self.storage = .mappedFile(mapped, mappingLength, descriptor) + } + + deinit { + switch storage { + case .mapped(let mapping, let mappingLength): + _ = Darwin.munmap(mapping, mappingLength) + case .mappedFile(let mapping, let mappingLength, let descriptor): + _ = Darwin.munmap(mapping, mappingLength) + Darwin.close(descriptor) + } + } + + func copyData(count: Int) throws -> Data { + guard count >= 0, count <= capacity else { + throw POSIXError(.EPROTO) + } + switch storage { + case .mapped(let mapping, _): + return Data(bytes: mapping, count: count) + case .mappedFile(let mapping, _, _): + return Data(bytes: mapping, count: count) + } + } + + func copyBytes( + from sourceOffset: Int, + count requestedCount: Int, + to destination: UnsafeMutableRawPointer + ) -> Bool { + guard sourceOffset >= 0, + requestedCount >= 0, + sourceOffset <= capacity, + requestedCount <= capacity - sourceOffset else { + return false + } + guard requestedCount > 0 else { return true } + switch storage { + case .mapped(let mapping, _): + Darwin.memcpy( + destination, + mapping.advanced(by: sourceOffset), + requestedCount + ) + return true + case .mappedFile(let mapping, _, _): + Darwin.memcpy( + destination, + mapping.advanced(by: sourceOffset), + requestedCount + ) + return true + } + } + + private static func prefaultReadMapping(_ mapping: UnsafeMutableRawPointer, length: Int) { + guard length > 0 else { return } + _ = Darwin.madvise(mapping, length, MADV_WILLNEED) + if Darwin.mlock(mapping, length) == 0 { + _ = Darwin.munlock(mapping, length) + } + } +} + +final class WireConnection { + private let lock = NSLock() + private var descriptor: WireDescriptor + private var socket: Int32 = -1 + private var requestID: UInt64 = 1 + private var maxPayload = defaultMaxPayload + private(set) var supportsContentGeneration = false + + var maximumPayload: Int { + maxPayload + } + + var maximumReadBytes: Int { + max(0, maxPayload - MemoryLayout.size) + } + + init(descriptor: WireDescriptor) throws { + self.descriptor = descriptor + try connect() + var hello = WireWriter() + hello.bytes(descriptor.token) + hello.uint32( + capabilityNativeReadFD | + capabilitySharedReadFD | + capabilitySharedWindow | + capabilitySharedFileWindow | + capabilityContentGeneration + ) + let response = try requestLocked(operation: .hello, generation: 0, payload: hello.data) + if response.flags != 0 || + response.nativeReadFD != nil || + response.sharedReadFD != nil || + response.sharedWindowFD != nil { + Self.closeDescriptors(response) + throw POSIXError(.EPROTO) + } + var reader = WireReader(response.payload) + maxPayload = Int(try reader.uint32()) + _ = try reader.uint64() + let acceptedCapabilities = reader.remaining == 0 ? 0 : try reader.uint32() + try reader.finish() + supportsContentGeneration = acceptedCapabilities & capabilityContentGeneration != 0 + guard maxPayload >= 4096 else { + throw POSIXError(.EPROTO) + } + } + + deinit { + closeSocket() + } + + func request(_ operation: WireOperation, payload: Data = Data()) throws -> Data { + lock.lock() + defer { lock.unlock() } + let response = try requestLocked(operation: operation, generation: descriptor.generation, payload: payload) + guard response.flags == 0, + response.nativeReadFD == nil, + response.sharedReadFD == nil, + response.sharedWindowFD == nil else { + Self.closeDescriptors(response) + throw POSIXError(.EPROTO) + } + return response.payload + } + + func requestWithReadFD(_ operation: WireOperation, payload: Data = Data()) throws -> WireResponse { + lock.lock() + defer { lock.unlock() } + return try requestLocked(operation: operation, generation: descriptor.generation, payload: payload) + } + + func requestRead( + payload: Data, + requestedLength: Int, + sharedWindow: WireSharedReadWindow? + ) throws -> Data { + let result = try requestReadResult( + payload: payload, + requestedLength: requestedLength, + sharedWindow: sharedWindow, + borrowSharedWindow: false + ) + guard case .copied(let data) = result else { + throw POSIXError(.EPROTO) + } + return data + } + + func requestReadBorrowingSharedWindow( + payload: Data, + requestedLength: Int, + sharedWindow: WireSharedReadWindow + ) throws -> WireReadResult { + try requestReadResult( + payload: payload, + requestedLength: requestedLength, + sharedWindow: sharedWindow, + borrowSharedWindow: true + ) + } + + private func requestReadResult( + payload: Data, + requestedLength: Int, + sharedWindow: WireSharedReadWindow?, + borrowSharedWindow: Bool + ) throws -> WireReadResult { + guard requestedLength >= 0 else { throw POSIXError(.EINVAL) } + lock.lock() + defer { lock.unlock() } + let response = try requestLocked( + operation: .read, + generation: descriptor.generation, + payload: payload + ) + let nativeReadFD = response.nativeReadFD + var sharedReadFD = response.sharedReadFD + let sharedWindowFD = response.sharedWindowFD + do { + guard nativeReadFD == nil, sharedWindowFD == nil else { + throw POSIXError(.EPROTO) + } + if response.flags & (flagSharedWindow | flagSharedFileWindow) != 0 { + guard let sharedWindow, + response.flags == sharedWindow.wireFlag, + sharedReadFD == nil, + response.flags == flagSharedWindow || response.flags == flagSharedFileWindow else { + throw POSIXError(.EPROTO) + } + var reader = WireReader(response.payload) + let count = Int(try reader.uint32()) + try reader.finish() + guard count <= requestedLength else { + throw POSIXError(.EPROTO) + } + if borrowSharedWindow { + return .sharedWindow(count: count) + } + // Keep the connection lock until the bytes are copied. A later + // read on this handle may immediately overwrite the same window. + return .copied(try sharedWindow.copyData(count: count)) + } + if let descriptor = sharedReadFD { + guard response.flags == flagSharedReadFD else { + throw POSIXError(.EPROTO) + } + sharedReadFD = nil + defer { Darwin.close(descriptor) } + var reader = WireReader(response.payload) + let count = Int(try reader.uint32()) + try reader.finish() + guard count > 0, count <= requestedLength else { + throw POSIXError(.EPROTO) + } + return .copied(try Self.mapSharedReadFD( + descriptor, + count: count, + maximumCount: requestedLength + )) + } + guard response.flags == 0 else { + throw POSIXError(.EPROTO) + } + var reader = WireReader(response.payload) + let data = try reader.bytes(limit: maxPayload) + try reader.finish() + guard data.count <= requestedLength else { + throw POSIXError(.EPROTO) + } + return .copied(data) + } catch { + if let nativeReadFD { + Darwin.close(nativeReadFD) + } + if let sharedReadFD { + Darwin.close(sharedReadFD) + } + if let sharedWindowFD { + Darwin.close(sharedWindowFD) + } + throw error + } + } + + func close() { + lock.lock() + closeSocket() + lock.unlock() + } + + private func connect() throws { + let descriptor = Darwin.socket(AF_UNIX, SOCK_STREAM, 0) + guard descriptor >= 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + var noSigPipe: Int32 = 1 + _ = setsockopt(descriptor, SOL_SOCKET, SO_NOSIGPIPE, &noSigPipe, socklen_t(MemoryLayout.size)) + var socketBuffer = socketBufferBytes + _ = setsockopt(descriptor, SOL_SOCKET, SO_RCVBUF, &socketBuffer, socklen_t(MemoryLayout.size)) + _ = setsockopt(descriptor, SOL_SOCKET, SO_SNDBUF, &socketBuffer, socklen_t(MemoryLayout.size)) + var address = sockaddr_un() + address.sun_family = sa_family_t(AF_UNIX) + let pathBytes = Array(self.descriptor.socketPath.utf8CString) + guard pathBytes.count <= MemoryLayout.size(ofValue: address.sun_path) else { + Darwin.close(descriptor) + throw POSIXError(.ENAMETOOLONG) + } + withUnsafeMutableBytes(of: &address.sun_path) { destination in + destination.initializeMemory(as: UInt8.self, repeating: 0) + for (index, value) in pathBytes.enumerated() { + destination[index] = UInt8(bitPattern: value) + } + } + let result = withUnsafePointer(to: &address) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { socketAddress in + Darwin.connect(descriptor, socketAddress, socklen_t(MemoryLayout.size)) + } + } + guard result == 0 else { + let code = errno + Darwin.close(descriptor) + throw POSIXError(POSIXErrorCode(rawValue: code) ?? .EIO) + } + socket = descriptor + } + + private func requestLocked(operation: WireOperation, generation: UInt64, payload: Data) throws -> WireResponse { + guard socket >= 0 else { + throw POSIXError(.ENOTCONN) + } + guard payload.count <= maxPayload else { + throw POSIXError(.E2BIG) + } + let currentID = requestID + requestID &+= 1 + var header = WireWriter() + header.raw(wireMagic) + header.uint16(wireVersion) + header.uint8(1) + header.uint8(operation.rawValue) + header.uint32(0) + header.uint64(currentID) + header.uint64(generation) + header.uint32(0) + header.uint32(UInt32(payload.count)) + header.uint32(0) + try writeAll(header.data) + try writeAll(payload) + + var responseReader = WireReader(try readExact(count: wireHeaderSize)) + guard try responseReader.raw(count: 4) == wireMagic, + try responseReader.uint16() == wireVersion, + try responseReader.uint8() == 2, + try responseReader.uint8() == operation.rawValue else { + throw POSIXError(.EPROTO) + } + let flags = try responseReader.uint32() + guard try responseReader.uint64() == currentID, + try responseReader.uint64() == descriptor.generation else { + throw POSIXError(.EPROTO) + } + let status = Int32(bitPattern: try responseReader.uint32()) + let payloadLength = Int(try responseReader.uint32()) + _ = try responseReader.uint32() + try responseReader.finish() + guard payloadLength <= maxPayload else { + throw POSIXError(.E2BIG) + } + let responsePayload = try readExact(count: payloadLength) + var nativeReadFD: Int32? + var sharedReadFD: Int32? + var sharedWindowFD: Int32? + do { + let knownFlags = flagNativeReadFD | flagSharedReadFD | flagSharedWindow | flagSharedFileWindow + let transferFlags = flags & knownFlags + guard flags & ~knownFlags == 0, + transferFlags.nonzeroBitCount <= 1 else { + throw POSIXError(.EPROTO) + } + if flags & flagNativeReadFD != 0 { + guard operation == .open, status == 0 else { + throw POSIXError(.EPROTO) + } + nativeReadFD = try readReadFD(expectedMarker: nativeReadFDMarker) + } + if flags & flagSharedReadFD != 0 { + guard operation == .read, status == 0 else { + throw POSIXError(.EPROTO) + } + sharedReadFD = try readReadFD(expectedMarker: sharedReadFDMarker) + } + if flags & flagSharedWindow != 0 { + guard status == 0 else { + throw POSIXError(.EPROTO) + } + if operation == .open { + sharedWindowFD = try readReadFD(expectedMarker: sharedWindowFDMarker) + } else if operation != .read { + throw POSIXError(.EPROTO) + } + } + if flags & flagSharedFileWindow != 0 { + guard status == 0 else { + throw POSIXError(.EPROTO) + } + if operation == .open { + sharedWindowFD = try readReadFD(expectedMarker: sharedFileWindowFDMarker) + } else if operation != .read { + throw POSIXError(.EPROTO) + } + } + } catch { + if let nativeReadFD { Darwin.close(nativeReadFD) } + if let sharedReadFD { Darwin.close(sharedReadFD) } + if let sharedWindowFD { Darwin.close(sharedWindowFD) } + throw error + } + if status != 0 { + if let nativeReadFD { Darwin.close(nativeReadFD) } + if let sharedReadFD { Darwin.close(sharedReadFD) } + if let sharedWindowFD { Darwin.close(sharedWindowFD) } + throw POSIXError(POSIXErrorCode(rawValue: status) ?? .EIO) + } + return WireResponse( + operation: operation, + flags: flags, + requestID: currentID, + generation: descriptor.generation, + status: status, + payload: responsePayload, + nativeReadFD: nativeReadFD, + sharedReadFD: sharedReadFD, + sharedWindowFD: sharedWindowFD + ) + } + + private static func closeDescriptors(_ response: WireResponse) { + if let nativeReadFD = response.nativeReadFD { + Darwin.close(nativeReadFD) + } + if let sharedReadFD = response.sharedReadFD { + Darwin.close(sharedReadFD) + } + if let sharedWindowFD = response.sharedWindowFD { + Darwin.close(sharedWindowFD) + } + } + + private static func mapSharedReadFD(_ descriptor: Int32, count: Int, maximumCount: Int) throws -> Data { + guard descriptor >= 0, count > 0, maximumCount >= count else { + throw POSIXError(.EINVAL) + } + var metadata = stat() + guard Darwin.fstat(descriptor, &metadata) == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + let pageSize = Int64(Darwin.getpagesize()) + let mappedBytes = Int64(count) + let mappingBytes = ((mappedBytes + pageSize - 1) / pageSize) * pageSize + let maximumFileBytes = ((Int64(maximumCount) + pageSize - 1) / pageSize) * pageSize + guard Int64(metadata.st_size) >= mappedBytes, + Int64(metadata.st_size) <= maximumFileBytes else { + throw POSIXError(.EPROTO) + } + let mappingLength = Int(mappingBytes) + let mapping = Darwin.mmap(nil, mappingLength, PROT_READ, MAP_SHARED, descriptor, 0) + let mapError = errno + guard mapping != MAP_FAILED, let mapping else { + throw POSIXError(POSIXErrorCode(rawValue: mapError) ?? .EIO) + } + return Data(bytesNoCopy: mapping, count: count, deallocator: .custom { pointer, _ in + _ = Darwin.munmap(pointer, mappingLength) + }) + } + + private func readExact(count: Int) throws -> Data { + var result = Data(count: count) + var completed = 0 + while completed < count { + let amount = result.withUnsafeMutableBytes { bytes in + Darwin.read(socket, bytes.baseAddress!.advanced(by: completed), count - completed) + } + if amount == 0 { + throw POSIXError(.ECONNRESET) + } + if amount < 0 { + if errno == EINTR { continue } + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + completed += amount + } + return result + } + + private func readReadFD(expectedMarker: UInt8) throws -> Int32 { + var marker: UInt8 = 0 + let headerSize = MemoryLayout.size + let alignment = MemoryLayout.alignment + let align: (Int) -> Int = { value in + (value + alignment - 1) & ~(alignment - 1) + } + let dataOffset = align(headerSize) + let controlSize = dataOffset + align(MemoryLayout.size) + var control = [UInt8](repeating: 0, count: controlSize) + var received = -1 + var receiveError: Int32 = 0 + var receivedFlags: Int32 = 0 + var controlLength = 0 + + while true { + control = [UInt8](repeating: 0, count: controlSize) + received = -1 + receiveError = 0 + controlLength = 0 + withUnsafeMutablePointer(to: &marker) { markerPointer in + var vector = iovec( + iov_base: UnsafeMutableRawPointer(markerPointer), + iov_len: 1 + ) + withUnsafeMutablePointer(to: &vector) { vectorPointer in + control.withUnsafeMutableBytes { controlBytes in + var message = msghdr() + message.msg_iov = vectorPointer + message.msg_iovlen = 1 + message.msg_control = controlBytes.baseAddress + message.msg_controllen = socklen_t(controlBytes.count) + let result = Darwin.recvmsg(socket, &message, 0) + received = result + if result < 0 { + receiveError = errno + } + receivedFlags = message.msg_flags + controlLength = Int(message.msg_controllen) + } + } + } + if received >= 0 || receiveError != EINTR { + break + } + } + + if received < 0 { + throw POSIXError(POSIXErrorCode(rawValue: receiveError) ?? .EIO) + } + let usedControl = min(controlLength, control.count) + var descriptors: [Int32] = [] + var parseError = false + control.withUnsafeBytes { controlBytes in + guard let baseAddress = controlBytes.baseAddress else { + parseError = true + return + } + var offset = 0 + while offset + headerSize <= usedControl { + let header = baseAddress + .advanced(by: offset) + .assumingMemoryBound(to: cmsghdr.self) + .pointee + let messageLength = Int(header.cmsg_len) + guard messageLength >= dataOffset, + messageLength <= usedControl - offset else { + parseError = true + return + } + if header.cmsg_level == SOL_SOCKET && header.cmsg_type == SCM_RIGHTS { + let byteCount = messageLength - dataOffset + guard byteCount >= MemoryLayout.size, + byteCount % MemoryLayout.size == 0 else { + parseError = true + return + } + let dataAddress = baseAddress.advanced(by: offset + dataOffset) + for index in 0..<(byteCount / MemoryLayout.size) { + let descriptor = dataAddress + .advanced(by: index * MemoryLayout.size) + .loadUnaligned(as: Int32.self) + descriptors.append(descriptor) + } + } + let nextOffset = offset + align(messageLength) + guard nextOffset > offset else { + parseError = true + return + } + if nextOffset >= usedControl { + offset = usedControl + break + } + offset = nextOffset + } + if usedControl - offset >= headerSize { + parseError = true + } + } + guard received == 1, + marker == expectedMarker, + receivedFlags & (MSG_CTRUNC | MSG_TRUNC) == 0, + !parseError, + descriptors.count == 1, + descriptors[0] >= 0 else { + for descriptor in descriptors where descriptor >= 0 { + Darwin.close(descriptor) + } + throw POSIXError(.EPROTO) + } + let descriptor = descriptors[0] + if Darwin.fcntl(descriptor, F_SETFD, FD_CLOEXEC) < 0 { + Darwin.close(descriptor) + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + return descriptor + } + + private func writeAll(_ data: Data) throws { + var completed = 0 + while completed < data.count { + let amount = data.withUnsafeBytes { bytes in + Darwin.write(socket, bytes.baseAddress!.advanced(by: completed), data.count - completed) + } + if amount < 0 { + if errno == EINTR { continue } + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + guard amount > 0 else { + throw POSIXError(.EIO) + } + completed += amount + } + } + + private func closeSocket() { + if socket >= 0 { + Darwin.close(socket) + socket = -1 + } + } +} + +final class DaemonClient { + let descriptor: WireDescriptor + private let control: WireConnection + + init(descriptor: WireDescriptor) throws { + self.descriptor = descriptor + control = try WireConnection(descriptor: descriptor) + } + + func newConnection() throws -> WireConnection { + try WireConnection(descriptor: descriptor) + } + + func ping() throws { + _ = try control.request(.ping) + } + + func getattr(_ path: String) throws -> WireEntry { + var writer = WireWriter() + writer.string(path) + var reader = WireReader(try control.request(.getattr, payload: writer.data)) + let entry = try WireEntry(reader: &reader, includesContentGeneration: control.supportsContentGeneration) + try reader.finish() + return entry + } + + func readDir(_ path: String) throws -> [WireEntry] { + var writer = WireWriter() + writer.string(path) + var reader = WireReader(try control.request(.readDir, payload: writer.data)) + let count = Int(try reader.uint32()) + var entries: [WireEntry] = [] + entries.reserveCapacity(count) + for _ in 0.. WireOpenResult { + var writer = WireWriter() + writer.string(path) + writer.uint32(UInt32(bitPattern: flags)) + let response = try connection.requestWithReadFD(.open, payload: writer.data) + var nativeReadFD = response.nativeReadFD + let sharedReadFD = response.sharedReadFD + var sharedWindowFD = response.sharedWindowFD + do { + var reader = WireReader(response.payload) + let handle = try reader.uint64() + var sharedReadWindow: WireSharedReadWindow? + switch response.flags { + case 0: + guard nativeReadFD == nil, + sharedReadFD == nil, + sharedWindowFD == nil else { + throw POSIXError(.EPROTO) + } + case flagNativeReadFD: + guard nativeReadFD != nil, + sharedReadFD == nil, + sharedWindowFD == nil else { + throw POSIXError(.EPROTO) + } + case flagSharedWindow, flagSharedFileWindow: + guard nativeReadFD == nil, + sharedReadFD == nil, + let descriptor = sharedWindowFD else { + throw POSIXError(.EPROTO) + } + let capacity = Int(try reader.uint32()) + sharedWindowFD = nil + if response.flags == flagSharedFileWindow { + sharedReadWindow = try WireSharedReadWindow( + fileDescriptor: descriptor, + capacity: capacity + ) + } else { + sharedReadWindow = try WireSharedReadWindow( + descriptor: descriptor, + capacity: capacity + ) + } + default: + throw POSIXError(.EPROTO) + } + try reader.finish() + let result = WireOpenResult( + handle: handle, + nativeReadFD: nativeReadFD, + sharedReadWindow: sharedReadWindow + ) + nativeReadFD = nil + return result + } catch { + if let nativeReadFD { + Darwin.close(nativeReadFD) + } + if let sharedReadFD { + Darwin.close(sharedReadFD) + } + if let sharedWindowFD { + Darwin.close(sharedWindowFD) + } + throw error + } + } + + func create(_ path: String, flags: Int32) throws -> (WireConnection, UInt64, WireEntry) { + let connection = try newConnection() + var writer = WireWriter() + writer.string(path) + writer.uint32(UInt32(bitPattern: flags)) + var reader = WireReader(try connection.request(.create, payload: writer.data)) + let handle = try reader.uint64() + let entry = try WireEntry(reader: &reader, includesContentGeneration: connection.supportsContentGeneration) + try reader.finish() + return (connection, handle, entry) + } + + func read( + handle: UInt64, + offset: Int64, + length: Int, + connection: WireConnection, + sharedWindow: WireSharedReadWindow? = nil + ) throws -> Data { + guard offset >= 0, length >= 0 else { throw POSIXError(.EINVAL) } + guard length > 0 else { return Data() } + let maximumReadBytes = connection.maximumReadBytes + guard maximumReadBytes > 0 else { throw POSIXError(.EPROTO) } + if length <= maximumReadBytes { + return try readChunk( + handle: handle, + offset: offset, + length: length, + connection: connection, + sharedWindow: sharedWindow + ) + } + var result = Data(capacity: length) + var completed = 0 + while completed < length { + let chunkLength = min(maximumReadBytes, length - completed) + let chunk = try readChunk( + handle: handle, + offset: offset + Int64(completed), + length: chunkLength, + connection: connection, + sharedWindow: sharedWindow + ) + result.append(chunk) + completed += chunk.count + if chunk.count < chunkLength { + break + } + } + return result + } + + func readBorrowingSharedWindow( + handle: UInt64, + offset: Int64, + length: Int, + connection: WireConnection, + sharedWindow: WireSharedReadWindow + ) throws -> WireReadResult { + guard offset >= 0, length > 0, length <= connection.maximumReadBytes else { + throw POSIXError(.EINVAL) + } + var writer = WireWriter() + writer.uint64(handle) + writer.int64(offset) + writer.uint32(UInt32(length)) + return try connection.requestReadBorrowingSharedWindow( + payload: writer.data, + requestedLength: length, + sharedWindow: sharedWindow + ) + } + + private func readChunk( + handle: UInt64, + offset: Int64, + length: Int, + connection: WireConnection, + sharedWindow: WireSharedReadWindow? + ) throws -> Data { + var writer = WireWriter() + writer.uint64(handle) + writer.int64(offset) + writer.uint32(UInt32(length)) + return try connection.requestRead( + payload: writer.data, + requestedLength: length, + sharedWindow: sharedWindow + ) + } + + func write(handle: UInt64, offset: Int64, data: Data, connection: WireConnection) throws -> Int { + var writer = WireWriter() + writer.uint64(handle) + writer.int64(offset) + writer.bytes(data) + var reader = WireReader(try connection.request(.write, payload: writer.data)) + let count = Int(try reader.uint32()) + try reader.finish() + return count + } + + func handleOperation(_ operation: WireOperation, handle: UInt64, connection: WireConnection) throws { + var writer = WireWriter() + writer.uint64(handle) + _ = try connection.request(operation, payload: writer.data) + } + + func truncate(_ path: String, size: UInt64) throws { + guard size <= UInt64(Int64.max) else { throw POSIXError(.EFBIG) } + var writer = WireWriter() + writer.string(path) + writer.int64(Int64(size)) + _ = try control.request(.truncate, payload: writer.data) + } + + func mkdir(_ path: String, mode: UInt32) throws { + var writer = WireWriter() + writer.string(path) + writer.uint32(mode) + _ = try control.request(.mkdir, payload: writer.data) + } + + func rename(_ oldPath: String, _ newPath: String) throws { + var writer = WireWriter() + writer.string(oldPath) + writer.string(newPath) + _ = try control.request(.rename, payload: writer.data) + } + + func remove(_ path: String, directory: Bool) throws { + var writer = WireWriter() + writer.string(path) + _ = try control.request(directory ? .rmdir : .unlink, payload: writer.data) + } + + func statfs() throws -> WireStatFS { + var reader = WireReader(try control.request(.statfs)) + let result = try WireStatFS(reader: &reader) + try reader.finish() + return result + } + + func sync() throws { + _ = try control.request(.sync) + } + + func namespaceVersion() throws -> UInt64 { + var reader = WireReader(try control.request(.namespaceVersion)) + let version = try reader.uint64() + try reader.finish() + return version + } + + func setAttributes( + _ path: String, + valid: UInt32, + mode: UInt32, + uid: UInt32, + gid: UInt32, + accessTime: timespec, + modifyTime: timespec + ) throws { + var writer = WireWriter() + writer.string(path) + writer.uint32(valid) + writer.uint32(mode) + writer.uint32(uid) + writer.uint32(gid) + writer.time(accessTime) + writer.time(modifyTime) + _ = try control.request(.setattr, payload: writer.data) + } + + func getXattr(_ path: String, name: String) throws -> Data { + var writer = WireWriter() + writer.string(path) + writer.string(name) + var reader = WireReader(try control.request(.getXattr, payload: writer.data)) + let value = try reader.bytes(limit: defaultMaxPayload) + try reader.finish() + return value + } + + func setXattr(_ path: String, name: String, value: Data, policy: UInt32) throws { + var writer = WireWriter() + writer.string(path) + writer.string(name) + writer.uint32(policy) + writer.bytes(value) + _ = try control.request(.setXattr, payload: writer.data) + } + + func listXattrs(_ path: String) throws -> [String] { + var writer = WireWriter() + writer.string(path) + var reader = WireReader(try control.request(.listXattrs, payload: writer.data)) + let count = Int(try reader.uint32()) + guard count <= defaultMaxPayload / 4 else { + throw POSIXError(.E2BIG) + } + var attributes: [String] = [] + attributes.reserveCapacity(count) + for _ in 0.. 1 { + do { + exit(try runCommand(Array(CommandLine.arguments.dropFirst()))) + } catch { + fputs("CodexFoldFSKit: \(error)\n", stderr) + exit(1) + } + } + NSApplication.shared.setActivationPolicy(.accessory) + NSApplication.shared.terminate(nil) + } + + private static func runCommand(_ arguments: [String]) throws -> Int32 { + guard let root = FileManager.default.containerURL( + forSecurityApplicationGroupIdentifier: appGroupIdentifier + ) else { + throw POSIXError(.ENOENT) + } + switch arguments.first { + case "--app-group-path": + print(root.path) + return 0 + case "--app-group-write-probe": + let probe = root.appendingPathComponent("host-write-probe", isDirectory: false) + try Data("ok\n".utf8).write(to: probe, options: .atomic) + try FileManager.default.removeItem(at: probe) + return 0 + case "--run-helper": + guard arguments.count >= 2 else { + throw POSIXError(.EINVAL) + } + return try runHelper(executable: arguments[1], arguments: Array(arguments.dropFirst(2))) + default: + throw POSIXError(.EINVAL) + } + } + + private static func runHelper(executable: String, arguments: [String]) throws -> Int32 { + guard executable.hasPrefix("/") else { + throw POSIXError(.EINVAL) + } + let process = Process() + process.executableURL = URL(fileURLWithPath: executable) + process.arguments = arguments + process.standardInput = FileHandle.standardInput + process.standardOutput = FileHandle.standardOutput + process.standardError = FileHandle.standardError + + var environment: [String: String] = [:] + let parentEnvironment = ProcessInfo.processInfo.environment + for key in inheritedEnvironmentKeys { + if let value = parentEnvironment[key] { + environment[key] = value + } + } + environment["CODEXFOLD_LAUNCHER_PARENT_PID"] = String(Darwin.getpid()) + process.environment = environment + + var signalSources: [DispatchSourceSignal] = [] + for signalNumber in [SIGTERM, SIGINT, SIGHUP] { + Darwin.signal(signalNumber, SIG_IGN) + let source = DispatchSource.makeSignalSource(signal: signalNumber, queue: .global()) + source.setEventHandler { + if process.isRunning { + _ = Darwin.kill(process.processIdentifier, signalNumber) + } + } + source.resume() + signalSources.append(source) + } + + try process.run() + process.waitUntilExit() + signalSources.forEach { $0.cancel() } + if process.terminationReason == .uncaughtSignal { + return 128 + process.terminationStatus + } + return process.terminationStatus + } +} diff --git a/platform/darwin/fskit/Host/Info.plist b/platform/darwin/fskit/Host/Info.plist new file mode 100644 index 0000000..3c7a3a8 --- /dev/null +++ b/platform/darwin/fskit/Host/Info.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDisplayName + CodexFold FSKit + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + $(MARKETING_VERSION) + CFBundleVersion + $(CURRENT_PROJECT_VERSION) + LSMinimumSystemVersion + $(MACOSX_DEPLOYMENT_TARGET) + LSUIElement + + + diff --git a/platform/darwin/fskit/Tests/ReadCacheTests.swift b/platform/darwin/fskit/Tests/ReadCacheTests.swift new file mode 100644 index 0000000..38b4255 --- /dev/null +++ b/platform/darwin/fskit/Tests/ReadCacheTests.swift @@ -0,0 +1,466 @@ +import Darwin +import Dispatch +import Foundation + +private enum TestFailure: Error, CustomStringConvertible { + case failed(String) + + var description: String { + switch self { + case .failed(let message): return message + } + } +} + +private func require(_ condition: @autoclosure () -> Bool, _ message: String) throws { + if !condition() { + throw TestFailure.failed(message) + } +} + +private typealias ShmOpenFunction = @convention(c) ( + UnsafePointer, + Int32, + mode_t +) -> Int32 + +private func makePOSIXSharedMemoryDescriptor(capacity: Int) throws -> Int32 { + guard capacity > 0 else { throw POSIXError(.EINVAL) } + guard let library = Darwin.dlopen(nil, RTLD_NOW) else { throw POSIXError(.ENOENT) } + defer { _ = Darwin.dlclose(library) } + guard let symbol = Darwin.dlsym(library, "shm_open") else { throw POSIXError(.ENOSYS) } + let openSharedMemory = unsafeBitCast(symbol, to: ShmOpenFunction.self) + let name = String(format: "/cfs-test-%08x", Darwin.arc4random()) + let descriptor = name.withCString { + openSharedMemory($0, O_RDWR | O_CREAT | O_EXCL, mode_t(S_IRUSR | S_IWUSR)) + } + guard descriptor >= 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + guard name.withCString({ Darwin.shm_unlink($0) }) == 0 else { + let code = errno + Darwin.close(descriptor) + throw POSIXError(POSIXErrorCode(rawValue: code) ?? .EIO) + } + let pageSize = Int(Darwin.getpagesize()) + let mappedLength = ((capacity + pageSize - 1) / pageSize) * pageSize + guard Darwin.ftruncate(descriptor, off_t(mappedLength)) == 0 else { + let code = errno + Darwin.close(descriptor) + throw POSIXError(POSIXErrorCode(rawValue: code) ?? .EIO) + } + return descriptor +} + +private func populateMappedDescriptor(_ descriptor: Int32, bytes: [UInt8]) throws { + let capacity = bytes.count + let pageSize = Int(Darwin.getpagesize()) + let mappedLength = ((capacity + pageSize - 1) / pageSize) * pageSize + let writable = Darwin.mmap(nil, mappedLength, PROT_READ | PROT_WRITE, MAP_SHARED, descriptor, 0) + guard writable != MAP_FAILED, let writable else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + defer { _ = Darwin.munmap(writable, mappedLength) } + _ = bytes.withUnsafeBytes { source in + Darwin.memcpy(writable, source.baseAddress!, capacity) + } +} + +private func makeWindow(bytes: [UInt8]) throws -> WireSharedReadWindow { + let capacity = bytes.count + let descriptor = try makePOSIXSharedMemoryDescriptor(capacity: bytes.count) + do { + try populateMappedDescriptor(descriptor, bytes: bytes) + } catch { + Darwin.close(descriptor) + throw error + } + return try WireSharedReadWindow(descriptor: descriptor, capacity: capacity) +} + +private func makeRegularFileDescriptor(bytes: [UInt8]) throws -> Int32 { + let capacity = bytes.count + let pageSize = Int(Darwin.getpagesize()) + let mappedLength = ((capacity + pageSize - 1) / pageSize) * pageSize + var template = Array("/private/tmp/codexfold-read-cache.XXXXXX".utf8CString) + let descriptor = template.withUnsafeMutableBufferPointer { buffer in + Darwin.mkstemp(buffer.baseAddress!) + } + guard descriptor >= 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + let path = String(cString: template) + _ = path.withCString { Darwin.unlink($0) } + guard Darwin.ftruncate(descriptor, off_t(mappedLength)) == 0 else { + let code = errno + Darwin.close(descriptor) + throw POSIXError(POSIXErrorCode(rawValue: code) ?? .EIO) + } + var completed = 0 + while completed < capacity { + let amount = bytes.withUnsafeBytes { source in + Darwin.pwrite( + descriptor, + source.baseAddress!.advanced(by: completed), + capacity - completed, + off_t(completed) + ) + } + guard amount > 0 else { + let code = errno + Darwin.close(descriptor) + throw POSIXError(POSIXErrorCode(rawValue: code) ?? .EIO) + } + completed += amount + } + return descriptor +} + +private func makeRegularFileWindow(bytes: [UInt8]) throws -> WireSharedReadWindow { + try WireSharedReadWindow( + fileDescriptor: makeRegularFileDescriptor(bytes: bytes), + capacity: bytes.count + ) +} + +private func makeWireEntry( + path: String = "/sessions/example.jsonl", + nodeID: UInt64 = 17, + parentID: UInt64 = 9, + type: WireEntryType = .file, + size: UInt64 = 4096, + allocSize: UInt64 = 4096, + modifyTime: timespec = timespec(tv_sec: 100, tv_nsec: 10), + changeTime: timespec = timespec(tv_sec: 101, tv_nsec: 20), + accessTime: timespec = timespec(tv_sec: 102, tv_nsec: 30), + namespaceID: UInt64 = 1, + contentGeneration: UInt64 = 0 +) throws -> WireEntry { + var writer = WireWriter() + writer.string(path) + writer.string((path as NSString).lastPathComponent) + writer.uint64(nodeID) + writer.uint64(parentID) + writer.uint8(type.rawValue) + writer.uint32(0o600) + writer.uint32(501) + writer.uint32(20) + writer.uint64(size) + writer.uint64(allocSize) + writer.time(modifyTime) + writer.time(changeTime) + writer.time(accessTime) + writer.uint64(namespaceID) + writer.uint64(contentGeneration) + var reader = WireReader(writer.data) + let entry = try WireEntry(reader: &reader, includesContentGeneration: true) + try reader.finish() + return entry +} + +private func testPOSIXSharedMemoryRejectsPreadButSupportsMapping() throws { + let bytes = (0..<4096).map { UInt8(truncatingIfNeeded: $0 * 19) } + let descriptor = try makePOSIXSharedMemoryDescriptor(capacity: bytes.count) + defer { Darwin.close(descriptor) } + try populateMappedDescriptor(descriptor, bytes: bytes) + var byte: UInt8 = 0 + errno = 0 + let amount = Darwin.pread(descriptor, &byte, 1, 0) + try require(amount == -1, "POSIX shared memory unexpectedly accepted pread") + try require(errno == ESPIPE || errno == EIO, "POSIX shared memory pread failed with unexpected errno \(errno)") + + let duplicate = Darwin.dup(descriptor) + guard duplicate >= 0 else { throw POSIXError(.EIO) } + let window = try WireSharedReadWindow(descriptor: duplicate, capacity: bytes.count) + let copied = try window.copyData(count: bytes.count) + try require(copied == Data(bytes), "mapped POSIX shared memory bytes changed") +} + +private func testRegularFileWindowSupportsConcurrentMappedCopies() throws { + let bytes = (0..<(4 * 1024 * 1024)).map { UInt8(truncatingIfNeeded: $0 * 23) } + let window = try makeRegularFileWindow(bytes: bytes) + let workers = DispatchGroup() + let failureLock = NSLock() + var failures: [String] = [] + + for worker in 0..<8 { + workers.enter() + DispatchQueue.global(qos: .userInitiated).async { + let offset = worker * 384 * 1024 + var copied = [UInt8](repeating: 0, count: 512 * 1024) + let copiedCount = copied.count + let success = copied.withUnsafeMutableBytes { destination in + window.copyBytes(from: offset, count: copiedCount, to: destination.baseAddress!) + } + if !success || copied != Array(bytes[offset..<(offset + copied.count)]) { + failureLock.withLock { failures.append("regular-file worker \(worker) observed corrupt bytes") } + } + workers.leave() + } + } + workers.wait() + try require(failureLock.withLock { failures.isEmpty }, failures.joined(separator: "; ")) +} + +private func testRegularFileWindowOwnsDescriptorLifetime() throws { + let bytes = Array("descriptor-lifetime".utf8) + let descriptor = try makeRegularFileDescriptor(bytes: bytes) + var window: WireSharedReadWindow? = try WireSharedReadWindow( + fileDescriptor: descriptor, + capacity: bytes.count + ) + try require(window?.capacity == bytes.count, "regular-file window lost its capacity") + try require(Darwin.fcntl(descriptor, F_GETFD) >= 0, "regular-file descriptor closed during window lifetime") + window = nil + errno = 0 + try require(Darwin.fcntl(descriptor, F_GETFD) == -1 && errno == EBADF, "regular-file descriptor leaked after window release") +} + +private func testRegularFileWindowMappedCopyThroughput() throws { + let windowBytes = 28 * 1024 * 1024 + let readBytes = 4 * 1024 * 1024 + let totalBytes = 256 * 1024 * 1024 + let bytes = (0..= Double(4 * 1024 * 1024 * 1024), "mapped copy throughput \(throughput) B/s is below 4 GiB/s") +} + +private func testSharedWindowCopiesExactRange() throws { + let bytes = (0..<4096).map { UInt8(truncatingIfNeeded: $0 * 17) } + let window = try makeWindow(bytes: bytes) + var releases = 0 + let block = try CodexFoldCachedReadBlock( + sharedWindow: window, + count: bytes.count, + release: { releases += 1 } + ) + var copied = [UInt8](repeating: 0, count: 777) + let copiedCount = copied.count + let success = copied.withUnsafeMutableBytes { destination in + block.copyBytes(from: 901, count: copiedCount, to: destination.baseAddress!) + } + try require(success, "shared-window copy was rejected") + try require(copied == Array(bytes[901..<(901 + copied.count)]), "shared-window bytes changed") + let rejected = copied.withUnsafeMutableBytes { destination in + block.copyBytes(from: bytes.count - 1, count: 2, to: destination.baseAddress!) + } + try require(!rejected, "out-of-range copy succeeded") + try require(releases == 0, "lease released while block remained alive") +} + +private func testReadAheadPolicyKeepsEightBlockHorizonWithEightWorkers() throws { + let policy = CodexFoldReadAheadPolicy(negotiatedReadBytes: 32 * 1024 * 1024) + try require(policy.readAheadBytes == 12 * 1024 * 1024, "read-ahead blocks must align with three 4 MiB reads") + try require(policy.concurrentPrefetchCount == 8, "prefetch concurrency must remain eight") + try require(policy.scheduledPrefetchCount == 8, "prefetch horizon must stay eight blocks ahead") + try require(policy.maxCachedBlocks == 9, "cache must retain the current block plus the eight-block horizon") +} + +private func testNamespaceRefreshRetainsOnlyUnchangedFileData() throws { + let original = try makeWireEntry() + let newNamespaceAndAccessTime = try makeWireEntry( + accessTime: timespec(tv_sec: 999, tv_nsec: 40), + namespaceID: 2 + ) + try require( + original.hasSameCachedFileData(as: newNamespaceAndAccessTime), + "namespace or access-time changes evicted unchanged file data" + ) + let resized = try makeWireEntry(size: 4097, allocSize: 8192) + let modified = try makeWireEntry(modifyTime: timespec(tv_sec: 103, tv_nsec: 10)) + let changed = try makeWireEntry(changeTime: timespec(tv_sec: 104, tv_nsec: 20)) + let moved = try makeWireEntry(path: "/sessions/moved.jsonl") + let replaced = try makeWireEntry(nodeID: 18) + try require( + !original.hasSameCachedFileData(as: resized), + "size changes retained stale file data" + ) + try require( + !original.hasSameCachedFileData(as: modified), + "mtime changes retained stale file data" + ) + try require( + !original.hasSameCachedFileData(as: changed), + "ctime changes retained stale file data" + ) + try require( + !original.hasSameCachedFileData(as: moved), + "path changes retained stale file data" + ) + try require( + !original.hasSameCachedFileData(as: replaced), + "node changes retained stale file data" + ) + let directory = try makeWireEntry(path: "/sessions", type: .directory) + try require( + !directory.hasSameCachedFileData(as: directory), + "directory was treated as cached file data" + ) + try require( + directory.hasSameObjectIdentity(as: directory), + "unchanged directory identity was discarded" + ) + try require( + directory.hasSameCachedDirectoryContents(as: directory), + "unchanged directory contents were discarded" + ) + let changedDirectory = try makeWireEntry( + path: "/sessions", + type: .directory, + modifyTime: timespec(tv_sec: 103, tv_nsec: 10) + ) + try require( + !directory.hasSameCachedDirectoryContents(as: changedDirectory), + "changed directory contents retained a stale name cache" + ) + let movedParent = try makeWireEntry(parentID: 99) + try require( + original.hasSameObjectIdentity(as: movedParent), + "a parent directory generation changed the child's own identity" + ) + try require( + original.hasSameCachedFileData(as: movedParent), + "a parent directory generation evicted unchanged child data" + ) +} + +private func testNormalizedWriteInvalidatesOnlyWhenVisibleLayoutDiverges() throws { + try require( + !writeRequiresKernelCacheInvalidation( + previousSize: 0, + offset: 0, + writtenBytes: 13, + visibleSize: 13 + ), + "ordinary initial write invalidated the kernel cache" + ) + try require( + !writeRequiresKernelCacheInvalidation( + previousSize: 13, + offset: 0, + writtenBytes: 26, + visibleSize: 26 + ), + "first full-page append snapshot invalidated matching cache data" + ) + try require( + writeRequiresKernelCacheInvalidation( + previousSize: 26, + offset: 0, + writtenBytes: 26, + visibleSize: 39 + ), + "normalized append failed to invalidate a shorter kernel snapshot" + ) + try require( + !writeRequiresKernelCacheInvalidation( + previousSize: 1024, + offset: 512, + writtenBytes: 128, + visibleSize: 1024 + ), + "in-place overwrite invalidated an unchanged visible layout" + ) +} + +private func testEvictionWaitsForLastReader() throws { + let bytes = (0..<(2 * 1024 * 1024)).map { UInt8(truncatingIfNeeded: $0 * 31) } + let window = try makeWindow(bytes: bytes) + let releaseLock = NSLock() + var releases = 0 + var cache: [Int64: CodexFoldCachedReadBlock] = [:] + cache[0] = try CodexFoldCachedReadBlock( + sharedWindow: window, + count: bytes.count, + release: { + releaseLock.withLock { releases += 1 } + } + ) + var reader = cache[0] + cache.removeAll() + try require(releaseLock.withLock { releases == 0 }, "eviction released an active reader") + + var copied = [UInt8](repeating: 0, count: 1024 * 1024) + let copiedCount = copied.count + let success = copied.withUnsafeMutableBytes { destination in + reader!.copyBytes(from: 512 * 1024, count: copiedCount, to: destination.baseAddress!) + } + try require(success, "evicted block could not finish its active read") + try require(copied == Array(bytes[(512 * 1024)..<(1536 * 1024)]), "evicted block bytes changed") + reader = nil + try require(releaseLock.withLock { releases == 1 }, "lease did not release after the final reader") +} + +private func testConcurrentReadersRetainLease() throws { + let bytes = (0..<(4 * 1024 * 1024)).map { UInt8(truncatingIfNeeded: $0 * 13) } + let window = try makeWindow(bytes: bytes) + let releaseLock = NSLock() + var releases = 0 + var block: CodexFoldCachedReadBlock? = try CodexFoldCachedReadBlock( + sharedWindow: window, + count: bytes.count, + release: { + releaseLock.withLock { releases += 1 } + } + ) + let start = DispatchSemaphore(value: 0) + let ready = DispatchGroup() + let workers = DispatchGroup() + let failureLock = NSLock() + var failures: [String] = [] + + for worker in 0..<8 { + let retained = block! + ready.enter() + workers.enter() + DispatchQueue.global(qos: .userInitiated).async { + ready.leave() + start.wait() + let offset = worker * 256 * 1024 + var copied = [UInt8](repeating: 0, count: 512 * 1024) + let copiedCount = copied.count + let success = copied.withUnsafeMutableBytes { destination in + retained.copyBytes(from: offset, count: copiedCount, to: destination.baseAddress!) + } + if !success || copied != Array(bytes[offset..<(offset + copied.count)]) { + failureLock.withLock { failures.append("worker \(worker) observed corrupt bytes") } + } + workers.leave() + } + } + ready.wait() + block = nil + try require(releaseLock.withLock { releases == 0 }, "lease released before concurrent readers started") + for _ in 0..<8 { start.signal() } + workers.wait() + try require(failureLock.withLock { failures.isEmpty }, failures.joined(separator: "; ")) + try require(releaseLock.withLock { releases == 1 }, "concurrent readers did not release exactly once") +} + +@main +private struct ReadCacheTests { + static func main() throws { + try testPOSIXSharedMemoryRejectsPreadButSupportsMapping() + try testRegularFileWindowSupportsConcurrentMappedCopies() + try testRegularFileWindowOwnsDescriptorLifetime() + try testRegularFileWindowMappedCopyThroughput() + try testSharedWindowCopiesExactRange() + try testReadAheadPolicyKeepsEightBlockHorizonWithEightWorkers() + try testNamespaceRefreshRetainsOnlyUnchangedFileData() + try testNormalizedWriteInvalidatesOnlyWhenVisibleLayoutDiverges() + try testEvictionWaitsForLastReader() + try testConcurrentReadersRetainLease() + print("ReadCacheTests: PASS") + } +} diff --git a/platform/darwin/fskit/project.yml b/platform/darwin/fskit/project.yml new file mode 100644 index 0000000..cccce89 --- /dev/null +++ b/platform/darwin/fskit/project.yml @@ -0,0 +1,41 @@ +name: CodexFoldFSKit +options: + bundleIdPrefix: vip.jstar.codexfold +settings: + base: + MACOSX_DEPLOYMENT_TARGET: "27.0" + DEVELOPMENT_TEAM: Y987FUR837 + CODE_SIGN_STYLE: Automatic + SWIFT_VERSION: "5.0" + MARKETING_VERSION: "0.3.0" + CURRENT_PROJECT_VERSION: "103" +targets: + CodexFoldFSKit: + type: application + platform: macOS + sources: + - path: Host + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: vip.jstar.codexfold.fskitprofileprobe + INFOPLIST_FILE: Host/Info.plist + CODE_SIGN_ENTITLEMENTS: CodexFoldFSKit.entitlements + REGISTER_APP_GROUPS: YES + dependencies: + - target: CodexFoldFSKitModule + embed: true + CodexFoldFSKitModule: + type: extensionkit-extension + platform: macOS + sources: + - path: Extension + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: vip.jstar.codexfold.fskitprofileprobe.module + INFOPLIST_FILE: Extension/Info.plist + CODE_SIGN_ENTITLEMENTS: Extension/CodexFoldFSKitModule.entitlements + REGISTER_APP_GROUPS: YES + ENABLE_APP_SANDBOX: YES + SKIP_INSTALL: YES + APPLICATION_EXTENSION_API_ONLY: YES + LD_RUNPATH_SEARCH_PATHS: "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks" diff --git a/scripts/activate-canonical-after-codex-exit.sh b/scripts/activate-canonical-after-codex-exit.sh new file mode 100755 index 0000000..4da41e9 --- /dev/null +++ b/scripts/activate-canonical-after-codex-exit.sh @@ -0,0 +1,153 @@ +#!/bin/zsh +set -euo pipefail + +export PATH="/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin" + +CODEX_HOME="${1:?Codex home is required}" +STORE="${2:?CodexFold store is required}" +MOUNT="${3:?mount path is required}" +NATIVE_ROOT="${4:?native root is required}" +BIN="${5:?CodexFold binary is required}" +REOPEN_APP="${6:-1}" +CRITICAL_IDS_FILE="${7:-${CODEXFOLD_CRITICAL_IDS_FILE:-}}" +RUN_ROOT="${STORE}/activation/canonical-$(date '+%Y%m%d-%H%M%S')" + +mkdir -p "${RUN_ROOT}" +exec >"${RUN_ROOT}/run.log" 2>&1 + +activated=0 +finish() { + exit_code=$? + if (( exit_code != 0 )); then + if (( activated == 1 )); then + "${BIN}" fs service stop --apply || true + if "${BIN}" fs namespace deactivate --apply \ + --codex-home "${CODEX_HOME}" --mount "${MOUNT}" --native-root "${NATIVE_ROOT}"; then + "${BIN}" fs service start --apply \ + --codex-home "${CODEX_HOME}" --mount "${MOUNT}" || true + fi + fi + date '+%Y-%m-%dT%H:%M:%S%z' >"${RUN_ROOT}/FAILED" + fi + if [[ "${REOPEN_APP}" == "1" ]]; then + open -a /Applications/ChatGPT.app || true + fi + exit "${exit_code}" +} +trap finish EXIT + +codex_running() { + pgrep -f '/Applications/ChatGPT.app/Contents/MacOS/ChatGPT($| )' >/dev/null 2>&1 || + pgrep -f '/opt/homebrew/(Cellar/codex/[^/]+/bin|bin)/codex($| )' >/dev/null 2>&1 +} + +app_servers_running() { + local pid command_line app_home + while IFS= read -r pid; do + [[ -z "${pid}" ]] && continue + command_line="$(ps eww -p "${pid}" -o command= 2>/dev/null || true)" + [[ -z "${command_line}" ]] && continue + app_home="$(sed -n 's/.* CODEX_HOME=\([^ ]*\).*/\1/p' <<<"${command_line}")" + if [[ -z "${app_home}" || "${app_home}" == "${CODEX_HOME}" ]]; then + return 0 + fi + done < <(pgrep -f '/Applications/ChatGPT.app/Contents/Resources/codex .*app-server' 2>/dev/null || true) + return 1 +} + +wait_for_codex_drain() { + echo "waiting for Codex Desktop and CLI to exit" + while codex_running; do + sleep 2 + done + for _ in 1 2 3; do + sleep 1 + if codex_running; then + while codex_running; do + sleep 2 + done + fi + done + for _ in {1..30}; do + if ! app_servers_running; then + return 0 + fi + sleep 1 + done + echo "real-home Codex app servers did not drain" + return 1 +} + +service_status="$(${BIN} fs service status --json)" +jq -e '.daemon_running == true and .mount_healthy == true' <<<"${service_status}" >/dev/null +compatibility="$(${BIN} fs compatibility --codex-home "${CODEX_HOME}" --store "${STORE}" --json)" +jq -e '.evaluation.approved == true and .evaluation.quarantine == false' <<<"${compatibility}" >/dev/null + +managed_count="$(find "${STORE}/fs/sessions" -type f -name state.json 2>/dev/null | wc -l | tr -d ' ')" +[[ "${managed_count}" == "0" ]] +fold_route_count="$(sqlite3 "${CODEX_HOME}/state_5.sqlite" "select count(*) from threads where rollout_path like '${MOUNT}/%';")" +[[ "${fold_route_count}" == "0" ]] + +snapshot_native_tree() { + local root="$1" + local output="$2" + ( + cd "${root}" + find -H sessions archived_sessions -type f ! -name '._*' \ + -exec stat -f '%N|%z|%m|%i|%p|%u|%g' {} + | LC_ALL=C sort + ) >"${output}" +} + +snapshot_critical() { + local root="$1" + local output="$2" + local id rollout digest + : >"${output}" + [[ -z "${CRITICAL_IDS_FILE}" ]] && return 0 + [[ -f "${CRITICAL_IDS_FILE}" ]] + while IFS= read -r id || [[ -n "${id}" ]]; do + [[ -z "${id}" || "${id}" == \#* ]] && continue + grep -Eq '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' <<<"${id}" + rollout="$(find -H "${root}/sessions" "${root}/archived_sessions" -type f -name "rollout-*-${id}.jsonl" ! -name '._*' -print -quit)" + [[ -n "${rollout}" ]] + digest="$(shasum -a 256 "${rollout}" | awk '{print $1}')" + printf '%s\t%s\n' "${id}" "${digest}" >>"${output}" + done <"${CRITICAL_IDS_FILE}" +} + +while true; do + wait_for_codex_drain + snapshot_native_tree "${CODEX_HOME}" "${RUN_ROOT}/tree.before" + snapshot_critical "${CODEX_HOME}" "${RUN_ROOT}/critical.before" + if codex_running || app_servers_running; then + echo "Codex restarted during activation preflight; waiting again" + continue + fi + break +done + +"${BIN}" fs namespace activate --apply \ + --codex-home "${CODEX_HOME}" --mount "${MOUNT}" --native-root "${NATIVE_ROOT}" --json \ + >"${RUN_ROOT}/activate.json" +activated=1 + +[[ "$(readlink "${CODEX_HOME}/sessions")" == "${MOUNT}/sessions" ]] +[[ "$(readlink "${CODEX_HOME}/archived_sessions")" == "${MOUNT}/archived_sessions" ]] +trigger_count="$(sqlite3 "${CODEX_HOME}/state_5.sqlite" "select count(*) from sqlite_master where type='trigger' and name like 'codexfold_normalize_rollout_path_%';")" +[[ "${trigger_count}" == "2" ]] + +snapshot_native_tree "${NATIVE_ROOT}" "${RUN_ROOT}/tree.native.after" +snapshot_critical "${NATIVE_ROOT}" "${RUN_ROOT}/critical.after" +diff -u "${RUN_ROOT}/tree.before" "${RUN_ROOT}/tree.native.after" +diff -u "${RUN_ROOT}/critical.before" "${RUN_ROOT}/critical.after" + +service_status="$(${BIN} fs service status --json)" +jq -e '.daemon_running == true and .mount_healthy == true' <<<"${service_status}" >/dev/null +namespace_status="$(${BIN} fs namespace status --codex-home "${CODEX_HOME}" --mount "${MOUNT}" --native-root "${NATIVE_ROOT}" --json)" +jq -e '.active == true' <<<"${namespace_status}" >/dev/null + +date '+%Y-%m-%dT%H:%M:%S%z' >"${RUN_ROOT}/COMPLETE" +trap - EXIT +if [[ "${REOPEN_APP}" == "1" ]]; then + open -a /Applications/ChatGPT.app +fi diff --git a/scripts/check-release.sh b/scripts/check-release.sh new file mode 100755 index 0000000..378ed44 --- /dev/null +++ b/scripts/check-release.sh @@ -0,0 +1,32 @@ +#!/bin/sh +set -eu + +root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +cd "$root" + +version=$(tr -d '[:space:]' < VERSION) +expected_tag="v$version" +requested_tag=${1:-$expected_tag} +marketing_version=${version%%-*} + +fail() { + printf 'release metadata error: %s\n' "$1" >&2 + exit 1 +} + +[ "$requested_tag" = "$expected_tag" ] || fail "tag $requested_tag does not match VERSION $version" +grep -Fqx 'module github.com/samekind/codexfold' go.mod || fail "canonical Go module is not samekind/codexfold" +grep -Fq "MARKETING_VERSION: \"$marketing_version\"" platform/darwin/fskit/project.yml || fail "FSKit marketing version does not match $marketing_version" +grep -Fq "## [$version]" CHANGELOG.md || fail "CHANGELOG has no $version entry" +[ -f "docs/releases/$expected_tag.md" ] || fail "release notes for $expected_tag are missing" +grep -Fq "github.com/samekind/codexfold/cmd/codexfold@$expected_tag" README.md || fail "README install command is not pinned to $expected_tag" + +if git grep -n 'github\.com/jstar0/codexfold' -- ':!CHANGELOG.md'; then + fail "legacy Go module references remain" +fi + +if git ls-files --error-unmatch codexfold >/dev/null 2>&1; then + fail "a built codexfold binary is tracked" +fi + +printf 'release metadata: %s\n' "$expected_tag" diff --git a/scripts/prepare-isolated-codex-home.sh b/scripts/prepare-isolated-codex-home.sh new file mode 100755 index 0000000..b6f4b96 --- /dev/null +++ b/scripts/prepare-isolated-codex-home.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + printf 'usage: %s SOURCE_CODEX_HOME TARGET_CODEX_HOME\n' "$0" >&2 + exit 2 +fi + +source_home=$(cd "$1" && pwd) +target_home=$2 + +if [[ "$source_home" == "$target_home" ]]; then + echo "source and target CODEX_HOME must differ" >&2 + exit 2 +fi +if [[ ! -f "$source_home/config.toml" || ! -f "$source_home/auth.json" ]]; then + echo "source CODEX_HOME must contain config.toml and auth.json" >&2 + exit 2 +fi +if [[ -e "$target_home" ]]; then + echo "target CODEX_HOME already exists: $target_home" >&2 + exit 2 +fi + +mkdir -p "$target_home" +chmod 700 "$target_home" + +# Keep provider/auth state byte-identical; only the home directory around it is isolated. +for name in config.toml auth.json; do + cp -p "$source_home/$name" "$target_home/$name" + chmod 600 "$target_home/$name" +done +if [[ -f "$source_home/models_cache.json" ]]; then + cp -p "$source_home/models_cache.json" "$target_home/models_cache.json" + chmod 600 "$target_home/models_cache.json" +fi + +mkdir -p "$target_home/sessions" "$target_home/archived_sessions" + +# These assets are immutable inputs for the canary. APFS clone avoids duplicating their +# contents while copy-on-write keeps a canary update from touching the real home. +for name in plugins skills vendor_sources computer-use; do + if [[ -e "$source_home/$name" ]]; then + cp -cR "$source_home/$name" "$target_home/$name" + fi +done + +printf 'prepared isolated CODEX_HOME: %s\n' "$target_home" diff --git a/scripts/test-cross-platform.sh b/scripts/test-cross-platform.sh new file mode 100755 index 0000000..21bface --- /dev/null +++ b/scripts/test-cross-platform.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -eu + +root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +cd "$root" + +build_dir=$(mktemp -d "${TMPDIR:-/tmp}/codexfold-build.XXXXXX") +trap 'rm -rf "$build_dir"' EXIT HUP INT TERM + +go test ./... -count=1 +go test -race ./... -count=1 +go vet ./... +go build -o "$build_dir/codexfold" ./cmd/codexfold + +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "$build_dir/codexfold-linux-amd64" ./cmd/codexfold +CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o "$build_dir/codexfold-linux-arm64" ./cmd/codexfold +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o "$build_dir/codexfold-windows-amd64.exe" ./cmd/codexfold +CGO_ENABLED=0 GOOS=windows GOARCH=arm64 go build -o "$build_dir/codexfold-windows-arm64.exe" ./cmd/codexfold +CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o "$build_dir/codexfold-darwin-amd64" ./cmd/codexfold +CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o "$build_dir/codexfold-darwin-arm64" ./cmd/codexfold +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -c -o "$build_dir/codexfold-testfs-linux.test" ./internal/testfs +CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go test -c -o "$build_dir/codexfold-testfs-windows.test.exe" ./internal/testfs diff --git a/scripts/test-native-fskit-cache.sh b/scripts/test-native-fskit-cache.sh new file mode 100755 index 0000000..bb2d84a --- /dev/null +++ b/scripts/test-native-fskit-cache.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "$0")/.." && pwd) +out=$(mktemp -d) +trap 'rm -rf "$out"' EXIT + +xcrun swiftc -O \ + -o "$out/codexfold-read-cache-tests" \ + "$root/platform/darwin/fskit/Extension/Wire.swift" \ + "$root/platform/darwin/fskit/Extension/ReadCache.swift" \ + "$root/platform/darwin/fskit/Tests/ReadCacheTests.swift" + +"$out/codexfold-read-cache-tests" diff --git a/scripts/tests/test-activate-canonical-symlink-snapshot.sh b/scripts/tests/test-activate-canonical-symlink-snapshot.sh new file mode 100755 index 0000000..92fee63 --- /dev/null +++ b/scripts/tests/test-activate-canonical-symlink-snapshot.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "$0")/../.." && pwd) +script="$repo_root/scripts/activate-canonical-after-codex-exit.sh" + +grep -Fq 'snapshot_native_tree()' "$script" +grep -Fq "-exec stat -f '%N|%z|%m|%i|%p|%u|%g'" "$script" +! grep -Fq 'snapshot_visible_tree()' "$script" +grep -Fq 'find -H "${root}/sessions" "${root}/archived_sessions"' "$script" +grep -Fq 'shasum -a 256 "${rollout}"' "$script" +[[ "$(grep -Fc 'shasum -a 256' "$script")" == "1" ]] +grep -Fq 'snapshot_critical "${NATIVE_ROOT}" "${RUN_ROOT}/critical.after"' "$script" +grep -Fq 'app_servers_running()' "$script" +grep -Fq 'real-home Codex app servers did not drain' "$script" +grep -Fq 'Codex restarted during activation preflight; waiting again' "$script" + +stop_line=$(grep -n 'fs service stop --apply' "$script" | head -n 1 | cut -d: -f1) +deactivate_line=$(grep -n 'fs namespace deactivate --apply' "$script" | head -n 1 | cut -d: -f1) +start_line=$(grep -n 'fs service start --apply' "$script" | head -n 1 | cut -d: -f1) +[[ -n "$stop_line" && -n "$deactivate_line" && -n "$start_line" ]] +(( stop_line < deactivate_line && deactivate_line < start_line )) + +root=$(mktemp -d) +runtime_root="" +trap 'rm -rf "$root"; [[ -z "$runtime_root" ]] || rm -rf "$runtime_root"' EXIT +mkdir -p "$root/native/sessions/2026/07/13" "$root/native/archived_sessions" +touch "$root/native/sessions/2026/07/13/rollout.jsonl" +ln -s "$root/native/sessions" "$root/sessions" +ln -s "$root/native/archived_sessions" "$root/archived_sessions" + +count=$(cd "$root" && find -H sessions archived_sessions -type f | wc -l | tr -d ' ') +[[ "$count" == "1" ]] + +echo "PASS: activation snapshots traverse canonical session symlinks" + +runtime_root=$(mktemp -d) +mkdir -p "$runtime_root/home/sessions" "$runtime_root/home/archived_sessions" "$runtime_root/store/fs/sessions" "$runtime_root/mount" "$runtime_root/native" "$runtime_root/bin" +sqlite3 "$runtime_root/home/state_5.sqlite" 'create table threads (rollout_path text);' + +cat >"$runtime_root/bin/pgrep" <<'EOF' +#!/bin/sh +if [ -e "$CODEXFOLD_FAKE_REOPEN_MARKER" ]; then + rm -f "$CODEXFOLD_FAKE_REOPEN_MARKER" + exit 0 +fi +exit 1 +EOF +cat >"$runtime_root/bin/sleep" <<'EOF' +#!/bin/sh +exit 0 +EOF +cat >"$runtime_root/bin/find" <<'EOF' +#!/bin/sh +case "$*" in + '-H sessions archived_sessions'*) + if [ ! -e "$CODEXFOLD_FAKE_FIND_TRIGGERED" ]; then + : >"$CODEXFOLD_FAKE_FIND_TRIGGERED" + : >"$CODEXFOLD_FAKE_REOPEN_MARKER" + fi + ;; +esac +exec /usr/bin/find "$@" +EOF +cat >"$runtime_root/bin/shasum" <<'EOF' +#!/bin/sh +echo "full-tree SHA must not run during activation" >&2 +exit 99 +EOF +cat >"$runtime_root/bin/codexfold" <<'EOF' +#!/bin/sh +printf '%s\n' "$*" >>"$CODEXFOLD_FAKE_LOG" +case "$*" in + 'fs service status --json') + printf '%s\n' '{"daemon_running":true,"mount_healthy":true}' + ;; + fs\ compatibility*) + printf '%s\n' '{"evaluation":{"approved":true,"quarantine":false}}' + ;; + 'fs namespace activate'*) + printf '%s\n' '{"active":true}' + ;; +esac +EOF +chmod +x "$runtime_root/bin/pgrep" "$runtime_root/bin/sleep" "$runtime_root/bin/find" "$runtime_root/bin/shasum" "$runtime_root/bin/codexfold" + +runtime_script="$runtime_root/activate.zsh" +sed "s|^export PATH=.*|export PATH=\"$runtime_root/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin\"|" "$script" >"$runtime_script" +runtime_log="$runtime_root/commands.log" +if CODEXFOLD_FAKE_LOG="$runtime_log" \ + CODEXFOLD_FAKE_REOPEN_MARKER="$runtime_root/reopened" \ + CODEXFOLD_FAKE_FIND_TRIGGERED="$runtime_root/find-triggered" \ + /bin/zsh "$runtime_script" \ + "$runtime_root/home" "$runtime_root/store" "$runtime_root/mount" "$runtime_root/native" \ + "$runtime_root/bin/codexfold" 0; then + echo "activation unexpectedly succeeded" >&2 + exit 1 +fi + +grep -Fqx 'fs service stop --apply' "$runtime_log" +grep -Fq 'fs namespace deactivate --apply' "$runtime_log" +grep -Fq 'fs service start --apply' "$runtime_log" +[[ "$(grep -Fc 'fs namespace activate' "$runtime_log")" == "1" ]] +run_log=$(find "$runtime_root/store/activation" -type f -name run.log -print -quit) +if ! grep -Fq 'Codex restarted during activation preflight; waiting again' "$run_log"; then + sed -n '1,120p' "$run_log" >&2 + exit 1 +fi +failed_marker=$(find "$runtime_root/store/activation" -type f -name FAILED -print -quit) +[[ -n "$failed_marker" ]] + +echo "PASS: activation failure trap restores the namespace under zsh" diff --git a/scripts/tests/test-prepare-isolated-codex-home.sh b/scripts/tests/test-prepare-isolated-codex-home.sh new file mode 100755 index 0000000..96605f5 --- /dev/null +++ b/scripts/tests/test-prepare-isolated-codex-home.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PREPARE="$ROOT_DIR/scripts/prepare-isolated-codex-home.sh" +TEST_ROOT="$(mktemp -d)" +trap 'rm -rf "$TEST_ROOT"' EXIT + +source_home="$TEST_ROOT/source" +target_home="$TEST_ROOT/target" +mkdir -p "$source_home/plugins/cache/example" "$source_home/skills/example" +printf 'model_provider = "main"\n[model_providers.main]\nname = "third-party"\nbase_url = "https://third-party.example/v1"\n' > "$source_home/config.toml" +printf '{"access_token":"test-token"}\n' > "$source_home/auth.json" +printf '{"client_version":"test","models":[]}\n' > "$source_home/models_cache.json" +printf 'plugin-source\n' > "$source_home/plugins/cache/example/state" +printf 'skill-source\n' > "$source_home/skills/example/SKILL.md" +chmod 700 "$source_home" +chmod 600 "$source_home/config.toml" "$source_home/auth.json" + +"$PREPARE" "$source_home" "$target_home" >/dev/null + +cmp -s "$source_home/config.toml" "$target_home/config.toml" +cmp -s "$source_home/auth.json" "$target_home/auth.json" +cmp -s "$source_home/models_cache.json" "$target_home/models_cache.json" +[[ "$(stat -f '%Lp' "$target_home/config.toml")" == 600 ]] +[[ "$(stat -f '%Lp' "$target_home/auth.json")" == 600 ]] +[[ "$(stat -f '%Lp' "$target_home/models_cache.json")" == 600 ]] +[[ -d "$target_home/plugins" && ! -L "$target_home/plugins" ]] +printf 'plugin-target\n' > "$target_home/plugins/cache/example/state" +grep -Fqx 'plugin-source' "$source_home/plugins/cache/example/state" + +echo "PASS: isolated CODEX_HOME preserves current provider/auth and clones mutable canary assets" diff --git a/scripts/tests/test-public-scripts-sanitized.sh b/scripts/tests/test-public-scripts-sanitized.sh new file mode 100755 index 0000000..d49407c --- /dev/null +++ b/scripts/tests/test-public-scripts-sanitized.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "$0")/../.." && pwd) + +if rg -n '/Users/jstar|/Volumes/JSData|019[0-9a-f]{5,}|mcxin|jstarctl' "$repo_root/scripts" --glob '!**/test-public-scripts-sanitized.sh'; then + echo "public scripts contain private paths, session IDs, or control-plane names" >&2 + exit 1 +fi + +grep -q 'CRITICAL_IDS_FILE' "$repo_root/scripts/activate-canonical-after-codex-exit.sh" + +echo "PASS: public scripts are sanitized and critical session checks are parameterized"